feat(selfhost): add PostHog error tracking as a parallel sink alongside Sentry

apps/loopover-ui/content/docs/self-hosting-operations.mdx

@@ -549,6 +549,50 @@ changing Sentry env, restart the `loopover` service — there is no hot reload.
   home to a maintainer-owned project unless you configure one.
 </Callout>
 
+## Enabling PostHog error tracking (parallel run alongside Sentry)
+
+PostHog error tracking is **opt-in and off by default**, and runs **in parallel with Sentry
+above** — both sinks are active simultaneously when both are configured; enabling one does not
+disable the other. Leave `POSTHOG_API_KEY` unset for a complete no-op with negligible overhead.
+The same key also activates MCP tool-call telemetry (`src/mcp/telemetry.ts`) — one PostHog
+project covers both surfaces.
+
+<CodeBlock
+  filename=".env"
+  code={`# Your PostHog project API key — never commit this to git
+POSTHOG_API_KEY=phc_examplePostHogProjectKey
+# Optional: EU-cloud ingestion (default is US-cloud, https://us.i.posthog.com)
+# POSTHOG_HOST=https://eu.i.posthog.com
+POSTHOG_ENVIRONMENT=production
+POSTHOG_SERVER_NAME=loopover-us-east
+POSTHOG_RELEASE=loopover-selfhost@2026.07.05
+# Optional: per-repo/global severity floors, independent of SENTRY_MIN_SEVERITY/
+# SENTRY_REPO_MIN_SEVERITY — an operator comparing the two sinks during the parallel-run window
+# may legitimately want PostHog quieter or noisier than Sentry.
+# POSTHOG_MIN_SEVERITY=error
+# POSTHOG_REPO_MIN_SEVERITY={"owner/repo":"warning"}`}
+/>
+
+Every captured event is scrubbed by the same redaction primitives Sentry's `beforeSend` uses
+(`src/selfhost/redaction-scrub.ts`, shared by both sinks so there is one security-critical
+implementation, not two independently-drifting copies): secret-shaped keys/values, JWTs, local
+filesystem paths, and installation ids (hashed, never raw) are all redacted before an event
+leaves the box.
+
+PostHog's own [exception autocapture](https://posthog.com/docs/error-tracking/installation/node)
+is enabled, so genuinely uncaught exceptions and unhandled rejections are captured automatically
+— mirroring Sentry's own default global-handler posture — in addition to the explicit capture
+call sites mirrored throughout the codebase.
+
+There is no PostHog equivalent to Sentry Cron Monitors' missed-check-in alerting, so the five
+recurring jobs (`scheduled-loop`, `orb-export`, `orb-relay-drain`, `orb-relay-register`,
+`queue-dead-letter-revive`) instead emit a plain heartbeat event, `orb_monitor_heartbeat`
+(properties: `monitor`, `status: "ok" | "error"`, `duration_ms`), on every run. Configure a
+PostHog insight alert on a no-data condition over that event (filtered to a given monitor name)
+to reproduce the same "silent death is an alert" property Sentry Crons provides.
+
+After changing PostHog env, restart the `loopover` service — there is no hot reload.
+
 ## Browser Sentry (operator UI)
 
 The operator UI (`apps/loopover-ui`) has its own, separate client-side Sentry

apps/loopover-ui/src/lib/selfhost-env-reference.ts

@@ -429,6 +429,14 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
     name: "PORT",
     firstReference: "src/server.ts",
   },
+  {
+    name: "POSTHOG_ENVIRONMENT",
+    firstReference: "src/server.ts",
+  },
+  {
+    name: "POSTHOG_RELEASE",
+    firstReference: "src/selfhost/posthog.ts",
+  },
   {
     name: "PUBLIC_API_ORIGIN",
     firstReference: "src/selfhost/preflight.ts",
@@ -656,6 +664,8 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [
   "| `PGPOOL_MAX` | `src/selfhost/queue-common.ts` |",
   "| `PGVECTOR_ENABLED` | `src/server.ts` |",
   "| `PORT` | `src/server.ts` |",
+  "| `POSTHOG_ENVIRONMENT` | `src/server.ts` |",
+  "| `POSTHOG_RELEASE` | `src/selfhost/posthog.ts` |",
   "| `PUBLIC_API_ORIGIN` | `src/selfhost/preflight.ts` |",
   "| `PUBLIC_ORIGIN_ACKNOWLEDGED` | `src/server.ts` |",
   "| `PUBLIC_SITE_ORIGIN` | `src/server.ts` |",

src/env.d.ts

@@ -629,15 +629,21 @@ declare global {
      *  token is sufficient — this probe never writes). A secret — never commit a real value. See
      *  CLOUDFLARE_D1_MONITOR_ACCOUNT_ID. */
     CLOUDFLARE_D1_MONITOR_API_TOKEN?: string;
-    /** Opt-in MCP telemetry (#6228/#6235): the PostHog project API key the typed `src/mcp/telemetry.ts`
-     *  wrapper sends anonymized tool-call counters to. Unset (default — every self-hoster who doesn't opt in)
-     *  ⇒ recordMcpToolCall is a safe no-op that records nothing, byte-identical to before this module existed.
-     *  Only the #6228 allowlist is ever sent — tool name, caller type, ok, coarse duration — never arguments,
-     *  source, or any wallet/hotkey/trust-score data. A secret — inject via `wrangler secret`, never commit. */
+    /** Opt-in PostHog project API key, shared by two independent surfaces (#8287's env-var decision): (1) MCP
+     *  telemetry (#6228/#6235) — the typed `src/mcp/telemetry.ts` wrapper sends anonymized tool-call counters
+     *  (tool name, caller type, ok, coarse duration — never arguments, source, or wallet/hotkey/trust-score
+     *  data); (2) self-host ORB error tracking (`src/selfhost/posthog.ts`) — the parallel-run PostHog sink
+     *  alongside SENTRY_DSN, both active simultaneously when both are configured. Unset (default — every
+     *  self-hoster who doesn't opt in) ⇒ both surfaces are safe no-ops, byte-identical to before either
+     *  module existed. A secret — inject via `wrangler secret` (Worker) or a mounted secret file (self-host),
+     *  never commit. */
     POSTHOG_API_KEY?: string;
-    /** Opt-in MCP telemetry host override (#6235): the PostHog ingestion host recordMcpToolCall points at
-     *  (e.g. https://eu.i.posthog.com for EU-cloud). Unset ⇒ the US-cloud default (https://us.i.posthog.com).
-     *  Only meaningful alongside POSTHOG_API_KEY; ignored when telemetry is unconfigured. */
+    /** PostHog ingestion host override (e.g. https://eu.i.posthog.com for EU-cloud), shared by both surfaces
+     *  POSTHOG_API_KEY activates. Unset ⇒ the US-cloud default (https://us.i.posthog.com). Only meaningful
+     *  alongside POSTHOG_API_KEY; ignored when unconfigured. Self-host error tracking also reads its own
+     *  process.env-only vars (POSTHOG_MIN_SEVERITY, POSTHOG_REPO_MIN_SEVERITY, POSTHOG_ENVIRONMENT,
+     *  POSTHOG_SERVER_NAME, POSTHOG_RELEASE) — not declared here, mirroring SENTRY_MIN_SEVERITY/
+     *  SENTRY_REPO_MIN_SEVERITY's identical self-host-only precedent (src/selfhost/sentry.ts). */
     POSTHOG_HOST?: string;
   }
 }

src/queue/ai-review-orchestration.ts

@@ -59,6 +59,7 @@ import {
   resolveEnrichmentLinkedIssueNumbers,
 } from "../review/enrichment-wire";
 import { captureReviewFailure } from "../selfhost/sentry";
+import { capturePostHogReviewFailure } from "../selfhost/posthog";
 import { isReputationEnabled, shouldSkipAiForReputation } from "../review/reputation-wire";
 import { isConvergenceRepoAllowed } from "../review/cutover-gate";
 import { resolveConvergedFeature } from "../review/feature-activation";
@@ -822,6 +823,20 @@ export async function runAiReviewForAdvisory(
         /* v8 ignore next -- current review runner always supplies diagnostics for completed AI attempts. */
         review_diagnostics: formatReviewDiagnosticsForCapture(result.reviewDiagnostics ?? []),
       }, "ai_review_inconclusive");
+      capturePostHogReviewFailure(new Error("AI review inconclusive — no usable verdict for the PR head"), {
+        kind: "review",
+        reason: "ai_review_inconclusive",
+        installationId: args.installationId,
+        owner: args.repoFullName.split("/")[0],
+        repo: args.repoFullName,
+        pr: args.pr.number,
+        head_sha: args.advisory.headSha,
+        ai_review_mode: args.settings.aiReviewMode,
+        reviewer_count: result.reviewerCount,
+        public_notes: hasPublicReviewAssessment(result.advisoryNotes),
+        /* v8 ignore next -- current review runner always supplies diagnostics for completed AI attempts. */
+        review_diagnostics: formatReviewDiagnosticsForCapture(result.reviewDiagnostics ?? []),
+      }, "ai_review_inconclusive");
     }
     args.advisory.findings.push(...findings);
     const metadataFor = (
@@ -900,6 +915,27 @@ export async function runAiReviewForAdvisory(
       },
       "ai_review_public_summary_missing",
     );
+    capturePostHogReviewFailure(
+      new Error("AI review did not produce public notes for the PR head"),
+      {
+        kind: "review",
+        reason: "ai_review_public_summary_missing",
+        installationId: args.installationId,
+        owner: args.repoFullName.split("/")[0],
+        repo: args.repoFullName,
+        pr: args.pr.number,
+        head_sha: args.advisory.headSha,
+        ai_review_mode: args.settings.aiReviewMode,
+        reviewer_count: result.reviewerCount,
+        /* v8 ignore next -- current review runner always supplies diagnostics for completed AI attempts. */
+        review_diagnostics: formatReviewDiagnosticsForCapture(result.reviewDiagnostics ?? []),
+        configured_reviewers:
+          env.AI_REVIEW_PLAN?.reviewers?.map((reviewer) => reviewer.model) ??
+          null,
+        combine: env.AI_REVIEW_PLAN?.combine ?? null,
+      },
+      "ai_review_public_summary_missing",
+    );
     return {
       notes:
         "AI review is unavailable for this PR head. LoopOver is holding this PR for manual review until the configured AI provider returns a usable public review summary.",
@@ -929,6 +965,13 @@ export async function runAiReviewForAdvisory(
       pr: args.pr.number,
       head_sha: args.advisory.headSha,
     }, "ai_review_failed");
+    capturePostHogReviewFailure(error, {
+      kind: "review",
+      installationId: args.installationId,
+      repo: args.repoFullName,
+      pr: args.pr.number,
+      head_sha: args.advisory.headSha,
+    }, "ai_review_failed");
     return undefined;
   } finally {
     // #regate-dup-prep: only release a lock THIS call actually claimed. A caller-supplied

src/queue/processors.ts

@@ -585,6 +585,7 @@ import { shouldApplyRepoCultureProfile } from "../review/repo-culture-profile-wi
 import { applyReviewMemorySuppression, getCachedReviewSuppressions, invalidateReviewSuppressionCache, shouldApplyReviewMemory } from "../review/review-memory-wire";
 import { isEnrichmentEnabled } from "../review/enrichment-wire";
 import { captureReviewFailure } from "../selfhost/sentry";
+import { capturePostHogReviewFailure } from "../selfhost/posthog";
 import {
   setReviewPipelineSpanOutcome,
   withReviewPipelineSpan,
@@ -9168,6 +9169,15 @@ async function maybePublishPrPublicSurface(
           head_sha: advisory.headSha,
           failedOutputs: failedOutputs.map((failure) => failure.output),
         }, "pr_public_surface_publish_failed");
+        capturePostHogReviewFailure(new Error("PR public-surface publish failed — review produced output but nothing was posted to the PR"), {
+          kind: "publish",
+          installationId,
+          owner: repoFullName.split("/")[0],
+          repo: repoFullName,
+          pr: pr.number,
+          head_sha: advisory.headSha,
+          failedOutputs: failedOutputs.map((failure) => failure.output),
+        }, "pr_public_surface_publish_failed");
         // At least one output failed for a reason that can plausibly clear on its own (rate limit / 5xx / momentary
         // token issue) — retry the whole job instead of leaving the review permanently unposted. A mix of transient
         // and permanent failures still retries: the permanent one re-fails identically next pass and re-audits, but
@@ -10367,6 +10377,16 @@ async function maybePublishPrPublicSurface(
         reviewer_count: aiReview?.reviewerCount ?? 0,
         public_notes: hasPublicReviewAssessment(aiReview?.notes),
       }, "ai_review_public_summary_missing");
+      capturePostHogReviewFailure(new Error(message), {
+        kind: "review",
+        reason: "ai_review_public_summary_missing",
+        installationId,
+        repo: repoFullName,
+        pr: pr.number,
+        head_sha: advisory.headSha,
+        reviewer_count: aiReview?.reviewerCount ?? 0,
+        public_notes: hasPublicReviewAssessment(aiReview?.notes),
+      }, "ai_review_public_summary_missing");
     }
 
     // Secrets-scan (#audit-3.4): always scans the REAL resolved diff and, on a CONCRETE credential hit, appends a

src/selfhost/monitored-work.ts

@@ -2,6 +2,20 @@ import type { EnqueueWebhookResult } from "../github/webhook";
 import { ORB_RELAY_REGISTER_UNHEALTHY_FAILURE_STREAK, type OrbRelayRegistrationState } from "../orb/broker-client";
 import { incr } from "./metrics";
 import { withSentryMonitor } from "./sentry";
+import { withPostHogMonitor } from "./posthog";
+
+/** Run both monitor wrappers around one callback: Sentry's structured check-in + PostHog's heartbeat event,
+ *  in parallel (#8287) -- neither observes the other's outcome, each independently no-ops when its own sink
+ *  is unconfigured. The callback itself only runs ONCE; withPostHogMonitor wraps a thunk that replays whatever
+ *  withSentryMonitor already produced/threw, so a failure is captured by both sinks without executing the
+ *  underlying work twice. */
+async function withBothMonitors<T>(
+  name: Parameters<typeof withSentryMonitor>[0] & Parameters<typeof withPostHogMonitor>[0],
+  context: Record<string, unknown> | undefined,
+  callback: () => Promise<T>,
+): Promise<T> {
+  return withPostHogMonitor(name, context, () => withSentryMonitor(name, context, callback));
+}
 
 export type OrbRelayEvent = {
   deliveryId: string;
@@ -59,7 +73,7 @@ export async function runScheduledLoopWithMonitor<T>(
   cron: string,
   scheduled: () => T | Promise<T>,
 ): Promise<T> {
-  return withSentryMonitor(
+  return withBothMonitors(
     "scheduled-loop",
     { jobType: "scheduled-loop", cron },
     () => Promise.resolve(scheduled()),
@@ -70,7 +84,7 @@ export async function runOrbExportWithMonitor(
   exportBatch: () => Promise<number>,
   log: (line: string) => void = console.log,
 ): Promise<void> {
-  await withSentryMonitor("orb-export", { jobType: "orb-export" }, async () => {
+  await withBothMonitors("orb-export", { jobType: "orb-export" }, async () => {
     const exported = await exportBatch();
     if (exported > 0)
       log(JSON.stringify({ event: "selfhost_orb_export", exported }));
@@ -91,7 +105,7 @@ export async function drainOrbRelayWithMonitor(args: {
   log?: (line: string) => void;
   nowMs?: number;
 }): Promise<void> {
-  await withSentryMonitor(
+  await withBothMonitors(
     "orb-relay-drain",
     { jobType: "orb-relay-drain", pendingAckCount: args.state.pendingAck.length },
     async () => {
@@ -247,7 +261,7 @@ export async function registerOrbRelayWithMonitor(args: {
   log?: (line: string) => void;
   nowMs?: number;
 }): Promise<void> {
-  await withSentryMonitor("orb-relay-register", { jobType: "orb-relay-register" }, async () => {
+  await withBothMonitors("orb-relay-register", { jobType: "orb-relay-register" }, async () => {
     const result = await args.register(args.env, args.state);
     if (result.status === "skipped" || result.status === "already_registered" || result.status === "backoff") return;
     const mode = args.env.ORB_RELAY_MODE === "pull" ? "pull" : "push";

src/selfhost/pg-queue.ts

@@ -9,6 +9,7 @@ import { incr } from "./metrics";
 import { withReviewSpan } from "./tracing";
 import { withOtelSpan } from "./otel";
 import { captureError, withSentryMonitor } from "./sentry";
+import { capturePostHogError, withPostHogMonitor } from "./posthog";
 import {
   consumingRetryDelayMs,
   deterministicJitterMs,
@@ -632,10 +633,15 @@ export function createPgQueue(
    *  fires; the outer try/catch (this function's actual job) still guards the setInterval callback. */
   async function reviveDeadLetterJobsSafely(): Promise<void> {
     try {
-      await withSentryMonitor(
+      await withPostHogMonitor(
         "queue-dead-letter-revive",
         { jobType: "queue-dead-letter-revive" },
-        reviveDeadLetterJobs,
+        () =>
+          withSentryMonitor(
+            "queue-dead-letter-revive",
+            { jobType: "queue-dead-letter-revive" },
+            reviveDeadLetterJobs,
+          ),
       );
     } catch (error) {
       console.error(
@@ -646,6 +652,7 @@ export function createPgQueue(
         }),
       );
       captureError(error, { kind: "queue_dead_letter_revive_crashed" }, "queue_dead_letter_revive_crashed");
+      capturePostHogError(error, { kind: "queue_dead_letter_revive_crashed" }, "queue_dead_letter_revive_crashed");
     }
   }
 
@@ -772,6 +779,7 @@ export function createPgQueue(
         }),
       );
       captureError(error, { kind: "queue_foreground_liveness_release_crashed" }, "queue_foreground_liveness_release_crashed");
+      capturePostHogError(error, { kind: "queue_foreground_liveness_release_crashed" }, "queue_foreground_liveness_release_crashed");
     }
   }
 
@@ -1102,6 +1110,12 @@ export function createPgQueue(
         recovered,
         timeoutMs: processingTimeoutMs,
       }, "processing_timeout");
+      capturePostHogError(new Error("self-host queue processing lease expired"), {
+        kind: "job_recovered",
+        reason: "processing_timeout",
+        recovered,
+        timeoutMs: processingTimeoutMs,
+      }, "processing_timeout");
     }
     const job = await claimNext();
     if (!job) return false;
@@ -1130,6 +1144,11 @@ export function createPgQueue(
           reason: "unparseable_payload",
           jobId: job.id,
         }, "unparseable_payload");
+        capturePostHogError(new Error("unparseable queue payload"), {
+          kind: "job_dead",
+          reason: "unparseable_payload",
+          jobId: job.id,
+        }, "unparseable_payload");
         return true;
       }
       const jobTraceParent = message.type === "github-webhook" ? message.traceParent : undefined;
@@ -1441,6 +1460,13 @@ export function createPgQueue(
             jobId: job.id,
             attempts,
           }, "job_dead");
+          capturePostHogError(error, {
+            kind: "job_dead",
+            reason: "max_retries_exhausted",
+            jobType: extractPayloadType(job.payload),
+            jobId: job.id,
+            attempts,
+          }, "job_dead");
         } else {
           const retryDelayMs = consumingRetryDelayMs(error, backoff(attempts));
           await pool.query(
@@ -1488,6 +1514,7 @@ export function createPgQueue(
         }),
       );
       captureError(error, { kind: "queue_pump_crashed" }, "queue_pump_crashed");
+      capturePostHogError(error, { kind: "queue_pump_crashed" }, "queue_pump_crashed");
     } finally {
       active--;
     }