Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions src/app/services/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1778,7 +1778,8 @@ export interface HotPRStatusUpdate {
*/
export async function fetchHotPRStatus(
octokit: GitHubOctokit,
nodeIds: string[]
nodeIds: string[],
options?: { reportToSentry?: boolean }
): Promise<{ results: Map<number, HotPRStatusUpdate>; hadErrors: boolean }> {
const results = new Map<number, HotPRStatusUpdate>();
if (nodeIds.length === 0) return { results, hadErrors: false };
Expand Down Expand Up @@ -1814,7 +1815,9 @@ export async function fetchHotPRStatus(
if (isUnauthorizedError(s.reason)) throw s.reason;
hadErrors = true;
console.warn("[hot-poll] PR status batch failed:", s.reason);
Sentry.captureException(s.reason, { tags: { source: "hot-poll-pr-batch" } });
if (options?.reportToSentry) {
Sentry.captureException(s.reason, { tags: { source: "hot-poll-pr-batch" } });
}
const reason = s.reason;
const partialErr = (reason && typeof reason === "object" && "data" in reason && reason.data && typeof reason.data === "object")
? reason.data as Partial<HotPRStatusResponse>
Expand Down
24 changes: 19 additions & 5 deletions src/app/services/poll.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,11 @@ const _hotRuns = new Map<number, { owner: string; repo: string }>();
const MAX_HOT_RUNS = 30;
const HOT_RUNS_CONCURRENCY = 10;

/** Consecutive prior hot-poll cycle failures required before a fetch error is
* reported to Sentry. Backoff/retry already self-heals isolated network blips
* (e.g. TypeError: Failed to fetch) — only escalate once it's clearly stuck. */
const HOT_POLL_SENTRY_THRESHOLD = 3;

/** Incremented each time rebuildHotSets() is called (full refresh completed).
* Allows hot poll callbacks to detect stale results that overlap with a fresh
* full refresh — if the captured generation no longer matches the current one,
Expand Down Expand Up @@ -465,13 +470,18 @@ export function rebuildHotSets(data: DashboardData): void {
* Evicts items from the hot sets when they settle (PR closed/merged/resolved,
* run completed). Returns captured generation alongside results so callers can
* detect staleness.
*
* @param priorConsecutiveFailures - Failures from prior cycles (before this one).
* Below HOT_POLL_SENTRY_THRESHOLD, fetch errors are logged/surfaced to the user
* but not reported to Sentry — retry/backoff already self-heals isolated blips.
*/
export async function fetchHotData(): Promise<{
export async function fetchHotData(priorConsecutiveFailures = 0): Promise<{
prUpdates: Map<number, HotPRStatusUpdate>;
runUpdates: Map<number, HotWorkflowRunUpdate>;
generation: number;
hadErrors: boolean;
}> {
const reportToSentry = priorConsecutiveFailures >= HOT_POLL_SENTRY_THRESHOLD;
// Capture generation BEFORE any async work so callers can detect if a full
// refresh occurred while this fetch was in flight.
const generation = _hotPollGeneration;
Expand All @@ -488,7 +498,7 @@ export async function fetchHotData(): Promise<{
// PR status fetch — wrap in try/catch so failures don't crash the hot poll
const nodeIds = [..._hotPRs.keys()];
try {
const prResult = await fetchHotPRStatus(octokit, nodeIds);
const prResult = await fetchHotPRStatus(octokit, nodeIds, { reportToSentry });
if (prResult.hadErrors) hadErrors = true;
for (const [id, update] of prResult.results) {
prUpdates.set(id, update);
Expand All @@ -497,7 +507,9 @@ export async function fetchHotData(): Promise<{
if (isUnauthorizedError(err)) throw err;
hadErrors = true;
console.warn("[hot-poll] PR status fetch failed:", err);
Sentry.captureException(err, { tags: { source: "hot-poll-pr-fetch" } });
if (reportToSentry) {
Sentry.captureException(err, { tags: { source: "hot-poll-pr-fetch" } });
}
// Items stay in _hotPRs for retry next cycle
}

Expand All @@ -514,7 +526,9 @@ export async function fetchHotData(): Promise<{
if (isUnauthorizedError(result.reason)) throw result.reason;
hadErrors = true;
console.warn("[hot-poll] Workflow run fetch failed:", result.reason);
Sentry.captureException(result.reason, { tags: { source: "hot-poll-run-fetch" } });
if (reportToSentry) {
Sentry.captureException(result.reason, { tags: { source: "hot-poll-run-fetch" } });
}
}
}

Expand Down Expand Up @@ -629,7 +643,7 @@ export function createHotPollCoordinator(
startedCycle = true;
options?.onStart?.(new Set(_hotPRs.values()), new Set(_hotRuns.keys()));
try {
const { prUpdates, runUpdates, generation, hadErrors } = await fetchHotData();
const { prUpdates, runUpdates, generation, hadErrors } = await fetchHotData(consecutiveFailures);
if (myGeneration !== chainGeneration) return; // Chain destroyed during fetch
if (hadErrors) {
consecutiveFailures++;
Expand Down
157 changes: 157 additions & 0 deletions tests/services/hot-poll.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,10 @@ vi.mock("../../src/app/lib/notifications", () => ({
_resetNotificationState: vi.fn(),
}));

vi.mock("@sentry/solid", () => ({
captureException: vi.fn(),
}));

vi.mock("../../src/app/stores/config", () => ({
config: {
selectedRepos: [],
Expand Down Expand Up @@ -704,6 +708,49 @@ describe("createHotPollCoordinator", () => {
});
});

it("only reports to Sentry once consecutiveFailures reaches the threshold across real cycles", async () => {
const onHotData = vi.fn();
const graphqlFn = vi.fn(() => Promise.reject(new Error("api error")));
const octokit = makeOctokit(undefined, graphqlFn);
mockGetClient.mockReturnValue(octokit);

rebuildHotSets({
...emptyData,
pullRequests: [makePullRequest({ id: 1, checkStatus: "pending", enriched: true, nodeId: "PR_a" })],
});

const Sentry = await import("@sentry/solid");
(Sentry.captureException as ReturnType<typeof vi.fn>).mockClear();

await createRoot(async (dispose) => {
createHotPollCoordinator(() => 10, onHotData);

// Cycle 1 (priorConsecutiveFailures=0) at 10s — below threshold
await vi.advanceTimersByTimeAsync(10_000);
expect(graphqlFn.mock.calls.length).toBe(1);
expect(Sentry.captureException).not.toHaveBeenCalled();

// Cycle 2 (priorConsecutiveFailures=1) at +20s (10s * 2^1) — below threshold
await vi.advanceTimersByTimeAsync(20_000);
expect(graphqlFn.mock.calls.length).toBe(2);
expect(Sentry.captureException).not.toHaveBeenCalled();

// Cycle 3 (priorConsecutiveFailures=2) at +40s (10s * 2^2) — below threshold
await vi.advanceTimersByTimeAsync(40_000);
expect(graphqlFn.mock.calls.length).toBe(3);
expect(Sentry.captureException).not.toHaveBeenCalled();

// Cycle 4 (priorConsecutiveFailures=3) at +80s (10s * min(2^3, 8)) — threshold met
await vi.advanceTimersByTimeAsync(80_000);
expect(graphqlFn.mock.calls.length).toBe(4);
expect(Sentry.captureException).toHaveBeenCalledWith(
expect.any(Error),
{ tags: { source: "hot-poll-pr-batch" } }
);
dispose();
});
});

it("silently reschedules when getClient throws", async () => {
const onHotData = vi.fn();
// getClient() throw is caught by the pre-onStart guard — schedules next cycle without pushError
Expand Down Expand Up @@ -993,6 +1040,40 @@ describe("fetchHotPRStatus edge cases", () => {

await expect(fetchHotPRStatus(octokit as never, ["PR_a"])).rejects.toMatchObject({ status: 401 });
});

it("does not report to Sentry when reportToSentry option is omitted", async () => {
const Sentry = await import("@sentry/solid");
(Sentry.captureException as ReturnType<typeof vi.fn>).mockClear();
const octokit = makeOctokit(undefined, () => Promise.reject(new Error("api error")));

await fetchHotPRStatus(octokit as never, ["PR_a"]);

expect(Sentry.captureException).not.toHaveBeenCalled();
});

it("does not report to Sentry when reportToSentry is false", async () => {
const Sentry = await import("@sentry/solid");
(Sentry.captureException as ReturnType<typeof vi.fn>).mockClear();
const octokit = makeOctokit(undefined, () => Promise.reject(new Error("api error")));

await fetchHotPRStatus(octokit as never, ["PR_a"], { reportToSentry: false });

expect(Sentry.captureException).not.toHaveBeenCalled();
});

it("reports to Sentry when reportToSentry is true", async () => {
const Sentry = await import("@sentry/solid");
(Sentry.captureException as ReturnType<typeof vi.fn>).mockClear();
const error = new Error("api error");
const octokit = makeOctokit(undefined, () => Promise.reject(error));

await fetchHotPRStatus(octokit as never, ["PR_a"], { reportToSentry: true });

expect(Sentry.captureException).toHaveBeenCalledWith(
error,
{ tags: { source: "hot-poll-pr-batch" } }
);
});
});

describe("rebuildHotSets caps", () => {
Expand Down Expand Up @@ -1258,6 +1339,82 @@ describe("fetchHotData hadErrors", () => {

await expect(fetchHotData()).rejects.toMatchObject({ status: 401 });
});

it("does not report PR fetch failure to Sentry below HOT_POLL_SENTRY_THRESHOLD", async () => {
const Sentry = await import("@sentry/solid");
(Sentry.captureException as ReturnType<typeof vi.fn>).mockClear();
const octokit = makeOctokit(undefined, () => Promise.reject(new Error("graphql error")));
mockGetClient.mockReturnValue(octokit);

rebuildHotSets({
...emptyData,
pullRequests: [makePullRequest({ id: 1, checkStatus: "pending", enriched: true, nodeId: "PR_a" })],
});

await fetchHotData(2); // 2 prior failures — still below the threshold of 3

expect(Sentry.captureException).not.toHaveBeenCalled();
});

it("reports PR fetch failure to Sentry once priorConsecutiveFailures reaches HOT_POLL_SENTRY_THRESHOLD", async () => {
const Sentry = await import("@sentry/solid");
(Sentry.captureException as ReturnType<typeof vi.fn>).mockClear();
const octokit = makeOctokit(undefined, () => Promise.reject(new Error("graphql error")));
mockGetClient.mockReturnValue(octokit);

rebuildHotSets({
...emptyData,
pullRequests: [makePullRequest({ id: 1, checkStatus: "pending", enriched: true, nodeId: "PR_a" })],
});

await fetchHotData(3); // 3 prior failures — meets the threshold

expect(Sentry.captureException).toHaveBeenCalledWith(
expect.any(Error),
{ tags: { source: "hot-poll-pr-batch" } }
);
});

it("does not report run fetch failure to Sentry below HOT_POLL_SENTRY_THRESHOLD", async () => {
const Sentry = await import("@sentry/solid");
(Sentry.captureException as ReturnType<typeof vi.fn>).mockClear();
const octokit = makeOctokit(
() => Promise.reject(new Error("network error")),
() => Promise.resolve({ nodes: [], rateLimit: { limit: 5000, remaining: 4999, resetAt: "2026-01-01T00:00:00Z" } }),
);
mockGetClient.mockReturnValue(octokit);

rebuildHotSets({
...emptyData,
workflowRuns: [makeWorkflowRun({ id: 10, status: "in_progress", conclusion: null, repoFullName: "o/r" })],
});

await fetchHotData(2); // still below threshold

expect(Sentry.captureException).not.toHaveBeenCalled();
});

it("reports run fetch failure to Sentry once priorConsecutiveFailures reaches HOT_POLL_SENTRY_THRESHOLD", async () => {
const Sentry = await import("@sentry/solid");
(Sentry.captureException as ReturnType<typeof vi.fn>).mockClear();
const octokit = makeOctokit(
() => Promise.reject(new Error("network error")),
() => Promise.resolve({ nodes: [], rateLimit: { limit: 5000, remaining: 4999, resetAt: "2026-01-01T00:00:00Z" } }),
);
mockGetClient.mockReturnValue(octokit);

rebuildHotSets({
...emptyData,
workflowRuns: [makeWorkflowRun({ id: 10, status: "in_progress", conclusion: null, repoFullName: "o/r" })],
});

await fetchHotData(3); // meets threshold

expect(Sentry.captureException).toHaveBeenCalledWith(
expect.any(Error),
{ tags: { source: "hot-poll-run-fetch" } }
);
});
});

describe("fetchHotPRStatus updateGraphqlRateLimit", () => {
Expand Down