Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
-- BLO-20649: `checkout` promotes an issue to `in_progress` on entry, but every
-- lock-release path clears only the execution-lock columns and leaves `status`
-- behind, so `in_progress` degrades into a high-water mark of every issue any
-- wake has ever touched.
--
-- Record the status the issue held immediately before checkout so a release that
-- did not advance the issue can put it back exactly (a `backlog` issue returns to
-- `backlog`, not `todo`). NULL means "no checkout-promotion to undo".
ALTER TABLE "issues" ADD COLUMN "checkout_restore_status" text;
7 changes: 7 additions & 0 deletions packages/db/src/migrations/meta/_journal.json
Original file line number Diff line number Diff line change
Expand Up @@ -1422,6 +1422,13 @@
"when": 1785514532668,
"tag": "0210_approval_linked_agent",
"breakpoints": true
},
{
"idx": 211,
"version": "7",
"when": 1785419604000,
"tag": "0211_issue_checkout_restore_status",
"breakpoints": true
}
]
}
6 changes: 6 additions & 0 deletions packages/db/src/schema/issues.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,12 @@ export const issues = pgTable(
executionRunId: uuid("execution_run_id").references(() => heartbeatRuns.id, { onDelete: "set null" }),
executionAgentNameKey: text("execution_agent_name_key"),
executionLockedAt: timestamp("execution_locked_at", { withTimezone: true }),
// Status held immediately before `checkout` promoted this row to
// `in_progress`. A release that did not advance the issue restores it from
// here, so `in_progress` stops accumulating as a high-water mark. Null means
// there is no checkout promotion to undo — either the row was never checked
// out, or a run has since written a status of its own. See BLO-20649.
checkoutRestoreStatus: text("checkout_restore_status"),
createdByAgentId: uuid("created_by_agent_id").references(() => agents.id),
createdByUserId: text("created_by_user_id"),
responsibleUserId: text("responsible_user_id"),
Expand Down
119 changes: 119 additions & 0 deletions server/src/__tests__/execution-lock-orphan-cleanup.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -458,4 +458,123 @@ describeEmbeddedPostgres("execution lock orphan cleanup", () => {
expect(unrelatedAfter?.executionAgentNameKey).toBe("se1");
});
});

// BLO-20649: `checkout` promotes an issue to `in_progress` and records what it
// displaced in `checkout_restore_status`. `releaseIssueExecutionAndPromote` is
// the primary terminal-run finalizer, so it is the path that actually decides
// whether `in_progress` is a statement about live work or a high-water mark of
// every issue any wake ever touched. These drive the real finalizer via
// `cancelRun` rather than calling the restore helper directly.
describe("checkout status restore through run finalization", () => {
// `issues.checkout_run_id` / `execution_run_id` are FKs, so the run row has
// to exist before any issue can point at it.
async function seedRun(
companyId: string,
agentId: string,
runId: string,
overrides: { status?: string; contextSnapshot?: Record<string, unknown> } = {},
) {
await db.insert(heartbeatRuns).values({
id: runId,
companyId,
agentId,
invocationSource: "assignment",
status: overrides.status ?? "queued",
...(overrides.contextSnapshot ? { contextSnapshot: overrides.contextSnapshot } : {}),
} as typeof heartbeatRuns.$inferInsert);
}

async function seedPromotedIssue(
companyId: string,
runId: string,
restoreStatus: "todo" | "backlog" | null,
) {
return seedIssue(companyId, {
status: "in_progress",
checkoutRestoreStatus: restoreStatus,
checkoutRunId: runId,
executionRunId: runId,
executionAgentNameKey: "ceo",
executionLockedAt: new Date(),
});
}

it("returns a checkout-promoted issue to its pre-checkout status when the run finalizes", async () => {
const companyId = await seedCompany();
const agentId = await seedAgent(companyId, "CEO");

const runId = randomUUID();
await seedRun(companyId, agentId, runId);
const todoIssueId = await seedPromotedIssue(companyId, runId, "todo");
const backlogIssueId = await seedPromotedIssue(companyId, runId, "backlog");
await db
.update(heartbeatRuns)
.set({ contextSnapshot: { issueId: todoIssueId } })
.where(eq(heartbeatRuns.id, runId));

await heartbeatService(db).cancelRun(runId);

const [todoAfter] = await db.select().from(issues).where(eq(issues.id, todoIssueId));
const [backlogAfter] = await db.select().from(issues).where(eq(issues.id, backlogIssueId));

// Restored to the exact tier each issue held, not a blanket `todo`, and
// across every sibling the run touched — not only its context issue.
expect(todoAfter?.status).toBe("todo");
expect(todoAfter?.checkoutRestoreStatus).toBeNull();
expect(backlogAfter?.status).toBe("backlog");
expect(backlogAfter?.checkoutRestoreStatus).toBeNull();
});

it("leaves a run-written status alone when the run finalizes", async () => {
const companyId = await seedCompany();
const agentId = await seedAgent(companyId, "CEO");

const runId = randomUUID();
await seedRun(companyId, agentId, runId);
// No marker: an explicit status write clears it, which is how a run that
// genuinely advanced the issue — or deliberately re-asserted
// `in_progress` — is protected from the reset.
const advancedIssueId = await seedPromotedIssue(companyId, runId, null);
await db
.update(heartbeatRuns)
.set({ contextSnapshot: { issueId: advancedIssueId } })
.where(eq(heartbeatRuns.id, runId));

await heartbeatService(db).cancelRun(runId);

const [after] = await db.select().from(issues).where(eq(issues.id, advancedIssueId));
expect(after?.status).toBe("in_progress");
expect(after?.executionRunId).toBeNull();
});

it("does not reset while a live retry run still claims the issue", async () => {
const companyId = await seedCompany();
const agentId = await seedAgent(companyId, "CEO");

const finalizingRunId = randomUUID();
const retryRunId = randomUUID();
await seedRun(companyId, agentId, finalizingRunId);
await seedRun(companyId, agentId, retryRunId, { status: "running" });
const issueId = await seedPromotedIssue(companyId, finalizingRunId, "todo");
// Retry hand-off: `executionRunId` has moved to a still-running retry while
// `checkoutRunId` stays pinned at the finalizing run. The lock clear
// releases only the checkout column, and the live retry must keep the
// issue in `in_progress` so its work is not demoted underneath it.
await db
.update(issues)
.set({ executionRunId: retryRunId })
.where(eq(issues.id, issueId));
await db
.update(heartbeatRuns)
.set({ contextSnapshot: { issueId } })
.where(eq(heartbeatRuns.id, finalizingRunId));

await heartbeatService(db).cancelRun(finalizingRunId);

const [after] = await db.select().from(issues).where(eq(issues.id, issueId));
expect(after?.status).toBe("in_progress");
expect(after?.checkoutRestoreStatus).toBe("todo");
expect(after?.executionRunId).toBe(retryRunId);
});
});
});
210 changes: 210 additions & 0 deletions server/src/__tests__/issues-service.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ import {
parseExecutiveHoldMarkerTimestamp,
} from "../services/issues.ts";
import { issueRecoveryActionService } from "../services/issue-recovery-actions.js";
import { restoreCheckoutPromotedStatus } from "../services/issue-checkout-status.ts";
import {
WORKSPACE_WORKTREE_REQUIRES_PROJECT_CODE,
WORKSPACE_WORKTREE_REQUIRES_PROJECT_MESSAGE,
Expand Down Expand Up @@ -618,6 +619,215 @@ describeEmbeddedPostgres("issueService.list participantAgentId", () => {
};
}

// BLO-20649: `checkout` promotes to `in_progress`; releasing the lock has to
// undo that promotion, or `in_progress` becomes a high-water mark of every
// issue any wake ever touched.
describe("checkout status restore", () => {
async function seedCheckoutFixture(status: "todo" | "backlog" | "blocked") {
const companyId = await seedAssignableAgentCompany();
const agentId = randomUUID();
await db.insert(agents).values(agentRow(companyId, { id: agentId, name: "RestoreCoder" }));
const issue = await svc.create(companyId, {
title: `Restore round trip from ${status}`,
description: null,
status,
priority: "medium",
});
const runId = randomUUID();
await db.insert(heartbeatRuns).values({
id: runId,
companyId,
agentId,
status: "running",
invocationSource: "manual",
});
return { companyId, agentId, issue, runId };
}

async function finishRun(runId: string, status = "succeeded") {
await db.update(heartbeatRuns).set({ status }).where(eq(heartbeatRuns.id, runId));
}

function readIssue(id: string) {
return db
.select({ status: issues.status, restore: issues.checkoutRestoreStatus })
.from(issues)
.where(eq(issues.id, id))
.then((rows) => rows[0]!);
}

it("returns a todo issue to todo when the run releases without advancing it", async () => {
const { agentId, issue, runId } = await seedCheckoutFixture("todo");

await svc.checkout(issue.id, agentId, ["todo"], runId);
expect(await readIssue(issue.id)).toMatchObject({ status: "in_progress", restore: "todo" });

await finishRun(runId);
await svc.clearCheckoutRunIfTerminal(issue.id);

expect(await readIssue(issue.id)).toMatchObject({ status: "todo", restore: null });
});

it("returns a backlog issue to backlog, not todo", async () => {
const { agentId, issue, runId } = await seedCheckoutFixture("backlog");

await svc.checkout(issue.id, agentId, ["backlog"], runId);
expect(await readIssue(issue.id)).toMatchObject({ status: "in_progress", restore: "backlog" });

await finishRun(runId);
await svc.clearCheckoutRunIfTerminal(issue.id);

expect(await readIssue(issue.id)).toMatchObject({ status: "backlog", restore: null });
});

it("keeps a status the run actually wrote", async () => {
const { agentId, issue, runId } = await seedCheckoutFixture("todo");

await svc.checkout(issue.id, agentId, ["todo"], runId);
await svc.update(issue.id, { status: "in_review" });
expect(await readIssue(issue.id)).toMatchObject({ status: "in_review", restore: null });

await finishRun(runId);
await svc.clearCheckoutRunIfTerminal(issue.id);

expect(await readIssue(issue.id)).toMatchObject({ status: "in_review", restore: null });
});

it("keeps an explicit in_progress write instead of resetting it", async () => {
const { agentId, issue, runId } = await seedCheckoutFixture("todo");

await svc.checkout(issue.id, agentId, ["todo"], runId);
// Re-asserting in_progress is a deliberate claim by the run, so it clears
// the marker and must survive the release.
await svc.update(issue.id, { status: "in_progress" });
expect(await readIssue(issue.id)).toMatchObject({ restore: null });

await finishRun(runId);
await svc.clearCheckoutRunIfTerminal(issue.id);

expect(await readIssue(issue.id)).toMatchObject({ status: "in_progress", restore: null });
});

it("does not reset while the checkout run is still live", async () => {
const { agentId, issue, runId } = await seedCheckoutFixture("todo");

await svc.checkout(issue.id, agentId, ["todo"], runId);
// Run is still `running`; both clear paths must decline.
await svc.clearExecutionRunIfTerminal(issue.id);
await svc.clearCheckoutRunIfTerminal(issue.id);

expect(await readIssue(issue.id)).toMatchObject({ status: "in_progress", restore: "todo" });
});

it("does not reset on execution-lock release while a live checkout still holds the row", async () => {
const { companyId, agentId, issue, runId } = await seedCheckoutFixture("todo");
await svc.checkout(issue.id, agentId, ["todo"], runId);

// Execution lock moves to a second, terminal run while the original
// checkout run keeps executing — a retry hand-off, not a release.
const retryRunId = randomUUID();
await db.insert(heartbeatRuns).values({
id: retryRunId,
companyId,
agentId,
status: "failed",
invocationSource: "manual",
});
await db
.update(issues)
.set({ executionRunId: retryRunId })
.where(eq(issues.id, issue.id));

await svc.clearExecutionRunIfTerminal(issue.id);

expect(await readIssue(issue.id)).toMatchObject({ status: "in_progress", restore: "todo" });
});

it("restores a pre-existing strand to todo when it is re-checked-out and released", async () => {
// Rows stranded before this fix carry no marker. Re-checkout adopts them
// with a `todo` marker so the backlog of strands drains instead of
// needing hand-demotion.
const { companyId, agentId, issue, runId } = await seedCheckoutFixture("todo");
await db
.update(issues)
.set({ status: "in_progress", assigneeAgentId: agentId, checkoutRestoreStatus: null })
.where(eq(issues.id, issue.id));

await svc.checkout(issue.id, agentId, ["todo", "in_progress"], runId);
expect(await readIssue(issue.id)).toMatchObject({ status: "in_progress", restore: "todo" });

await finishRun(runId);
await svc.clearCheckoutRunIfTerminal(issue.id);

expect(await readIssue(issue.id)).toMatchObject({ status: "todo", restore: null });
expect(companyId).toBeTruthy();
});

// The fallback checkout paths (stale-execution-lock adoption, and the
// clear-then-retry below it) also promote to `in_progress`. A promotion that
// does not record what it displaced is unrestorable, so these paths used to
// strand a row permanently even with the release side wired up.
for (const startStatus of ["todo", "backlog"] as const) {
it(`records a restore marker when adopting a stale execution lock from ${startStatus}`, async () => {
const { companyId, agentId, issue, runId } = await seedCheckoutFixture(startStatus);

// A previous run holds the execution lock and is already terminal, so
// checkout adopts the row rather than taking the primary path.
const deadRunId = randomUUID();
await db.insert(heartbeatRuns).values({
id: deadRunId,
companyId,
agentId,
status: "failed",
invocationSource: "manual",
});
await db
.update(issues)
.set({ executionRunId: deadRunId, executionLockedAt: new Date() })
.where(eq(issues.id, issue.id));

await svc.checkout(issue.id, agentId, [startStatus], runId);
expect(await readIssue(issue.id)).toMatchObject({
status: "in_progress",
restore: startStatus,
});

await finishRun(runId);
await svc.clearCheckoutRunIfTerminal(issue.id);

expect(await readIssue(issue.id)).toMatchObject({ status: startStatus, restore: null });
});
}

it("does not restore an issue belonging to another company", async () => {
// `restoreCheckoutPromotedStatus` takes issue ids from run context, which
// is not guaranteed to name an issue in the caller's company. The company
// predicate makes a cross-company reset structurally impossible.
const { agentId, issue, runId } = await seedCheckoutFixture("todo");
await svc.checkout(issue.id, agentId, ["todo"], runId);
await finishRun(runId);

const foreignCompanyId = await seedAssignableAgentCompany();
expect(
await restoreCheckoutPromotedStatus(db, {
issueId: issue.id,
companyId: foreignCompanyId,
}),
).toBe(false);
expect(await readIssue(issue.id)).toMatchObject({ status: "in_progress", restore: "todo" });

// Same call, correct company: the row is otherwise fully qualified, so
// this proves the company predicate is what declined above.
expect(
await restoreCheckoutPromotedStatus(db, {
issueId: issue.id,
companyId: issue.companyId,
}),
).toBe(true);
expect(await readIssue(issue.id)).toMatchObject({ status: "todo", restore: null });
});
});

it("rejects direct terminated assignees with structured conflict details", async () => {
const companyId = await seedAssignableAgentCompany();
const terminatedAgentId = randomUUID();
Expand Down
Loading
Loading