feat(supervisor): cancel a resumed run's in-flight checkpoint

Author: nicktrn

apps/supervisor/src/workloadServer/index.ts

@@ -60,6 +60,13 @@ const checkpointDeleteRequests = new Counter({
   registers: [register],
 });
 
+const checkpointCancelRequests = new Counter({
+  name: "checkpoint_cancel_requests_total",
+  help: "Checkpoint cancel requests attempted when a run continues, by outcome",
+  labelNames: ["result"], // "sent" | "no_client" | "not_applicable" | "http_error"
+  registers: [register],
+});
+
 const WorkloadActionParams = z.object({
   runFriendlyId: z.string(),
   snapshotFriendlyId: z.string(),
@@ -283,6 +290,35 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
     checkpointDeleteRequests.inc({ result: "sent" });
   }
 
+  /**
+   * cancelCheckpoints tells the checkpoint service a resumed run's in-flight checkpoint is moot.
+   * Must be called after the reply is sent: it never delays the runner. Without it the checkpoint
+   * is abandoned by a periodic snapshot poll instead, up to that interval later.
+   */
+  private async cancelCheckpoints(runFriendlyId: string): Promise<void> {
+    if (!this.checkpointClient) {
+      checkpointCancelRequests.inc({ result: "no_client" });
+      return;
+    }
+
+    if (this.snapshotService) {
+      checkpointCancelRequests.inc({ result: "not_applicable" });
+      return;
+    }
+
+    const [error, accepted] = await tryCatch(
+      this.checkpointClient.cancelCheckpoints({ runFriendlyId })
+    );
+
+    if (error || !accepted) {
+      checkpointCancelRequests.inc({ result: "http_error" });
+      this.logger.error("Failed to request checkpoint cancel", { runFriendlyId, error });
+      return;
+    }
+
+    checkpointCancelRequests.inc({ result: "sent" });
+  }
+
   /**
    * Sets common route meta on the wide-event state from URL params.
    */
@@ -643,6 +679,8 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
                 }
 
                 reply.json(continuationResult.data as WorkloadContinueRunExecutionResponseBody);
+
+                await this.cancelCheckpoints(params.runFriendlyId);
               }
             ),
         }

packages/core/src/v3/serverOnly/checkpointClient.ts

@@ -157,4 +157,21 @@ export class CheckpointClient {
 
     return true;
   }
+
+  async cancelCheckpoints({ runFriendlyId }: { runFriendlyId: string }): Promise<boolean> {
+    const res = await fetch(
+      new URL(`/api/v1/runs/${runFriendlyId}/checkpoints/cancel`, this.opts.apiUrl),
+      { method: "POST" }
+    );
+
+    if (!res.ok) {
+      this.logger.error("[CheckpointClient] Cancel checkpoints request failed", {
+        runFriendlyId,
+        status: res.status,
+      });
+      return false;
+    }
+
+    return true;
+  }
 }