feat(supervisor): cancel a resumed run's in-flight checkpoint (#4502)

A run controller must call the continue route to resume, so the
supervisor already knows synchronously that any checkpoint still running
for that run is pointless. It only acted on that for the compute path.

The continue route now cancels it for the Kubernetes path too, matching
what completion already does since #4493. Called after the reply so the
runner is never delayed, and skipped when there is no checkpoint client
or when the compute path owns the run. The request is bounded by a 5s
timeout so a hung call cannot leave the handler pending.

`checkpoint_cancel_requests_total{result}` records the outcome, using
the same label names as the delete path where they overlap: `sent`,
`no_client`, `not_applicable`, `http_error`.

No changeset: `CheckpointClient` is a server-only internal API, same as
#4493.

refs TRI-12915

Author: nicktrn

apps/supervisor/src/workloadServer/index.ts

@@ -60,6 +60,13 @@ const checkpointDeleteRequests = new Counter({
   registers: [register],
 });
 
+const checkpointCancelRequests = new Counter({
+  name: "checkpoint_cancel_requests_total",
+  help: "Checkpoint cancel requests attempted when a run continues, by outcome",
+  labelNames: ["result"],
+  registers: [register],
+});
+
 const WorkloadActionParams = z.object({
   runFriendlyId: z.string(),
   snapshotFriendlyId: z.string(),
@@ -283,6 +290,30 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
     checkpointDeleteRequests.inc({ result: "sent" });
   }
 
+  private async cancelCheckpointsAfterReply(runFriendlyId: string): Promise<void> {
+    if (!this.checkpointClient) {
+      checkpointCancelRequests.inc({ result: "no_client" });
+      return;
+    }
+
+    if (this.snapshotService) {
+      checkpointCancelRequests.inc({ result: "not_applicable" });
+      return;
+    }
+
+    const [error, accepted] = await tryCatch(
+      this.checkpointClient.cancelCheckpoints({ runFriendlyId })
+    );
+
+    if (error || !accepted) {
+      checkpointCancelRequests.inc({ result: "http_error" });
+      this.logger.error("Failed to request checkpoint cancel", { runFriendlyId, error });
+      return;
+    }
+
+    checkpointCancelRequests.inc({ result: "sent" });
+  }
+
   /**
    * Sets common route meta on the wide-event state from URL params.
    */
@@ -643,6 +674,8 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
                 }
 
                 reply.json(continuationResult.data as WorkloadContinueRunExecutionResponseBody);
+
+                await this.cancelCheckpointsAfterReply(params.runFriendlyId);
               }
             ),
         }

packages/core/src/v3/serverOnly/checkpointClient.ts

@@ -13,6 +13,8 @@ export type CheckpointClientOptions = {
   orchestrator: CheckpointType;
 };
 
+const CANCEL_TIMEOUT_MS = 5_000;
+
 export class CheckpointClient {
   private readonly logger = new SimpleStructuredLogger("checkpoint-client");
 
@@ -157,4 +159,21 @@ export class CheckpointClient {
 
     return true;
   }
+
+  async cancelCheckpoints({ runFriendlyId }: { runFriendlyId: string }): Promise<boolean> {
+    const res = await fetch(
+      new URL(`/api/v1/runs/${runFriendlyId}/checkpoints/cancel`, this.opts.apiUrl),
+      { method: "POST", signal: AbortSignal.timeout(CANCEL_TIMEOUT_MS) }
+    );
+
+    if (!res.ok) {
+      this.logger.error("[CheckpointClient] Cancel checkpoints request failed", {
+        runFriendlyId,
+        status: res.status,
+      });
+      return false;
+    }
+
+    return true;
+  }
 }