fix: retry cleanup of leaked VM disks

apps/dashboard/src/lib/remote/admin-vms.remote.ts

@@ -118,7 +118,7 @@ export const listAllAdminVms = query(async (): Promise<AdminVm[]> => {
 		const staleDeleting = rows.filter(
 			(row) =>
 				row.active &&
-				row.status === 'deleting' &&
+				(row.status === 'deleting' || row.status === 'error') &&
 				!(row.proxmoxId != null ? liveByProxmoxId.get(row.proxmoxId) : null) &&
 				!liveById.get(row.id)
 		);

apps/dashboard/src/lib/remote/vms.remote.ts

@@ -348,7 +348,7 @@ export const getVm = query(getParams, async (params) => {
 		console.warn(`Failed to load live VM state for ${row.id}`, err);
 		if (
 			row.active &&
-			row.status === 'deleting' &&
+			(row.status === 'deleting' || row.status === 'error') &&
 			err instanceof Error &&
 			err.message.includes('not found on any Proxmox node')
 		) {

apps/dashboard/src/lib/server/backends/proxmox/client.ts

@@ -2,6 +2,7 @@ import ky, { HTTPError, type KyInstance } from 'ky';
 import type { Fetcher } from '@cloudflare/workers-types';
 import { createVpcFetch, insecureDirectFetch } from '$lib/server/vpc';
 import type {
+	PveTaskLogLine,
 	PveResponse,
 	PveNode,
 	PveQemuVm,
@@ -25,6 +26,9 @@ export interface ProxmoxClientConfig {
 	vpc?: Fetcher;
 }
 
+const isTaskWarning = (exitstatus?: string): exitstatus is string =>
+	!!exitstatus && exitstatus.startsWith('WARNINGS:');
+
 export class ProxmoxClient {
 	private api: KyInstance;
 
@@ -347,6 +351,16 @@ export class ProxmoxClient {
 		return res.data;
 	}
 
+	async deleteStorageVolume(node: string, storage: string, volume: string): Promise<string> {
+		const res = await this.api
+			.delete(
+				`nodes/${encodeURIComponent(node)}/storage/${encodeURIComponent(storage)}/content/${encodeURIComponent(volume)}`,
+				{ timeout: 120_000 }
+			)
+			.json<PveResponse<string>>();
+		return res.data;
+	}
+
 	async importStorageContentFromUrl(
 		node: string,
 		storage: string,
@@ -394,6 +408,15 @@ export class ProxmoxClient {
 		return res.data;
 	}
 
+	async getTaskLog(node: string, upid: string): Promise<PveTaskLogLine[]> {
+		const res = await this.api
+			.get(`nodes/${encodeURIComponent(node)}/tasks/${encodeURIComponent(upid)}/log`, {
+				searchParams: { limit: '500' }
+			})
+			.json<PveResponse<PveTaskLogLine[]>>();
+		return res.data;
+	}
+
 	async waitForTask(
 		node: string,
 		upid: string,
@@ -406,6 +429,10 @@ export class ProxmoxClient {
 		while (Date.now() < deadline) {
 			const status = await this.getTaskStatus(node, upid);
 			if (status.status === 'stopped') {
+				if (isTaskWarning(status.exitstatus)) {
+					await this.logTaskWarnings(node, upid, status.exitstatus);
+					return status;
+				}
 				if (status.exitstatus && status.exitstatus !== 'OK') {
 					throw new Error(`Proxmox task failed: ${status.exitstatus} (UPID: ${upid})`);
 				}
@@ -417,6 +444,19 @@ export class ProxmoxClient {
 		throw new Error(`Proxmox task timed out after ${timeout}ms (UPID: ${upid})`);
 	}
 
+	private async logTaskWarnings(node: string, upid: string, exitstatus: string): Promise<void> {
+		try {
+			const lines = await this.getTaskLog(node, upid);
+			const warnings = lines.map((l) => l.t).filter((t) => /^WARN/.test(t));
+			console.warn(`Proxmox task finished with ${exitstatus} (UPID: ${upid})`, warnings);
+		} catch (err) {
+			console.warn(
+				`Proxmox task finished with ${exitstatus} (UPID: ${upid}); log unavailable`,
+				err
+			);
+		}
+	}
+
 	// Cluster
 
 	async getNextVmId(): Promise<number> {

apps/dashboard/src/lib/server/backends/proxmox/index.ts

@@ -799,10 +799,52 @@ export class ProxmoxBackend implements VmBackend {
 			throw err;
 		}
 
+		await this.destroyVmAndWait(node, vmid);
+	}
+
+	private async destroyVmAndWait(node: string, vmid: number): Promise<void> {
+		await new Promise((r) => setTimeout(r, 3_000));
 		const upid = await this.destroyVm(node, vmid);
 		if (!upid) return;
-
-		await this.client.waitForTask(node, upid);
+		const status = await this.client.waitForTask(node, upid);
+		const leaked = await this.leakedDisksAfterDestroy(node, upid, status.exitstatus);
+		if (leaked.length > 0) await this.retryLeakedDiskRemoval(node, vmid, leaked);
+	}
+
+	private async leakedDisksAfterDestroy(
+		node: string,
+		upid: string,
+		exitstatus?: string
+	): Promise<string[]> {
+		if (!exitstatus?.startsWith('WARNINGS:')) return [];
+		const lines = await this.client.getTaskLog(node, upid);
+		return lines
+			.map((l) => l.t)
+			.filter((t) => t.includes('image still has watchers'))
+			.map((t) => t.match(/rbd rm '([^']+)'/)?.[1])
+			.filter((name): name is string => !!name);
+	}
+
+	private async retryLeakedDiskRemoval(node: string, vmid: number, disks: string[]): Promise<void> {
+		const storage = config.proxmox.vmDiskStorage;
+		let remaining = disks;
+		for (let attempt = 1; remaining.length > 0 && attempt <= 3; attempt++) {
+			await new Promise((r) => setTimeout(r, 5_000 * attempt));
+			const stillLeaked: string[] = [];
+			for (const disk of remaining) {
+				try {
+					const upid = await this.client.deleteStorageVolume(node, storage, `${storage}:${disk}`);
+					await this.client.waitForTask(node, upid);
+				} catch (err) {
+					if (err instanceof HTTPError && err.response.status === 404) continue;
+					stillLeaked.push(disk);
+				}
+			}
+			remaining = stillLeaked;
+		}
+		if (remaining.length > 0) {
+			console.error(`VM ${vmid} on ${node} destroyed but left undeletable disks`, remaining);
+		}
 	}
 
 	private async ensureVmStopped(node: string, vmid: number): Promise<void> {

apps/dashboard/src/lib/server/backends/proxmox/types.ts

@@ -103,6 +103,11 @@ export interface PveTask {
 	user: string;
 }
 
+export interface PveTaskLogLine {
+	n: number;
+	t: string;
+}
+
 export interface PveTaskStatus {
 	status: 'running' | 'stopped';
 	exitstatus?: string;