diff --git a/apps/dashboard/src/lib/remote/admin-vms.remote.ts b/apps/dashboard/src/lib/remote/admin-vms.remote.ts index 10ebaa0..034b751 100644 --- a/apps/dashboard/src/lib/remote/admin-vms.remote.ts +++ b/apps/dashboard/src/lib/remote/admin-vms.remote.ts @@ -118,7 +118,7 @@ export const listAllAdminVms = query(async (): Promise => { const staleDeleting = rows.filter( (row) => row.active && - row.status === 'deleting' && + (row.status === 'deleting' || row.status === 'error') && !(row.proxmoxId != null ? liveByProxmoxId.get(row.proxmoxId) : null) && !liveById.get(row.id) ); diff --git a/apps/dashboard/src/lib/remote/vms.remote.ts b/apps/dashboard/src/lib/remote/vms.remote.ts index c2ec289..27f8234 100644 --- a/apps/dashboard/src/lib/remote/vms.remote.ts +++ b/apps/dashboard/src/lib/remote/vms.remote.ts @@ -348,7 +348,7 @@ export const getVm = query(getParams, async (params) => { console.warn(`Failed to load live VM state for ${row.id}`, err); if ( row.active && - row.status === 'deleting' && + (row.status === 'deleting' || row.status === 'error') && err instanceof Error && err.message.includes('not found on any Proxmox node') ) { diff --git a/apps/dashboard/src/lib/server/backends/proxmox/client.ts b/apps/dashboard/src/lib/server/backends/proxmox/client.ts index 4feac0a..baf3767 100644 --- a/apps/dashboard/src/lib/server/backends/proxmox/client.ts +++ b/apps/dashboard/src/lib/server/backends/proxmox/client.ts @@ -2,6 +2,7 @@ import ky, { HTTPError, type KyInstance } from 'ky'; import type { Fetcher } from '@cloudflare/workers-types'; import { createVpcFetch, insecureDirectFetch } from '$lib/server/vpc'; import type { + PveTaskLogLine, PveResponse, PveNode, PveQemuVm, @@ -25,6 +26,9 @@ export interface ProxmoxClientConfig { vpc?: Fetcher; } +const isTaskWarning = (exitstatus?: string): exitstatus is string => + !!exitstatus && exitstatus.startsWith('WARNINGS:'); + export class ProxmoxClient { private api: KyInstance; @@ -347,6 +351,16 @@ export class ProxmoxClient { return res.data; } + async deleteStorageVolume(node: string, storage: string, volume: string): Promise { + const res = await this.api + .delete( + `nodes/${encodeURIComponent(node)}/storage/${encodeURIComponent(storage)}/content/${encodeURIComponent(volume)}`, + { timeout: 120_000 } + ) + .json>(); + return res.data; + } + async importStorageContentFromUrl( node: string, storage: string, @@ -394,6 +408,15 @@ export class ProxmoxClient { return res.data; } + async getTaskLog(node: string, upid: string): Promise { + const res = await this.api + .get(`nodes/${encodeURIComponent(node)}/tasks/${encodeURIComponent(upid)}/log`, { + searchParams: { limit: '500' } + }) + .json>(); + return res.data; + } + async waitForTask( node: string, upid: string, @@ -406,6 +429,10 @@ export class ProxmoxClient { while (Date.now() < deadline) { const status = await this.getTaskStatus(node, upid); if (status.status === 'stopped') { + if (isTaskWarning(status.exitstatus)) { + await this.logTaskWarnings(node, upid, status.exitstatus); + return status; + } if (status.exitstatus && status.exitstatus !== 'OK') { throw new Error(`Proxmox task failed: ${status.exitstatus} (UPID: ${upid})`); } @@ -417,6 +444,19 @@ export class ProxmoxClient { throw new Error(`Proxmox task timed out after ${timeout}ms (UPID: ${upid})`); } + private async logTaskWarnings(node: string, upid: string, exitstatus: string): Promise { + try { + const lines = await this.getTaskLog(node, upid); + const warnings = lines.map((l) => l.t).filter((t) => /^WARN/.test(t)); + console.warn(`Proxmox task finished with ${exitstatus} (UPID: ${upid})`, warnings); + } catch (err) { + console.warn( + `Proxmox task finished with ${exitstatus} (UPID: ${upid}); log unavailable`, + err + ); + } + } + // Cluster async getNextVmId(): Promise { diff --git a/apps/dashboard/src/lib/server/backends/proxmox/index.ts b/apps/dashboard/src/lib/server/backends/proxmox/index.ts index 212b75d..1e99a80 100644 --- a/apps/dashboard/src/lib/server/backends/proxmox/index.ts +++ b/apps/dashboard/src/lib/server/backends/proxmox/index.ts @@ -799,10 +799,52 @@ export class ProxmoxBackend implements VmBackend { throw err; } + await this.destroyVmAndWait(node, vmid); + } + + private async destroyVmAndWait(node: string, vmid: number): Promise { + await new Promise((r) => setTimeout(r, 3_000)); const upid = await this.destroyVm(node, vmid); if (!upid) return; - - await this.client.waitForTask(node, upid); + const status = await this.client.waitForTask(node, upid); + const leaked = await this.leakedDisksAfterDestroy(node, upid, status.exitstatus); + if (leaked.length > 0) await this.retryLeakedDiskRemoval(node, vmid, leaked); + } + + private async leakedDisksAfterDestroy( + node: string, + upid: string, + exitstatus?: string + ): Promise { + if (!exitstatus?.startsWith('WARNINGS:')) return []; + const lines = await this.client.getTaskLog(node, upid); + return lines + .map((l) => l.t) + .filter((t) => t.includes('image still has watchers')) + .map((t) => t.match(/rbd rm '([^']+)'/)?.[1]) + .filter((name): name is string => !!name); + } + + private async retryLeakedDiskRemoval(node: string, vmid: number, disks: string[]): Promise { + const storage = config.proxmox.vmDiskStorage; + let remaining = disks; + for (let attempt = 1; remaining.length > 0 && attempt <= 3; attempt++) { + await new Promise((r) => setTimeout(r, 5_000 * attempt)); + const stillLeaked: string[] = []; + for (const disk of remaining) { + try { + const upid = await this.client.deleteStorageVolume(node, storage, `${storage}:${disk}`); + await this.client.waitForTask(node, upid); + } catch (err) { + if (err instanceof HTTPError && err.response.status === 404) continue; + stillLeaked.push(disk); + } + } + remaining = stillLeaked; + } + if (remaining.length > 0) { + console.error(`VM ${vmid} on ${node} destroyed but left undeletable disks`, remaining); + } } private async ensureVmStopped(node: string, vmid: number): Promise { diff --git a/apps/dashboard/src/lib/server/backends/proxmox/types.ts b/apps/dashboard/src/lib/server/backends/proxmox/types.ts index c253809..c4fdcd0 100644 --- a/apps/dashboard/src/lib/server/backends/proxmox/types.ts +++ b/apps/dashboard/src/lib/server/backends/proxmox/types.ts @@ -103,6 +103,11 @@ export interface PveTask { user: string; } +export interface PveTaskLogLine { + n: number; + t: string; +} + export interface PveTaskStatus { status: 'running' | 'stopped'; exitstatus?: string;