fix: proxmox shows node online, get data from elsewhere

apps/dashboard/.env.example

@@ -77,6 +77,11 @@ PROXMOX_VM_FIREWALL_SECURITY_GROUP="stack-dev"
 # VM CPU model. Unset defaults to x86-64-v4, which needs AVX-512 on the host.
 # PROXMOX_VM_CPU_TYPE="EPYC-v4"
 
+# Comma-separated Proxmox node names that must never receive new VMs, even if
+# the cluster reports them online. Nodes in HA maintenance or fence state are
+# skipped automatically.
+# PROXMOX_EXCLUDED_NODES="pxmx-04"
+
 # In production, snippet uploads are routed through the SNIPPETS Workers VPC binding.
 # VPC bindings only exist in the deployed Workers — keep this "false" in local dev to
 # talk directly to the endpoint below (PROXMOX_SNIPPETS_ENDPOINT_VERIFY_SSL then

apps/dashboard/src/app.d.ts

@@ -68,6 +68,7 @@ declare global {
 				PROXMOX_SNIPPETS_ENDPOINT_VERIFY_SSL?: string;
 				PROXMOX_SNIPPETS_STORAGE?: string;
 				PROXMOX_VM_FIREWALL_SECURITY_GROUP?: string;
+				PROXMOX_EXCLUDED_NODES?: string;
 				PROXMOX_VM_CPU_TYPE?: string;
 			};
 		}

apps/dashboard/src/lib/server/backends/env.ts

@@ -18,6 +18,7 @@ export interface BackendEnv {
 	PROXMOX_SNIPPETS_STORAGE?: string;
 	PROXMOX_VM_FIREWALL_SECURITY_GROUP?: string;
 	PROXMOX_VM_CPU_TYPE?: string;
+	PROXMOX_EXCLUDED_NODES?: string;
 }
 
 export function getBackendEnv(): BackendEnv {
@@ -39,7 +40,8 @@ export function getBackendEnv(): BackendEnv {
 			PROXMOX_SNIPPETS_ENDPOINT_VERIFY_SSL: platformEnv.PROXMOX_SNIPPETS_ENDPOINT_VERIFY_SSL,
 			PROXMOX_SNIPPETS_STORAGE: platformEnv.PROXMOX_SNIPPETS_STORAGE,
 			PROXMOX_VM_FIREWALL_SECURITY_GROUP: platformEnv.PROXMOX_VM_FIREWALL_SECURITY_GROUP,
-			PROXMOX_VM_CPU_TYPE: platformEnv.PROXMOX_VM_CPU_TYPE
+			PROXMOX_VM_CPU_TYPE: platformEnv.PROXMOX_VM_CPU_TYPE,
+			PROXMOX_EXCLUDED_NODES: platformEnv.PROXMOX_EXCLUDED_NODES
 		};
 	}
 
@@ -59,6 +61,7 @@ export function getBackendEnv(): BackendEnv {
 		PROXMOX_SNIPPETS_ENDPOINT_VERIFY_SSL: privateEnv.PROXMOX_SNIPPETS_ENDPOINT_VERIFY_SSL,
 		PROXMOX_SNIPPETS_STORAGE: privateEnv.PROXMOX_SNIPPETS_STORAGE,
 		PROXMOX_VM_FIREWALL_SECURITY_GROUP: privateEnv.PROXMOX_VM_FIREWALL_SECURITY_GROUP,
-		PROXMOX_VM_CPU_TYPE: privateEnv.PROXMOX_VM_CPU_TYPE
+		PROXMOX_VM_CPU_TYPE: privateEnv.PROXMOX_VM_CPU_TYPE,
+		PROXMOX_EXCLUDED_NODES: privateEnv.PROXMOX_EXCLUDED_NODES
 	};
 }

apps/dashboard/src/lib/server/backends/index.ts

@@ -22,6 +22,14 @@ export { VmNotFoundError, VmResizeError } from './types';
 
 let cached: { key: string; backend: VmBackend } | null = null;
 
+function parseNodeList(value: string | undefined): string[] {
+	if (!value) return [];
+	return value
+		.split(',')
+		.map((node) => node.trim())
+		.filter(Boolean);
+}
+
 function createProxmox(): ProxmoxBackend {
 	const started = performance.now();
 	const env = getBackendEnv();
@@ -64,7 +72,8 @@ function createProxmox(): ProxmoxBackend {
 		snippetsEndpointVerifySsl: env.PROXMOX_SNIPPETS_ENDPOINT_VERIFY_SSL !== 'false',
 		snippetsStorage: env.PROXMOX_SNIPPETS_STORAGE,
 		firewallSecurityGroup: env.PROXMOX_VM_FIREWALL_SECURITY_GROUP,
-		vmCpuType: env.PROXMOX_VM_CPU_TYPE
+		vmCpuType: env.PROXMOX_VM_CPU_TYPE,
+		excludedNodes: parseNodeList(env.PROXMOX_EXCLUDED_NODES)
 	});
 
 	timingLog('backend.proxmox.create.end', {

apps/dashboard/src/lib/server/backends/proxmox/index.ts

@@ -97,8 +97,11 @@ type ProxmoxBackendOptions = {
 	snippetsStorage?: string;
 	firewallSecurityGroup?: string;
 	vmCpuType?: string;
+	excludedNodes?: string[];
 };
 
+const UNSCHEDULABLE_HA_STATES = new Set(['maintenance', 'fence', 'gone']);
+
 type CloudInitVendorConfigParams = {
 	enableSshPasswordAuth?: boolean;
 };
@@ -513,15 +516,34 @@ export class ProxmoxBackend implements VmBackend {
 		return new Set(resources.flatMap((r) => (r.vmid != null ? [r.vmid] : [])));
 	}
 
-	private async firstOnlineNode() {
-		const nodes = await this.client.listNodes();
-		const online = nodes.filter((node) => node.status === 'online');
-		if (online.length === 0) throw new Error('No online Proxmox nodes available');
-		return online.sort((a, b) => a.node.localeCompare(b.node))[0];
+	private async listSchedulableNodes() {
+		const [nodes, resources] = await Promise.all([
+			this.client.listNodes(),
+			this.getClusterResources('node')
+		]);
+		const haStateByNode = new Map(resources.map((r) => [r.node, r.hastate]));
+		const excluded = new Set(this.options.excludedNodes ?? []);
+		const schedulable = nodes.filter((node) => {
+			if (node.status !== 'online') return false;
+			if (excluded.has(node.node)) return false;
+			const haState = haStateByNode.get(node.node);
+			return !haState || !UNSCHEDULABLE_HA_STATES.has(haState);
+		});
+		if (schedulable.length === 0) {
+			throw new Error(
+				'No schedulable Proxmox nodes available (all offline, excluded, or in maintenance)'
+			);
+		}
+		return schedulable;
+	}
+
+	private async firstSchedulableNode() {
+		const nodes = await this.listSchedulableNodes();
+		return nodes.sort((a, b) => a.node.localeCompare(b.node))[0];
 	}
 
 	async listImages(): Promise<BackendImage[]> {
-		const node = await this.firstOnlineNode();
+		const node = await this.firstSchedulableNode();
 		const storages = await this.client.listStorage(node.node);
 		const importStorages = storages.filter((storage) => this.isActiveImportStorage(storage));
 
@@ -548,7 +570,7 @@ export class ProxmoxBackend implements VmBackend {
 	}
 
 	async listImageImportTargets(): Promise<BackendImageImportTarget[]> {
-		const node = await this.firstOnlineNode();
+		const node = await this.firstSchedulableNode();
 		const storages = await this.client.listStorage(node.node);
 		return storages
 			.filter((storage) => this.isActiveImportStorage(storage))
@@ -672,12 +694,8 @@ export class ProxmoxBackend implements VmBackend {
 	async createVm(params: VmCreateParams): Promise<VmCreateResult> {
 		clearProxmoxReadCaches();
 		const vmid = params.proxmoxId;
-		const nodes = await this.client.listNodes();
-
-		// Pick the online node with the most free memory
-		const online = nodes.filter((n) => n.status === 'online');
-		if (!online.length) throw new Error('No online Proxmox nodes available');
-		const node = online.sort((a, b) => b.maxmem - b.mem - (a.maxmem - a.mem))[0];
+		const nodes = await this.listSchedulableNodes();
+		const node = nodes.sort((a, b) => b.maxmem - b.mem - (a.maxmem - a.mem))[0];
 
 		const sshKeysEncoded = params.sshKeys
 			? encodeURIComponent(params.sshKeys.join('\n'))

apps/dashboard/src/lib/server/backends/proxmox/types.ts

@@ -152,6 +152,7 @@ export interface PveClusterResource {
 	vmid?: number;
 	name?: string;
 	status?: string;
+	hastate?: string;
 	tags?: string;
 	maxcpu?: number;
 	maxmem?: number;