From 2cdea5d89105f95f0c59152c1a7f26302af7d90c Mon Sep 17 00:00:00 2001 From: Addison LeClair Date: Mon, 21 Sep 2026 04:04:15 +0000 Subject: [PATCH] fix: proxmox shows node online, get data from elsewhere --- apps/dashboard/.env.example | 5 +++ apps/dashboard/src/app.d.ts | 1 + apps/dashboard/src/lib/server/backends/env.ts | 7 ++- .../src/lib/server/backends/index.ts | 11 ++++- .../src/lib/server/backends/proxmox/index.ts | 44 +++++++++++++------ .../src/lib/server/backends/proxmox/types.ts | 1 + 6 files changed, 53 insertions(+), 16 deletions(-) diff --git a/apps/dashboard/.env.example b/apps/dashboard/.env.example index d05fd965..58744ff7 100644 --- a/apps/dashboard/.env.example +++ b/apps/dashboard/.env.example @@ -77,6 +77,11 @@ PROXMOX_VM_FIREWALL_SECURITY_GROUP="stack-dev" # VM CPU model. Unset defaults to x86-64-v4, which needs AVX-512 on the host. # PROXMOX_VM_CPU_TYPE="EPYC-v4" +# Comma-separated Proxmox node names that must never receive new VMs, even if +# the cluster reports them online. Nodes in HA maintenance or fence state are +# skipped automatically. +# PROXMOX_EXCLUDED_NODES="pxmx-04" + # In production, snippet uploads are routed through the SNIPPETS Workers VPC binding. # VPC bindings only exist in the deployed Workers — keep this "false" in local dev to # talk directly to the endpoint below (PROXMOX_SNIPPETS_ENDPOINT_VERIFY_SSL then diff --git a/apps/dashboard/src/app.d.ts b/apps/dashboard/src/app.d.ts index 7740391f..ce4e459c 100644 --- a/apps/dashboard/src/app.d.ts +++ b/apps/dashboard/src/app.d.ts @@ -68,6 +68,7 @@ declare global { PROXMOX_SNIPPETS_ENDPOINT_VERIFY_SSL?: string; PROXMOX_SNIPPETS_STORAGE?: string; PROXMOX_VM_FIREWALL_SECURITY_GROUP?: string; + PROXMOX_EXCLUDED_NODES?: string; PROXMOX_VM_CPU_TYPE?: string; }; } diff --git a/apps/dashboard/src/lib/server/backends/env.ts b/apps/dashboard/src/lib/server/backends/env.ts index 9e301ede..5729d29b 100644 --- a/apps/dashboard/src/lib/server/backends/env.ts +++ b/apps/dashboard/src/lib/server/backends/env.ts @@ -18,6 +18,7 @@ export interface BackendEnv { PROXMOX_SNIPPETS_STORAGE?: string; PROXMOX_VM_FIREWALL_SECURITY_GROUP?: string; PROXMOX_VM_CPU_TYPE?: string; + PROXMOX_EXCLUDED_NODES?: string; } export function getBackendEnv(): BackendEnv { @@ -39,7 +40,8 @@ export function getBackendEnv(): BackendEnv { PROXMOX_SNIPPETS_ENDPOINT_VERIFY_SSL: platformEnv.PROXMOX_SNIPPETS_ENDPOINT_VERIFY_SSL, PROXMOX_SNIPPETS_STORAGE: platformEnv.PROXMOX_SNIPPETS_STORAGE, PROXMOX_VM_FIREWALL_SECURITY_GROUP: platformEnv.PROXMOX_VM_FIREWALL_SECURITY_GROUP, - PROXMOX_VM_CPU_TYPE: platformEnv.PROXMOX_VM_CPU_TYPE + PROXMOX_VM_CPU_TYPE: platformEnv.PROXMOX_VM_CPU_TYPE, + PROXMOX_EXCLUDED_NODES: platformEnv.PROXMOX_EXCLUDED_NODES }; } @@ -59,6 +61,7 @@ export function getBackendEnv(): BackendEnv { PROXMOX_SNIPPETS_ENDPOINT_VERIFY_SSL: privateEnv.PROXMOX_SNIPPETS_ENDPOINT_VERIFY_SSL, PROXMOX_SNIPPETS_STORAGE: privateEnv.PROXMOX_SNIPPETS_STORAGE, PROXMOX_VM_FIREWALL_SECURITY_GROUP: privateEnv.PROXMOX_VM_FIREWALL_SECURITY_GROUP, - PROXMOX_VM_CPU_TYPE: privateEnv.PROXMOX_VM_CPU_TYPE + PROXMOX_VM_CPU_TYPE: privateEnv.PROXMOX_VM_CPU_TYPE, + PROXMOX_EXCLUDED_NODES: privateEnv.PROXMOX_EXCLUDED_NODES }; } diff --git a/apps/dashboard/src/lib/server/backends/index.ts b/apps/dashboard/src/lib/server/backends/index.ts index 3980713c..46d92cb6 100644 --- a/apps/dashboard/src/lib/server/backends/index.ts +++ b/apps/dashboard/src/lib/server/backends/index.ts @@ -22,6 +22,14 @@ export { VmNotFoundError, VmResizeError } from './types'; let cached: { key: string; backend: VmBackend } | null = null; +function parseNodeList(value: string | undefined): string[] { + if (!value) return []; + return value + .split(',') + .map((node) => node.trim()) + .filter(Boolean); +} + function createProxmox(): ProxmoxBackend { const started = performance.now(); const env = getBackendEnv(); @@ -64,7 +72,8 @@ function createProxmox(): ProxmoxBackend { snippetsEndpointVerifySsl: env.PROXMOX_SNIPPETS_ENDPOINT_VERIFY_SSL !== 'false', snippetsStorage: env.PROXMOX_SNIPPETS_STORAGE, firewallSecurityGroup: env.PROXMOX_VM_FIREWALL_SECURITY_GROUP, - vmCpuType: env.PROXMOX_VM_CPU_TYPE + vmCpuType: env.PROXMOX_VM_CPU_TYPE, + excludedNodes: parseNodeList(env.PROXMOX_EXCLUDED_NODES) }); timingLog('backend.proxmox.create.end', { diff --git a/apps/dashboard/src/lib/server/backends/proxmox/index.ts b/apps/dashboard/src/lib/server/backends/proxmox/index.ts index b1cd0104..88e769dd 100644 --- a/apps/dashboard/src/lib/server/backends/proxmox/index.ts +++ b/apps/dashboard/src/lib/server/backends/proxmox/index.ts @@ -97,8 +97,11 @@ type ProxmoxBackendOptions = { snippetsStorage?: string; firewallSecurityGroup?: string; vmCpuType?: string; + excludedNodes?: string[]; }; +const UNSCHEDULABLE_HA_STATES = new Set(['maintenance', 'fence', 'gone']); + type CloudInitVendorConfigParams = { enableSshPasswordAuth?: boolean; }; @@ -513,15 +516,34 @@ export class ProxmoxBackend implements VmBackend { return new Set(resources.flatMap((r) => (r.vmid != null ? [r.vmid] : []))); } - private async firstOnlineNode() { - const nodes = await this.client.listNodes(); - const online = nodes.filter((node) => node.status === 'online'); - if (online.length === 0) throw new Error('No online Proxmox nodes available'); - return online.sort((a, b) => a.node.localeCompare(b.node))[0]; + private async listSchedulableNodes() { + const [nodes, resources] = await Promise.all([ + this.client.listNodes(), + this.getClusterResources('node') + ]); + const haStateByNode = new Map(resources.map((r) => [r.node, r.hastate])); + const excluded = new Set(this.options.excludedNodes ?? []); + const schedulable = nodes.filter((node) => { + if (node.status !== 'online') return false; + if (excluded.has(node.node)) return false; + const haState = haStateByNode.get(node.node); + return !haState || !UNSCHEDULABLE_HA_STATES.has(haState); + }); + if (schedulable.length === 0) { + throw new Error( + 'No schedulable Proxmox nodes available (all offline, excluded, or in maintenance)' + ); + } + return schedulable; + } + + private async firstSchedulableNode() { + const nodes = await this.listSchedulableNodes(); + return nodes.sort((a, b) => a.node.localeCompare(b.node))[0]; } async listImages(): Promise { - const node = await this.firstOnlineNode(); + const node = await this.firstSchedulableNode(); const storages = await this.client.listStorage(node.node); const importStorages = storages.filter((storage) => this.isActiveImportStorage(storage)); @@ -548,7 +570,7 @@ export class ProxmoxBackend implements VmBackend { } async listImageImportTargets(): Promise { - const node = await this.firstOnlineNode(); + const node = await this.firstSchedulableNode(); const storages = await this.client.listStorage(node.node); return storages .filter((storage) => this.isActiveImportStorage(storage)) @@ -672,12 +694,8 @@ export class ProxmoxBackend implements VmBackend { async createVm(params: VmCreateParams): Promise { clearProxmoxReadCaches(); const vmid = params.proxmoxId; - const nodes = await this.client.listNodes(); - - // Pick the online node with the most free memory - const online = nodes.filter((n) => n.status === 'online'); - if (!online.length) throw new Error('No online Proxmox nodes available'); - const node = online.sort((a, b) => b.maxmem - b.mem - (a.maxmem - a.mem))[0]; + const nodes = await this.listSchedulableNodes(); + const node = nodes.sort((a, b) => b.maxmem - b.mem - (a.maxmem - a.mem))[0]; const sshKeysEncoded = params.sshKeys ? encodeURIComponent(params.sshKeys.join('\n')) diff --git a/apps/dashboard/src/lib/server/backends/proxmox/types.ts b/apps/dashboard/src/lib/server/backends/proxmox/types.ts index 237e7d68..1d75bb9a 100644 --- a/apps/dashboard/src/lib/server/backends/proxmox/types.ts +++ b/apps/dashboard/src/lib/server/backends/proxmox/types.ts @@ -152,6 +152,7 @@ export interface PveClusterResource { vmid?: number; name?: string; status?: string; + hastate?: string; tags?: string; maxcpu?: number; maxmem?: number;