From 75127885c067452465f25fab593949b60d3bd74a Mon Sep 17 00:00:00 2001
From: Eric Wang <37554696+ericwang401@users.noreply.github.com>
Date: Tue, 29 Sep 2026 04:20:39 +0000
Subject: [PATCH 1/2] refactor(jobs): key overlap locks by ID alone

WithoutOverlapping already scopes its lock by the job's class (laravel-queue-overlap:<class>:<key>), and no job shares a key with another, so prefixes like "server.send-power-command#" repeated what the class name already says. Key each lock by the server, backup or ISO ID alone.
---
 app/Jobs/Concerns/LocksPerSubject.php         | 31 +++++++++++++++++++
 app/Jobs/Node/MonitorIsoDownloadJob.php       |  2 +-
 app/Jobs/Server/BuildServerJob.php            |  4 +--
 app/Jobs/Server/DeleteServerJob.php           |  4 +--
 app/Jobs/Server/MonitorBackupJob.php          |  2 +-
 .../Server/MonitorBackupRestorationJob.php    |  2 +-
 app/Jobs/Server/PurgeBackupsJob.php           |  4 +--
 app/Jobs/Server/SendPowerCommandJob.php       |  4 +--
 app/Jobs/Server/SyncBuildJob.php              |  4 +--
 app/Jobs/Server/SyncNetworkSettings.php       |  4 +--
 app/Jobs/Server/UpdatePasswordJob.php         |  4 +--
 11 files changed, 41 insertions(+), 24 deletions(-)
 create mode 100644 app/Jobs/Concerns/LocksPerSubject.php

diff --git a/app/Jobs/Concerns/LocksPerSubject.php b/app/Jobs/Concerns/LocksPerSubject.php
new file mode 100644
index 00000000000..31d4a1030fd
--- /dev/null
+++ b/app/Jobs/Concerns/LocksPerSubject.php
@@ -0,0 +1,31 @@
+<?php
+
+namespace Convoy\Jobs\Concerns;
+
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+
+/**
+ * Keeps two jobs of the same class from running on the same server (or backup,
+ * or ISO) at once, with a lock that can't outlive a killed job.
+ *
+ * WithoutOverlapping's lock never expires by default, and it's only released
+ * when the job finishes. A job the worker kills for running past its timeout
+ * never gets there, so the lock stayed forever and every later job of that
+ * class for that subject failed straight away -- a slow Proxmox call during a
+ * force-stop could break reinstalling and deleting that server until the lock
+ * was cleared from Redis by hand.
+ *
+ * The lock now expires shortly after the job's own timeout, and a job that finds
+ * it held waits a few seconds and tries again instead of giving up at once.
+ * Laravel already scopes the key by the job's class, so the subject's ID is key
+ * enough.
+ */
+trait LocksPerSubject
+{
+    protected function lockPerSubject(int|string $subjectId): WithoutOverlapping
+    {
+        return (new WithoutOverlapping($subjectId))
+            ->expireAfter($this->timeout + 30)
+            ->releaseAfter(5);
+    }
+}
diff --git a/app/Jobs/Node/MonitorIsoDownloadJob.php b/app/Jobs/Node/MonitorIsoDownloadJob.php
index e648dc1cbd9..cbfb463e9aa 100644
--- a/app/Jobs/Node/MonitorIsoDownloadJob.php
+++ b/app/Jobs/Node/MonitorIsoDownloadJob.php
@@ -27,7 +27,7 @@ public function __construct(protected int $isoId, protected string $upid)
 
     public function middleware()
     {
-        return [new WithoutOverlapping("node:iso.download#{$this->isoId}")];
+        return [new WithoutOverlapping($this->isoId)];
     }
 
     public function handle(IsoMonitorService $service): void
diff --git a/app/Jobs/Server/BuildServerJob.php b/app/Jobs/Server/BuildServerJob.php
index 20abf957c83..ddba76c914e 100644
--- a/app/Jobs/Server/BuildServerJob.php
+++ b/app/Jobs/Server/BuildServerJob.php
@@ -28,9 +28,7 @@ public function __construct(protected int $serverId, protected int $templateId)
 
     public function middleware(): array
     {
-        return [new SkipIfBatchCancelled(), new WithoutOverlapping(
-            "server.build#{$this->serverId}",
-        )];
+        return [new SkipIfBatchCancelled(), new WithoutOverlapping($this->serverId)];
     }
 
     public function handle(ServerBuildService $service): void
diff --git a/app/Jobs/Server/DeleteServerJob.php b/app/Jobs/Server/DeleteServerJob.php
index 1918889742e..545a66fbb70 100644
--- a/app/Jobs/Server/DeleteServerJob.php
+++ b/app/Jobs/Server/DeleteServerJob.php
@@ -28,9 +28,7 @@ public function __construct(protected int $serverId)
 
     public function middleware(): array
     {
-        return [new SkipIfBatchCancelled(), new WithoutOverlapping(
-            "server.delete#{$this->serverId}",
-        )];
+        return [new SkipIfBatchCancelled(), new WithoutOverlapping($this->serverId)];
     }
 
     public function handle(ServerBuildService $service): void
diff --git a/app/Jobs/Server/MonitorBackupJob.php b/app/Jobs/Server/MonitorBackupJob.php
index e68d56ca281..22dfb447214 100644
--- a/app/Jobs/Server/MonitorBackupJob.php
+++ b/app/Jobs/Server/MonitorBackupJob.php
@@ -27,7 +27,7 @@ public function __construct(protected int $backupId, protected string $upid)
 
     public function middleware(): array
     {
-        return [new WithoutOverlapping("server:backup.create#{$this->backupId}")];
+        return [new WithoutOverlapping($this->backupId)];
     }
 
     public function handle(BackupMonitorService $service): void
diff --git a/app/Jobs/Server/MonitorBackupRestorationJob.php b/app/Jobs/Server/MonitorBackupRestorationJob.php
index 189a9789bcf..c8a75b65d08 100644
--- a/app/Jobs/Server/MonitorBackupRestorationJob.php
+++ b/app/Jobs/Server/MonitorBackupRestorationJob.php
@@ -27,7 +27,7 @@ public function __construct(protected int $serverId, protected string $upid)
 
     public function middleware(): array
     {
-        return [new WithoutOverlapping("server:backup.restore#{$this->serverId}")];
+        return [new WithoutOverlapping($this->serverId)];
     }
 
     public function handle(BackupMonitorService $service): void
diff --git a/app/Jobs/Server/PurgeBackupsJob.php b/app/Jobs/Server/PurgeBackupsJob.php
index 6d9a6bcbbf2..3b487f36bd6 100644
--- a/app/Jobs/Server/PurgeBackupsJob.php
+++ b/app/Jobs/Server/PurgeBackupsJob.php
@@ -28,9 +28,7 @@ public function __construct(protected int $serverId)
 
     public function middleware(): array
     {
-        return [new SkipIfBatchCancelled(), new WithoutOverlapping(
-            "server:backups.purge#{$this->serverId}",
-        )];
+        return [new SkipIfBatchCancelled(), new WithoutOverlapping($this->serverId)];
     }
 
     public function handle(PurgeBackupsService $service): void
diff --git a/app/Jobs/Server/SendPowerCommandJob.php b/app/Jobs/Server/SendPowerCommandJob.php
index d63b5c78d7b..b5b0f6796fe 100644
--- a/app/Jobs/Server/SendPowerCommandJob.php
+++ b/app/Jobs/Server/SendPowerCommandJob.php
@@ -28,9 +28,7 @@ public function __construct(protected int $serverId, protected PowerAction $powe
 
     public function middleware(): array
     {
-        return [new SkipIfBatchCancelled(), new WithoutOverlapping(
-            "server.send-power-command#{$this->serverId}",
-        )];
+        return [new SkipIfBatchCancelled(), new WithoutOverlapping($this->serverId)];
     }
 
     public function handle(ProxmoxPowerRepository $repository): void
diff --git a/app/Jobs/Server/SyncBuildJob.php b/app/Jobs/Server/SyncBuildJob.php
index a9f47bbc0e6..0103b2d75f7 100644
--- a/app/Jobs/Server/SyncBuildJob.php
+++ b/app/Jobs/Server/SyncBuildJob.php
@@ -28,9 +28,7 @@ public function __construct(protected int $serverId)
 
     public function middleware(): array
     {
-        return [new SkipIfBatchCancelled(), new WithoutOverlapping(
-            "server.sync#{$this->serverId}",
-        )];
+        return [new SkipIfBatchCancelled(), new WithoutOverlapping($this->serverId)];
     }
 
     public function handle(SyncBuildService $service): void
diff --git a/app/Jobs/Server/SyncNetworkSettings.php b/app/Jobs/Server/SyncNetworkSettings.php
index ec30b39f00d..d971993d381 100644
--- a/app/Jobs/Server/SyncNetworkSettings.php
+++ b/app/Jobs/Server/SyncNetworkSettings.php
@@ -28,9 +28,7 @@ public function __construct(protected int $serverId)
 
     public function middleware(): array
     {
-        return [new SkipIfBatchCancelled(), new WithoutOverlapping(
-            "server.sync-network-settings#$this->serverId",
-        )];
+        return [new SkipIfBatchCancelled(), new WithoutOverlapping($this->serverId)];
     }
 
     public function handle(NetworkService $service): void
diff --git a/app/Jobs/Server/UpdatePasswordJob.php b/app/Jobs/Server/UpdatePasswordJob.php
index 996580f2435..119345f34cc 100644
--- a/app/Jobs/Server/UpdatePasswordJob.php
+++ b/app/Jobs/Server/UpdatePasswordJob.php
@@ -32,9 +32,7 @@ public function __construct(protected int $serverId, protected string $password)
 
     public function middleware(): array
     {
-        return [new SkipIfBatchCancelled(), new WithoutOverlapping(
-            "server.update-password#{$this->serverId}",
-        )];
+        return [new SkipIfBatchCancelled(), new WithoutOverlapping($this->serverId)];
     }
 
     public function handle(ServerAuthService $service): void

From a84ada40bb32d7fe3eccc2c06cb3e6d14623d722 Mon Sep 17 00:00:00 2001
From: Eric Wang <37554696+ericwang401@users.noreply.github.com>
Date: Tue, 29 Sep 2026 04:21:03 +0000
Subject: [PATCH 2/2] fix(jobs): expire overlap locks, and back off between
 retries

Locks: WithoutOverlapping's lock never expires by default and is only
released when the job finishes, so a job the worker killed for running
past its timeout left it held forever, and every later job of that class
for that server failed at once. SendPowerCommandJob (15s) and
UpdatePasswordJob (10s) could hit that with a single slow Proxmox call,
which is itself allowed 15s. A new LocksPerSubject trait gives each lock
an expiry of the job's timeout + 30s and a 5s release when held. Jobs
without a timeout of their own get one, since the expiry is based on it:
300s for SyncBuildJob and PurgeBackupsJob, 60s for the rest.

Retries: jobs whose Proxmox call throws were retried immediately until
their tries or retryUntil ran out -- 7,388 attempts in two seconds for a
MonitorStateJob on a missing VM. The polling jobs now back off 3s, 10s,
then 30s; the three-try jobs 5s then 15s; the node sync jobs 10s then 30s.
WaitUntilVmIsDeletedJob is left to the delete/disconnect branch, which
already changes it.

Tested: a unit test pins every lock's key, expiry and release; a power
cycle (start, wait running, kill, wait stopped) through Horizon on a real
Proxmox node, which left no locks behind.
---
 CHANGELOG.md                                  | 14 ++++++++
 app/Jobs/Node/MonitorIsoDownloadJob.php       | 17 +++++++--
 app/Jobs/Node/PruneUsersJob.php               |  8 +++++
 app/Jobs/Node/SyncServerRateLimitsJob.php     |  8 +++++
 app/Jobs/Node/SyncServerUsagesJob.php         |  8 +++++
 app/Jobs/Server/BuildServerJob.php            | 16 ++++++---
 app/Jobs/Server/DeleteServerJob.php           | 16 ++++++---
 app/Jobs/Server/MonitorBackupJob.php          | 17 +++++++--
 .../Server/MonitorBackupRestorationJob.php    | 17 +++++++--
 app/Jobs/Server/MonitorStateJob.php           | 11 ++++++
 app/Jobs/Server/PurgeBackupsJob.php           | 14 ++++++--
 app/Jobs/Server/SendPowerCommandJob.php       | 16 ++++++---
 app/Jobs/Server/SyncBuildJob.php              | 17 +++++++--
 app/Jobs/Server/SyncNetworkSettings.php       | 17 +++++++--
 app/Jobs/Server/UpdatePasswordJob.php         |  8 ++---
 app/Jobs/Server/WaitUntilVmIsCreatedJob.php   | 11 ++++++
 tests/Unit/Jobs/LocksPerSubjectTest.php       | 36 +++++++++++++++++++
 17 files changed, 217 insertions(+), 34 deletions(-)
 create mode 100644 tests/Unit/Jobs/LocksPerSubjectTest.php

diff --git a/CHANGELOG.md b/CHANGELOG.md
index deb96919c74..277d21f1962 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,20 @@ This file is a running track of new features and fixes to each version of the pa
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project
 follows [Semantic Versioning](https://semver.org) guidelines.
 
+## Unreleased
+
+### Fixed
+
+- Fixed a server's jobs getting permanently stuck after one slow request to Proxmox. The locks that keep two jobs of the
+  same kind from running on a server at once never expired, so a job the queue killed for running past its time limit
+  left its lock behind for good: every later job of that kind for the server -- including the force-stop that reinstall
+  and delete start with -- failed instantly until the lock was cleared from Redis by hand. Locks now expire shortly
+  after their job's time limit, a job that finds one held waits and retries, and the power and password jobs' time
+  limits no longer fall below a single Proxmox request.
+- Fixed failing jobs retrying with no pause. A job whose request to Proxmox failed was retried at once, again and
+  again, until it ran out of time -- thousands of requests a minute against a node that was already struggling. Jobs
+  now wait a few seconds, then longer, between retries.
+
 ## v4.7.0-rc.2
 
 ### Fixed
diff --git a/app/Jobs/Node/MonitorIsoDownloadJob.php b/app/Jobs/Node/MonitorIsoDownloadJob.php
index cbfb463e9aa..ebf21cdd5ad 100644
--- a/app/Jobs/Node/MonitorIsoDownloadJob.php
+++ b/app/Jobs/Node/MonitorIsoDownloadJob.php
@@ -8,13 +8,15 @@
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
-use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Convoy\Jobs\Concerns\LocksPerSubject;
 use Illuminate\Queue\SerializesModels;
 use Illuminate\Support\Carbon;
 
 class MonitorIsoDownloadJob implements ShouldQueue
 {
-    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels, LocksPerSubject;
+
+    public int $timeout = 60;
 
     public function retryUntil(): Carbon
     {
@@ -25,9 +27,18 @@ public function __construct(protected int $isoId, protected string $upid)
     {
     }
 
+    /**
+     * Wait before retrying after an error instead of retrying in a tight loop.
+     * The last delay repeats until the job gives up.
+     */
+    public function backoff(): array
+    {
+        return [3, 10, 30];
+    }
+
     public function middleware()
     {
-        return [new WithoutOverlapping($this->isoId)];
+        return [$this->lockPerSubject($this->isoId)];
     }
 
     public function handle(IsoMonitorService $service): void
diff --git a/app/Jobs/Node/PruneUsersJob.php b/app/Jobs/Node/PruneUsersJob.php
index 0c1e7183974..f93588368f1 100644
--- a/app/Jobs/Node/PruneUsersJob.php
+++ b/app/Jobs/Node/PruneUsersJob.php
@@ -22,6 +22,14 @@ public function __construct(protected int $nodeId)
     {
     }
 
+    /**
+     * Wait before retrying a failed sync instead of retrying at once.
+     */
+    public function backoff(): array
+    {
+        return [10, 30];
+    }
+
     public function handle(UserPruneService $service): void
     {
         $node = Node::findOrFail($this->nodeId);
diff --git a/app/Jobs/Node/SyncServerRateLimitsJob.php b/app/Jobs/Node/SyncServerRateLimitsJob.php
index 82829755105..7c4ea63596a 100644
--- a/app/Jobs/Node/SyncServerRateLimitsJob.php
+++ b/app/Jobs/Node/SyncServerRateLimitsJob.php
@@ -20,6 +20,14 @@ public function __construct(protected int $nodeId)
     {
     }
 
+    /**
+     * Wait before retrying a failed sync instead of retrying at once.
+     */
+    public function backoff(): array
+    {
+        return [10, 30];
+    }
+
     public function handle(ServerRateLimitsSyncService $service): void
     {
         $node = Node::findOrFail($this->nodeId);
diff --git a/app/Jobs/Node/SyncServerUsagesJob.php b/app/Jobs/Node/SyncServerUsagesJob.php
index 710ddaa5205..305b143bf05 100644
--- a/app/Jobs/Node/SyncServerUsagesJob.php
+++ b/app/Jobs/Node/SyncServerUsagesJob.php
@@ -20,6 +20,14 @@ public function __construct(protected int $nodeId)
     {
     }
 
+    /**
+     * Wait before retrying a failed sync instead of retrying at once.
+     */
+    public function backoff(): array
+    {
+        return [10, 30];
+    }
+
     public function handle(ServerUsagesSyncService $service): void
     {
         $node = Node::findOrFail($this->nodeId);
diff --git a/app/Jobs/Server/BuildServerJob.php b/app/Jobs/Server/BuildServerJob.php
index ddba76c914e..fff07fdb44e 100644
--- a/app/Jobs/Server/BuildServerJob.php
+++ b/app/Jobs/Server/BuildServerJob.php
@@ -11,24 +11,32 @@
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\Middleware\SkipIfBatchCancelled;
-use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Convoy\Jobs\Concerns\LocksPerSubject;
 use Illuminate\Queue\SerializesModels;
 
 class BuildServerJob implements ShouldQueue
 {
-    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels, LocksPerSubject;
 
     public int $tries = 3;
 
-    public int $timeout = 20;
+    public int $timeout = 60;
 
     public function __construct(protected int $serverId, protected int $templateId)
     {
     }
 
+    /**
+     * Wait before retrying a failed Proxmox call instead of retrying at once.
+     */
+    public function backoff(): array
+    {
+        return [5, 15];
+    }
+
     public function middleware(): array
     {
-        return [new SkipIfBatchCancelled(), new WithoutOverlapping($this->serverId)];
+        return [new SkipIfBatchCancelled(), $this->lockPerSubject($this->serverId)];
     }
 
     public function handle(ServerBuildService $service): void
diff --git a/app/Jobs/Server/DeleteServerJob.php b/app/Jobs/Server/DeleteServerJob.php
index 545a66fbb70..6fb1bd60aa0 100644
--- a/app/Jobs/Server/DeleteServerJob.php
+++ b/app/Jobs/Server/DeleteServerJob.php
@@ -10,25 +10,33 @@
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\Middleware\SkipIfBatchCancelled;
-use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Convoy\Jobs\Concerns\LocksPerSubject;
 use Illuminate\Queue\SerializesModels;
 
 class DeleteServerJob implements ShouldQueue
 {
-    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels, Batchable;
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels, Batchable, LocksPerSubject;
 
     public int $tries = 3;
 
-    public int $timeout = 20;
+    public int $timeout = 60;
 
     public function __construct(protected int $serverId)
     {
         //
     }
 
+    /**
+     * Wait before retrying a failed Proxmox call instead of retrying at once.
+     */
+    public function backoff(): array
+    {
+        return [5, 15];
+    }
+
     public function middleware(): array
     {
-        return [new SkipIfBatchCancelled(), new WithoutOverlapping($this->serverId)];
+        return [new SkipIfBatchCancelled(), $this->lockPerSubject($this->serverId)];
     }
 
     public function handle(ServerBuildService $service): void
diff --git a/app/Jobs/Server/MonitorBackupJob.php b/app/Jobs/Server/MonitorBackupJob.php
index 22dfb447214..2d5c07d6af3 100644
--- a/app/Jobs/Server/MonitorBackupJob.php
+++ b/app/Jobs/Server/MonitorBackupJob.php
@@ -8,13 +8,15 @@
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
-use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Convoy\Jobs\Concerns\LocksPerSubject;
 use Illuminate\Queue\SerializesModels;
 use Illuminate\Support\Carbon;
 
 class MonitorBackupJob implements ShouldQueue
 {
-    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels, LocksPerSubject;
+
+    public int $timeout = 60;
 
     public function retryUntil(): Carbon
     {
@@ -25,9 +27,18 @@ public function __construct(protected int $backupId, protected string $upid)
     {
     }
 
+    /**
+     * Wait before retrying after an error instead of retrying in a tight loop.
+     * The last delay repeats until the job gives up.
+     */
+    public function backoff(): array
+    {
+        return [3, 10, 30];
+    }
+
     public function middleware(): array
     {
-        return [new WithoutOverlapping($this->backupId)];
+        return [$this->lockPerSubject($this->backupId)];
     }
 
     public function handle(BackupMonitorService $service): void
diff --git a/app/Jobs/Server/MonitorBackupRestorationJob.php b/app/Jobs/Server/MonitorBackupRestorationJob.php
index c8a75b65d08..ef88de2666b 100644
--- a/app/Jobs/Server/MonitorBackupRestorationJob.php
+++ b/app/Jobs/Server/MonitorBackupRestorationJob.php
@@ -8,13 +8,15 @@
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
-use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Convoy\Jobs\Concerns\LocksPerSubject;
 use Illuminate\Queue\SerializesModels;
 use Illuminate\Support\Carbon;
 
 class MonitorBackupRestorationJob implements ShouldQueue
 {
-    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels, LocksPerSubject;
+
+    public int $timeout = 60;
 
     public function retryUntil(): Carbon
     {
@@ -25,9 +27,18 @@ public function __construct(protected int $serverId, protected string $upid)
     {
     }
 
+    /**
+     * Wait before retrying after an error instead of retrying in a tight loop.
+     * The last delay repeats until the job gives up.
+     */
+    public function backoff(): array
+    {
+        return [3, 10, 30];
+    }
+
     public function middleware(): array
     {
-        return [new WithoutOverlapping($this->serverId)];
+        return [$this->lockPerSubject($this->serverId)];
     }
 
     public function handle(BackupMonitorService $service): void
diff --git a/app/Jobs/Server/MonitorStateJob.php b/app/Jobs/Server/MonitorStateJob.php
index 9375cc992d0..95c6d70d4d7 100644
--- a/app/Jobs/Server/MonitorStateJob.php
+++ b/app/Jobs/Server/MonitorStateJob.php
@@ -19,6 +19,8 @@ class MonitorStateJob implements ShouldQueue
 {
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels, Batchable;
 
+    public int $timeout = 60;
+
     public function retryUntil(): Carbon
     {
         return now()->addMinutes(2);
@@ -33,6 +35,15 @@ public function __construct(
         //
     }
 
+    /**
+     * Wait before retrying after an error instead of retrying in a tight loop.
+     * The last delay repeats until the job gives up.
+     */
+    public function backoff(): array
+    {
+        return [3, 10, 30];
+    }
+
     public function middleware(): array
     {
         return [new SkipIfBatchCancelled()];
diff --git a/app/Jobs/Server/PurgeBackupsJob.php b/app/Jobs/Server/PurgeBackupsJob.php
index 3b487f36bd6..944eb0d8b69 100644
--- a/app/Jobs/Server/PurgeBackupsJob.php
+++ b/app/Jobs/Server/PurgeBackupsJob.php
@@ -10,12 +10,12 @@
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\Middleware\SkipIfBatchCancelled;
-use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Convoy\Jobs\Concerns\LocksPerSubject;
 use Illuminate\Queue\SerializesModels;
 
 class PurgeBackupsJob implements ShouldQueue
 {
-    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels, Batchable;
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels, Batchable, LocksPerSubject;
 
     public int $tries = 3;
 
@@ -26,9 +26,17 @@ public function __construct(protected int $serverId)
         //
     }
 
+    /**
+     * Wait before retrying a failed Proxmox call instead of retrying at once.
+     */
+    public function backoff(): array
+    {
+        return [5, 15];
+    }
+
     public function middleware(): array
     {
-        return [new SkipIfBatchCancelled(), new WithoutOverlapping($this->serverId)];
+        return [new SkipIfBatchCancelled(), $this->lockPerSubject($this->serverId)];
     }
 
     public function handle(PurgeBackupsService $service): void
diff --git a/app/Jobs/Server/SendPowerCommandJob.php b/app/Jobs/Server/SendPowerCommandJob.php
index b5b0f6796fe..1d5154b8e26 100644
--- a/app/Jobs/Server/SendPowerCommandJob.php
+++ b/app/Jobs/Server/SendPowerCommandJob.php
@@ -11,24 +11,32 @@
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\Middleware\SkipIfBatchCancelled;
-use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Convoy\Jobs\Concerns\LocksPerSubject;
 use Illuminate\Queue\SerializesModels;
 
 class SendPowerCommandJob implements ShouldQueue
 {
-    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels, Batchable;
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels, Batchable, LocksPerSubject;
 
     public int $tries = 3;
 
-    public int $timeout = 15;
+    public int $timeout = 60;
 
     public function __construct(protected int $serverId, protected PowerAction $power)
     {
     }
 
+    /**
+     * Wait before retrying a failed Proxmox call instead of retrying at once.
+     */
+    public function backoff(): array
+    {
+        return [5, 15];
+    }
+
     public function middleware(): array
     {
-        return [new SkipIfBatchCancelled(), new WithoutOverlapping($this->serverId)];
+        return [new SkipIfBatchCancelled(), $this->lockPerSubject($this->serverId)];
     }
 
     public function handle(ProxmoxPowerRepository $repository): void
diff --git a/app/Jobs/Server/SyncBuildJob.php b/app/Jobs/Server/SyncBuildJob.php
index 0103b2d75f7..7d7245a5b29 100644
--- a/app/Jobs/Server/SyncBuildJob.php
+++ b/app/Jobs/Server/SyncBuildJob.php
@@ -9,13 +9,15 @@
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\Middleware\SkipIfBatchCancelled;
-use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Convoy\Jobs\Concerns\LocksPerSubject;
 use Illuminate\Queue\SerializesModels;
 use Illuminate\Support\Carbon;
 
 class SyncBuildJob implements ShouldQueue
 {
-    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels, LocksPerSubject;
+
+    public int $timeout = 300;
 
     public function retryUntil(): Carbon
     {
@@ -26,9 +28,18 @@ public function __construct(protected int $serverId)
     {
     }
 
+    /**
+     * Wait before retrying after an error instead of retrying in a tight loop.
+     * The last delay repeats until the job gives up.
+     */
+    public function backoff(): array
+    {
+        return [3, 10, 30];
+    }
+
     public function middleware(): array
     {
-        return [new SkipIfBatchCancelled(), new WithoutOverlapping($this->serverId)];
+        return [new SkipIfBatchCancelled(), $this->lockPerSubject($this->serverId)];
     }
 
     public function handle(SyncBuildService $service): void
diff --git a/app/Jobs/Server/SyncNetworkSettings.php b/app/Jobs/Server/SyncNetworkSettings.php
index d971993d381..03946e17975 100644
--- a/app/Jobs/Server/SyncNetworkSettings.php
+++ b/app/Jobs/Server/SyncNetworkSettings.php
@@ -9,13 +9,15 @@
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\Middleware\SkipIfBatchCancelled;
-use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Convoy\Jobs\Concerns\LocksPerSubject;
 use Illuminate\Queue\SerializesModels;
 use Illuminate\Support\Carbon;
 
 class SyncNetworkSettings implements ShouldQueue
 {
-    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels, LocksPerSubject;
+
+    public int $timeout = 60;
 
     public function retryUntil(): Carbon
     {
@@ -26,9 +28,18 @@ public function __construct(protected int $serverId)
     {
     }
 
+    /**
+     * Wait before retrying after an error instead of retrying in a tight loop.
+     * The last delay repeats until the job gives up.
+     */
+    public function backoff(): array
+    {
+        return [3, 10, 30];
+    }
+
     public function middleware(): array
     {
-        return [new SkipIfBatchCancelled(), new WithoutOverlapping($this->serverId)];
+        return [new SkipIfBatchCancelled(), $this->lockPerSubject($this->serverId)];
     }
 
     public function handle(NetworkService $service): void
diff --git a/app/Jobs/Server/UpdatePasswordJob.php b/app/Jobs/Server/UpdatePasswordJob.php
index 119345f34cc..46c144800f8 100644
--- a/app/Jobs/Server/UpdatePasswordJob.php
+++ b/app/Jobs/Server/UpdatePasswordJob.php
@@ -9,16 +9,16 @@
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\Middleware\SkipIfBatchCancelled;
-use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Convoy\Jobs\Concerns\LocksPerSubject;
 use Illuminate\Queue\SerializesModels;
 
 class UpdatePasswordJob implements ShouldQueue
 {
-    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels, LocksPerSubject;
 
     public int $tries = 15;
 
-    public int $timeout = 10;
+    public int $timeout = 60;
 
     public function backoff(): int
     {
@@ -32,7 +32,7 @@ public function __construct(protected int $serverId, protected string $password)
 
     public function middleware(): array
     {
-        return [new SkipIfBatchCancelled(), new WithoutOverlapping($this->serverId)];
+        return [new SkipIfBatchCancelled(), $this->lockPerSubject($this->serverId)];
     }
 
     public function handle(ServerAuthService $service): void
diff --git a/app/Jobs/Server/WaitUntilVmIsCreatedJob.php b/app/Jobs/Server/WaitUntilVmIsCreatedJob.php
index 9f6d81076bf..3ec03ed1922 100644
--- a/app/Jobs/Server/WaitUntilVmIsCreatedJob.php
+++ b/app/Jobs/Server/WaitUntilVmIsCreatedJob.php
@@ -16,11 +16,22 @@ class WaitUntilVmIsCreatedJob implements ShouldQueue
 {
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
+    public int $timeout = 60;
+
     public function retryUntil(): Carbon
     {
         return now()->addMinutes(30);
     }
 
+    /**
+     * Wait before retrying after an error instead of retrying in a tight loop.
+     * The last delay repeats until the job gives up.
+     */
+    public function backoff(): array
+    {
+        return [3, 10, 30];
+    }
+
     public function middleware(): array
     {
         return [new SkipIfBatchCancelled()];
diff --git a/tests/Unit/Jobs/LocksPerSubjectTest.php b/tests/Unit/Jobs/LocksPerSubjectTest.php
new file mode 100644
index 00000000000..0e828d7258d
--- /dev/null
+++ b/tests/Unit/Jobs/LocksPerSubjectTest.php
@@ -0,0 +1,36 @@
+<?php
+
+use Convoy\Enums\Server\PowerAction;
+use Convoy\Jobs\Node\MonitorIsoDownloadJob;
+use Convoy\Jobs\Server\BuildServerJob;
+use Convoy\Jobs\Server\DeleteServerJob;
+use Convoy\Jobs\Server\MonitorBackupJob;
+use Convoy\Jobs\Server\MonitorBackupRestorationJob;
+use Convoy\Jobs\Server\PurgeBackupsJob;
+use Convoy\Jobs\Server\SendPowerCommandJob;
+use Convoy\Jobs\Server\SyncBuildJob;
+use Convoy\Jobs\Server\SyncNetworkSettings;
+use Convoy\Jobs\Server\UpdatePasswordJob;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+
+// A lock that never expires outlives any job the worker kills for running past
+// its timeout, and blocks that job class for that server until cleared by hand.
+it('gives each per-subject lock an expiry past the job timeout', function (object $job) {
+    $lock = collect($job->middleware())->first(fn ($m) => $m instanceof WithoutOverlapping);
+
+    expect($lock)->not->toBeNull()
+        ->and($lock->key)->toBe(42)
+        ->and($lock->expiresAfter)->toBeGreaterThan($job->timeout)
+        ->and($lock->releaseAfter)->toBeGreaterThan(0);
+})->with([
+    'power' => fn () => new SendPowerCommandJob(42, PowerAction::START),
+    'password' => fn () => new UpdatePasswordJob(42, 'Rt-Aa1!bcdefgh9'),
+    'build' => fn () => new BuildServerJob(42, 1),
+    'delete' => fn () => new DeleteServerJob(42),
+    'purge backups' => fn () => new PurgeBackupsJob(42),
+    'sync build' => fn () => new SyncBuildJob(42),
+    'sync network' => fn () => new SyncNetworkSettings(42),
+    'monitor backup' => fn () => new MonitorBackupJob(42, 'UPID:x'),
+    'monitor restore' => fn () => new MonitorBackupRestorationJob(42, 'UPID:x'),
+    'monitor ISO download' => fn () => new MonitorIsoDownloadJob(42, 'UPID:x'),
+]);