From 0085d50f78ec2bffd6334ba0993ddef8e8a43934 Mon Sep 17 00:00:00 2001 From: Eric Wang <37554696+ericwang401@users.noreply.github.com> Date: Tue, 29 Sep 2026 04:00:58 +0000 Subject: [PATCH 1/3] fix(servers): let reinstall recover a VM-less server and fail properly Two problems surfaced by reinstalling a server whose install had failed: - rebuild() always stopped and deleted the old VM first. When the failed install died before or during the clone there is no VM, Proxmox answers the stop with "does not exist", and the chain fails every time. It now skips straight to the build when isVmDeleted() says the VM is gone. If that check is wrong because the VM does exist, the clone refuses the VMID and nothing is lost. - The chain's catch wrote install_failed through the server captured at dispatch. That copy is serialized with the chain, and when it already read install_failed -- a retry starts from exactly that -- Eloquent saw nothing dirty and skipped the write, leaving the server on "installing" forever. It now updates by ID. --- .../Servers/ServerBuildDispatchService.php | 30 +++++++++++++++++-- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/app/Services/Servers/ServerBuildDispatchService.php b/app/Services/Servers/ServerBuildDispatchService.php index 42a6357f2a0..e430400383c 100644 --- a/app/Services/Servers/ServerBuildDispatchService.php +++ b/app/Services/Servers/ServerBuildDispatchService.php @@ -20,12 +20,16 @@ class ServerBuildDispatchService { + public function __construct(private ServerBuildService $buildService) + { + } + public function build(ServerDeploymentData $deployment): void { $jobs = $this->getChainedBuildJobs($deployment); Bus::chain($jobs) - ->catch(fn () => $deployment->server->update(['status' => Status::INSTALL_FAILED->value])) + ->catch(self::markInstallFailed($deployment->server->id)) ->dispatch(); $deployment->server->update(['status' => Status::INSTALLING->value]); @@ -42,18 +46,38 @@ public function delete(Server $server): void public function rebuild(ServerDeploymentData $deployment): void { + // A retry after a failed install can find no VM at all -- the build + // may have died before or during the clone. Stopping and deleting a VM + // that does not exist fails on Proxmox's "does not exist", which would + // fail the whole chain, so go straight to building instead. If the VM + // does exist after all, the clone refuses its VMID and nothing is lost. $jobs = [ - ...$this->getChainedDeleteJobs($deployment->server), + ...($this->buildService->isVmDeleted($deployment->server) + ? [] + : $this->getChainedDeleteJobs($deployment->server)), ...$this->getChainedBuildJobs($deployment), ]; Bus::chain($jobs) - ->catch(fn () => $deployment->server->update(['status' => Status::INSTALL_FAILED->value])) + ->catch(self::markInstallFailed($deployment->server->id)) ->dispatch(); $deployment->server->update(['status' => Status::INSTALLING->value]); } + /** + * The chain's failure handler. It writes by ID rather than through the + * model captured at dispatch: that copy is serialized with the chain, and + * if it already read install_failed -- as it does when retrying a failed + * install -- Eloquent sees nothing dirty and skips the write, leaving the + * server stuck on "installing". + */ + private static function markInstallFailed(int $serverId): \Closure + { + return static fn () => Server::whereKey($serverId) + ->update(['status' => Status::INSTALL_FAILED->value]); + } + private function getChainedBuildJobs(ServerDeploymentData $deployment): array { // Base jobs: either create a new server or sync an existing one From 1c3c983dcfc388e7b781aa0b8b6f223e828c867c Mon Sep 17 00:00:00 2001 From: Eric Wang <37554696+ericwang401@users.noreply.github.com> Date: Tue, 29 Sep 2026 04:00:59 +0000 Subject: [PATCH 2/3] feat(servers): retry a failed install from the install-failed screen A failed install was a dead end: the client showed "contact your administrator" and the API refused every settings route with 409. The screen now carries the reinstall form (template, a new OS password, start after install), and AuthenticateServerAccess lets install_failed servers through to the two routes it needs -- the template list and reinstall. Any other blocked state, like suspended, still gets 409. ReinstallServerCard takes an optional title, description and button label so the same form serves both places. Tested with Pest (the missing-VM chain, the stale-status catch, and that other states stay blocked), a Playwright spec against a seeded install_failed server, and a real retry on a Proxmox node of a server whose VM was never created. --- CHANGELOG.md | 17 +++ .../Server/AuthenticateServerAccess.php | 10 ++ database/seeders/E2eSeeder.php | 23 +++- lang/en_US/server/settings.php | 7 ++ .../servers/InstallFailedScreen.tsx | 37 ++++++ .../partials/general/ReinstallServerCard.tsx | 16 ++- resources/scripts/routers/ServerRouter.tsx | 11 +- routes/api-client.php | 5 +- .../Client/Servers/RetryInstallTest.php | 119 ++++++++++++++++++ tests/e2e/servers/retry-install.spec.ts | 38 ++++++ 10 files changed, 268 insertions(+), 15 deletions(-) create mode 100644 resources/scripts/components/servers/InstallFailedScreen.tsx create mode 100644 tests/Feature/Controllers/Client/Servers/RetryInstallTest.php create mode 100644 tests/e2e/servers/retry-install.spec.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index deb96919c74..cc0c607dc87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,23 @@ This file is a running track of new features and fixes to each version of the pa The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project follows [Semantic Versioning](https://semver.org) guidelines. +## Unreleased + +### Added + +- A server whose installation failed can now be retried. The "Install failed" screen was a dead end that only said to + contact an administrator; it now offers the reinstall form, and the API accepts a reinstall for a server in that + state. Every other blocked state, such as suspended, still refuses it. + +### Fixed + +- Fixed reinstalling a server whose VM no longer exists failing every time. Reinstall stopped and deleted the old VM + first, and Proxmox answers both with "does not exist" when a previous install died before or during the clone. It + now skips straight to building when there is no VM. +- Fixed a failed reinstall of an already-failed server staying on "Installing" forever instead of going back to + "Install failed". The failure handler wrote through the copy of the server captured when the reinstall started, and + when that copy already read "install_failed" the write was skipped as a no-op. + ## v4.7.0-rc.2 ### Fixed diff --git a/app/Http/Middleware/Client/Server/AuthenticateServerAccess.php b/app/Http/Middleware/Client/Server/AuthenticateServerAccess.php index 5a35d92af26..aa4a73f969c 100644 --- a/app/Http/Middleware/Client/Server/AuthenticateServerAccess.php +++ b/app/Http/Middleware/Client/Server/AuthenticateServerAccess.php @@ -6,6 +6,7 @@ use Convoy\Models\Server; use Illuminate\Http\Request; use Symfony\Component\HttpFoundation\Response; +use Convoy\Enums\Server\Status; use Convoy\Exceptions\Http\Server\ServerStatusConflictException; use Symfony\Component\HttpKernel\Exception\NotFoundHttpException; @@ -39,6 +40,15 @@ public function handle(Request $request, Closure $next): Response return $next($request); } + // A failed install can be retried: reinstalling is the retry, and + // the form needs the template list. Every other state stays blocked. + if ( + $server->status === Status::INSTALL_FAILED->value + && $request->routeIs('client.servers.settings.template-groups', 'client.servers.settings.reinstall') + ) { + return $next($request); + } + throw $exception; } diff --git a/database/seeders/E2eSeeder.php b/database/seeders/E2eSeeder.php index df537cc5b75..d091c814f58 100644 --- a/database/seeders/E2eSeeder.php +++ b/database/seeders/E2eSeeder.php @@ -3,7 +3,9 @@ namespace Database\Seeders; use Convoy\Models\Location; +use Convoy\Enums\Server\Status; use Convoy\Models\Node; +use Convoy\Models\Server; use Convoy\Models\Template; use Convoy\Models\TemplateGroup; use Convoy\Models\User; @@ -23,7 +25,7 @@ class E2eSeeder extends Seeder { public function run(): void { - User::factory()->create([ + $admin = User::factory()->create([ 'name' => 'E2E Admin', 'email' => 'admin@e2e.test', 'root_admin' => true, @@ -46,5 +48,24 @@ public function run(): void 'vmid' => 9000, 'hidden' => false, ]); + + // A server whose install failed, for the retry screen. The UUID is + // fixed so the specs can open /servers/e2e00000. + Server::create([ + 'uuid' => 'e2e00000-0000-4000-8000-000000000001', + 'uuid_short' => 'e2e00000', + 'status' => Status::INSTALL_FAILED->value, + 'name' => 'E2E Failed Install', + 'hostname' => 'e2e-failed', + 'user_id' => $admin->id, + 'node_id' => $node->id, + 'vmid' => 9100, + 'cpu' => 1, + 'memory' => 512 * 1024 * 1024, + 'disk' => 4 * 1024 * 1024 * 1024, + 'snapshot_limit' => 0, + 'backup_limit' => null, + 'bandwidth_limit' => null, + ]); } } diff --git a/lang/en_US/server/settings.php b/lang/en_US/server/settings.php index 1564707efc3..526a3bf2ea6 100644 --- a/lang/en_US/server/settings.php +++ b/lang/en_US/server/settings.php @@ -16,6 +16,13 @@ 'description' => 'Are you sure you want to reinstall this server? All data will be lost.', ], ], + 'retry_install' => [ + 'failed_title' => 'Install failed', + 'failed_message' => 'Your server failed to install. Pick a template to try again, or contact your administrator if it keeps failing.', + 'title' => 'Retry Installation', + 'description' => 'Installs the server again from the template you pick, with a new OS password.', + 'button' => 'Retry Installation', + ], 'isos' => [ 'title' => 'Mountable ISOs', 'empty' => 'There are no ISOs', diff --git a/resources/scripts/components/servers/InstallFailedScreen.tsx b/resources/scripts/components/servers/InstallFailedScreen.tsx new file mode 100644 index 00000000000..27f8ab8f315 --- /dev/null +++ b/resources/scripts/components/servers/InstallFailedScreen.tsx @@ -0,0 +1,37 @@ +import { ExclamationCircleIcon } from '@heroicons/react/24/outline' +import { useTranslation } from 'react-i18next' + +import PageContentBlock from '@/components/elements/PageContentBlock' + +import ReinstallServerCard from '@/components/servers/settings/partials/general/ReinstallServerCard' + + +// A failed install used to be a dead end ("contact your administrator"). The +// server can be reinstalled from here instead: the retry is a reinstall, and +// the API allows it for install_failed servers. +const InstallFailedScreen = () => { + const { t } = useTranslation('server.settings') + + return ( + +
+
+ +

+ {t('retry_install.failed_title')} +

+

+ {t('retry_install.failed_message')} +

+
+ +
+
+ ) +} + +export default InstallFailedScreen diff --git a/resources/scripts/components/servers/settings/partials/general/ReinstallServerCard.tsx b/resources/scripts/components/servers/settings/partials/general/ReinstallServerCard.tsx index 165482dd87d..5877b3795dd 100644 --- a/resources/scripts/components/servers/settings/partials/general/ReinstallServerCard.tsx +++ b/resources/scripts/components/servers/settings/partials/general/ReinstallServerCard.tsx @@ -19,7 +19,13 @@ import TextInputForm from '@/components/elements/forms/TextInputForm' import TemplatesSelectForm from '@/components/servers/settings/TemplatesSelectForm' -const ReinstallServerCard = () => { +interface Props { + title?: string + description?: string + submitLabel?: string +} + +const ReinstallServerCard = ({ title, description, submitLabel }: Props) => { const { t } = useTranslation('server.settings') const { t: tStrings } = useTranslation('strings') const [isModalOpen, setIsModalOpen] = useState(false) @@ -87,13 +93,15 @@ const ReinstallServerCard = () => { - {t('reinstall.title')} + + {title ?? t('reinstall.title')} +

- {t('reinstall.description')} + {description ?? t('reinstall.description')}

@@ -119,7 +127,7 @@ const ReinstallServerCard = () => { color='danger' size='sm' > - {tStrings('reinstall')} + {submitLabel ?? tStrings('reinstall')} diff --git a/resources/scripts/routers/ServerRouter.tsx b/resources/scripts/routers/ServerRouter.tsx index f1a51f96e8a..f1bdf3f29ee 100644 --- a/resources/scripts/routers/ServerRouter.tsx +++ b/resources/scripts/routers/ServerRouter.tsx @@ -17,6 +17,8 @@ import ScreenBlock, { ErrorMessage } from '@/components/elements/ScreenBlock' import Spinner from '@/components/elements/Spinner' import { NavigationBarContext } from '@/components/elements/navigation/NavigationBar' +import InstallFailedScreen from '@/components/servers/InstallFailedScreen' + export const routes: Route[] = [ { @@ -206,14 +208,7 @@ const ServerRouter = () => { /> ) case 'install_failed': - return ( - - ) + return case null: return null default: diff --git a/routes/api-client.php b/routes/api-client.php index 8ae08a8e059..59210333391 100644 --- a/routes/api-client.php +++ b/routes/api-client.php @@ -31,8 +31,9 @@ Route::post('/rename', [Client\Servers\SettingsController::class, 'rename']); Route::get( '/template-groups', [Client\Servers\SettingsController::class, 'getTemplateGroups'], - ); - Route::post('/reinstall', [Client\Servers\SettingsController::class, 'reinstall']); + )->name('servers.settings.template-groups'); + Route::post('/reinstall', [Client\Servers\SettingsController::class, 'reinstall']) + ->name('servers.settings.reinstall'); Route::get( '/hardware/boot-order', [Client\Servers\SettingsController::class, 'getBootOrder'], diff --git a/tests/Feature/Controllers/Client/Servers/RetryInstallTest.php b/tests/Feature/Controllers/Client/Servers/RetryInstallTest.php new file mode 100644 index 00000000000..307b73ef6dc --- /dev/null +++ b/tests/Feature/Controllers/Client/Servers/RetryInstallTest.php @@ -0,0 +1,119 @@ + $node->id, 'name' => 'Linux', 'hidden' => false]); + + return Template::create([ + 'template_group_id' => $group->id, + 'name' => 'Debian 12', + 'vmid' => 9000, + 'hidden' => false, + ]); +} + +function fakeMissingVm(): void +{ + Http::fake([ + '*/config' => Http::response([ + 'data' => null, + 'message' => "Configuration file 'nodes/proxmox/qemu-server/100.conf' does not exist\n", + ], 500), + '*' => Http::response(['data' => 'dummy-upid'], 200), + ]); +} + +it('retries a failed install straight into a build when its VM is gone', function () { + fakeMissingVm(); + + [$user, $_, $node, $server] = createServerModel(); + $server->update(['status' => Status::INSTALL_FAILED->value]); + $template = createTemplateFor($node); + + $this->actingAs($user)->postJson("/api/client/servers/{$server->uuid}/settings/reinstall", [ + 'template_uuid' => $template->uuid, + 'account_password' => 'Rt-Aa1!bcdefgh9', + 'start_on_completion' => false, + ])->assertNoContent(); + + expect($server->fresh()->status)->toBe(Status::INSTALLING->value); + // Stopping and deleting a VM that isn't there fails on Proxmox, so the + // chain has to start at the build. + Queue::assertPushed(BuildServerJob::class); + Queue::assertNotPushed(SendPowerCommandJob::class); +}); + +it('still stops and deletes the old VM first when it exists', function () { + Http::fake([ + '*/config' => Http::response( + file_get_contents(base_path('tests/Fixtures/Repositories/Server/GetServerConfigData.json')), + 200, + ), + '*' => Http::response(['data' => 'dummy-upid'], 200), + ]); + + [$user, $_, $node, $server] = createServerModel(); + $template = createTemplateFor($node); + + $this->actingAs($user)->postJson("/api/client/servers/{$server->uuid}/settings/reinstall", [ + 'template_uuid' => $template->uuid, + 'account_password' => 'Rt-Aa1!bcdefgh9', + 'start_on_completion' => false, + ])->assertNoContent(); + + Queue::assertPushed(SendPowerCommandJob::class); + Queue::assertNotPushed(BuildServerJob::class); +}); + +it('marks a retried install failed when its chain fails', function () { + fakeMissingVm(); + // Serialize pushed jobs as the real queue does. The failure handler then + // runs against what was captured at dispatch -- a server that already read + // install_failed -- which is what used to leave it stuck on "installing". + Queue::fake()->serializeAndRestore(); + + [$_, $_, $node, $server] = createServerModel(); + $server->update(['status' => Status::INSTALL_FAILED->value]); + + app(ServerBuildDispatchService::class)->rebuild(ServerDeploymentData::from([ + 'server' => $server, + 'template' => createTemplateFor($node), + 'account_password' => 'Rt-Aa1!bcdefgh9', + 'should_create_server' => true, + 'start_on_completion' => false, + ])); + + expect($server->fresh()->status)->toBe(Status::INSTALLING->value); + + Queue::pushed(BuildServerJob::class)->first() + ->invokeChainCatchCallbacks(new RuntimeException('clone failed')); + + expect(Server::find($server->id)->status)->toBe(Status::INSTALL_FAILED->value); +}); + +it('keeps reinstall blocked for servers in any other state', function () { + fakeMissingVm(); + + [$user, $_, $node, $server] = createServerModel(); + $server->update(['status' => Status::SUSPENDED->value]); + + $this->actingAs($user)->postJson("/api/client/servers/{$server->uuid}/settings/reinstall", [ + 'template_uuid' => createTemplateFor($node)->uuid, + 'account_password' => 'Rt-Aa1!bcdefgh9', + 'start_on_completion' => false, + ])->assertConflict(); + + Queue::assertNothingPushed(); +}); diff --git a/tests/e2e/servers/retry-install.spec.ts b/tests/e2e/servers/retry-install.spec.ts new file mode 100644 index 00000000000..af49316bb92 --- /dev/null +++ b/tests/e2e/servers/retry-install.spec.ts @@ -0,0 +1,38 @@ +import { expect, test } from '@playwright/test' + +import { signInAsAdmin } from '../support/auth' + +test.beforeEach(async ({ page }) => { + await signInAsAdmin(page) +}) + +// A failed install used to be a dead end. The seeded server (E2eSeeder) is in +// install_failed, and the retry has to go through the reinstall endpoint. +test('offers to retry a failed install', async ({ page }) => { + await page.goto('/servers/e2e00000') + + await expect( + page.getByRole('heading', { name: 'Install failed' }) + ).toBeVisible() + + await page.getByLabel('Template Group').click() + await page.getByRole('option', { name: 'E2E Linux' }).click() + await page.getByLabel('Template', { exact: true }).click() + await page.getByRole('option', { name: 'E2E Debian 12' }).click() + await page.getByLabel('System OS Password').fill('Rt-Aa1!bcdefgh9') + + // The seeded node isn't a real Proxmox host, so answer the reinstall here + // rather than queue a build that could only fail. + let body: Record | undefined + await page.route('**/settings/reinstall', async route => { + body = route.request().postDataJSON() + await route.fulfill({ status: 204 }) + }) + + await page.getByRole('button', { name: 'Retry Installation' }).click() + await page.getByRole('button', { name: 'Confirm' }).click() + + await expect(page.getByRole('heading', { name: 'Installing' })).toBeVisible() + expect(body).toMatchObject({ account_password: 'Rt-Aa1!bcdefgh9' }) + expect(body?.template_uuid).toBeTruthy() +}) From 7456b332df5d8aca1676003c95252ae9a274e05e Mon Sep 17 00:00:00 2001 From: Eric Wang <37554696+ericwang401@users.noreply.github.com> Date: Fri, 16 Oct 2026 00:33:07 +0000 Subject: [PATCH 3/3] feat(servers): update the installing and restoring screens by themselves The "Installing", "Restoring Backup" and "Restoring Snapshot" screens only ever change when a queued job finishes, but nothing told the page: it stayed on the screen until the user refreshed. ServerRouter now re-fetches the server every 5s while it's in one of those states, so the screen moves on as soon as the work is done -- including straight after retrying a failed install. A failed poll is simply retried on the next tick. ScreenBlock takes a `spin` flag, used by those three screens, that turns the icon slowly (2s per turn, off for reduced motion) to show it's live. The icon's border box moves to a wrapper so only the glyph turns. The retry e2e spec now fakes the install finishing and expects the page to leave "Installing" without a reload; it fails without the polling. --- CHANGELOG.md | 3 +++ .../components/elements/ScreenBlock.tsx | 13 ++++++++++- resources/scripts/routers/ServerRouter.tsx | 23 +++++++++++++++++++ tests/e2e/servers/retry-install.spec.ts | 16 +++++++++++++ 4 files changed, 54 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc0c607dc87..e08c237ea01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,9 @@ follows [Semantic Versioning](https://semver.org) guidelines. - A server whose installation failed can now be retried. The "Install failed" screen was a dead end that only said to contact an administrator; it now offers the reinstall form, and the API accepts a reinstall for a server in that state. Every other blocked state, such as suspended, still refuses it. +- The "Installing", "Restoring Backup" and "Restoring Snapshot" screens now update by themselves. They check the + server every few seconds and move on once the work is done, with a spinning icon to show they're live; before, they + stayed put until the page was refreshed. ### Fixed diff --git a/resources/scripts/components/elements/ScreenBlock.tsx b/resources/scripts/components/elements/ScreenBlock.tsx index 728e4d7a409..cc9e87b3201 100644 --- a/resources/scripts/components/elements/ScreenBlock.tsx +++ b/resources/scripts/components/elements/ScreenBlock.tsx @@ -14,6 +14,8 @@ interface BaseProps { message: string full?: boolean center?: boolean + /** Spin the icon, for screens that update themselves when the work is done. */ + spin?: boolean children?: ReactNode onRetry?: () => void onBack?: () => void @@ -36,6 +38,7 @@ const ScreenBlock = ({ icon: Icon, message, center, + spin, onBack, onRetry, full, @@ -54,7 +57,15 @@ const ScreenBlock = ({ }`} > {Icon && ( - +
+ +
)}

diff --git a/resources/scripts/routers/ServerRouter.tsx b/resources/scripts/routers/ServerRouter.tsx index f1bdf3f29ee..da948e80dbe 100644 --- a/resources/scripts/routers/ServerRouter.tsx +++ b/resources/scripts/routers/ServerRouter.tsx @@ -169,6 +169,26 @@ const ServerRouter = () => { } }, [id]) + // These statuses are only ever cleared by a queued job in the background, so + // poll while the server is in one and let the screen move on by itself + // instead of waiting for the user to refresh. + const isBusy = + server?.status === 'installing' || + server?.status === 'restoring_backup' || + server?.status === 'restoring_snapshot' + + useEffect(() => { + if (!isBusy) return + + const interval = setInterval(() => { + // A failed poll is retried on the next tick; the screen keeps + // showing the last known status meanwhile. + getServer(id as string).catch(() => {}) + }, 5000) + + return () => clearInterval(interval) + }, [id, isBusy]) + const getScreenBlock = (status: EloquentStatus) => { switch (status) { case 'suspended': @@ -185,6 +205,7 @@ const ServerRouter = () => { @@ -194,6 +215,7 @@ const ServerRouter = () => { @@ -203,6 +225,7 @@ const ServerRouter = () => { diff --git a/tests/e2e/servers/retry-install.spec.ts b/tests/e2e/servers/retry-install.spec.ts index af49316bb92..5c8f6d5f490 100644 --- a/tests/e2e/servers/retry-install.spec.ts +++ b/tests/e2e/servers/retry-install.spec.ts @@ -35,4 +35,20 @@ test('offers to retry a failed install', async ({ page }) => { await expect(page.getByRole('heading', { name: 'Installing' })).toBeVisible() expect(body).toMatchObject({ account_password: 'Rt-Aa1!bcdefgh9' }) expect(body?.template_uuid).toBeTruthy() + + // Pretend the install has now finished. The Installing screen polls, so it + // has to move on by itself -- nobody reloads the page. + await page.route('**/api/client/servers/e2e00000', async route => { + const response = await route.fetch() + const json = await response.json() + json.data.status = null + await route.fulfill({ response, json }) + }) + + await expect(page.getByRole('heading', { name: 'Installing' })).toBeHidden({ + timeout: 15_000, + }) + await expect( + page.getByRole('heading', { name: 'Install failed' }) + ).toBeHidden() })