feat(servers): retry a failed install

CHANGELOG.md

@@ -7,11 +7,27 @@ follows [Semantic Versioning](https://semver.org) guidelines.
 
 ## Unreleased
 
+### Added
+
+- A server whose installation failed can now be retried. The "Install failed" screen was a dead end that only said to
+  contact an administrator; it now offers the reinstall form, and the API accepts a reinstall for a server in that
+  state. Every other blocked state, such as suspended, still refuses it.
+- The "Installing", "Restoring Backup" and "Restoring Snapshot" screens now update by themselves. They check the
+  server every few seconds and move on once the work is done, with a spinning icon to show they're live; before, they
+  stayed put until the page was refreshed.
+
 ### Fixed
 
 - Fixed every error from Proxmox filling the log with three copies of the same stack trace -- about 256 lines per error,
   every few seconds while a page polled a server whose VM was gone. An error is now logged once, with one trace, and its
   message is Proxmox's own ("Proxmox returned 500: Configuration file ... does not exist").
+- Fixed reinstalling a server whose VM no longer exists failing every time. Reinstall stopped and deleted the old VM
+  first, and Proxmox answers both with "does not exist" when a previous install died before or during the clone. It
+  now skips straight to building when there is no VM.
+- Fixed a failed reinstall of an already-failed server staying on "Installing" forever instead of going back to
+  "Install failed". The failure handler wrote through the copy of the server captured when the reinstall started, and
+  when that copy already read "install_failed" the write was skipped as a no-op.
+
 
 ## v4.7.0-rc.2
 

app/Http/Middleware/Client/Server/AuthenticateServerAccess.php

@@ -6,6 +6,7 @@
 use Convoy\Models\Server;
 use Illuminate\Http\Request;
 use Symfony\Component\HttpFoundation\Response;
+use Convoy\Enums\Server\Status;
 use Convoy\Exceptions\Http\Server\ServerStatusConflictException;
 use Symfony\Component\HttpKernel\Exception\NotFoundHttpException;
 
@@ -39,6 +40,15 @@ public function handle(Request $request, Closure $next): Response
                 return $next($request);
             }
 
+            // A failed install can be retried: reinstalling is the retry, and
+            // the form needs the template list. Every other state stays blocked.
+            if (
+                $server->status === Status::INSTALL_FAILED->value
+                && $request->routeIs('client.servers.settings.template-groups', 'client.servers.settings.reinstall')
+            ) {
+                return $next($request);
+            }
+
             throw $exception;
         }
 

app/Services/Servers/ServerBuildDispatchService.php

@@ -20,12 +20,16 @@
 
 class ServerBuildDispatchService
 {
+    public function __construct(private ServerBuildService $buildService)
+    {
+    }
+
     public function build(ServerDeploymentData $deployment): void
     {
         $jobs = $this->getChainedBuildJobs($deployment);
 
         Bus::chain($jobs)
-            ->catch(fn () => $deployment->server->update(['status' => Status::INSTALL_FAILED->value]))
+            ->catch(self::markInstallFailed($deployment->server->id))
             ->dispatch();
 
         $deployment->server->update(['status' => Status::INSTALLING->value]);
@@ -42,18 +46,38 @@ public function delete(Server $server): void
 
     public function rebuild(ServerDeploymentData $deployment): void
     {
+        // A retry after a failed install can find no VM at all -- the build
+        // may have died before or during the clone. Stopping and deleting a VM
+        // that does not exist fails on Proxmox's "does not exist", which would
+        // fail the whole chain, so go straight to building instead. If the VM
+        // does exist after all, the clone refuses its VMID and nothing is lost.
         $jobs = [
-            ...$this->getChainedDeleteJobs($deployment->server),
+            ...($this->buildService->isVmDeleted($deployment->server)
+                ? []
+                : $this->getChainedDeleteJobs($deployment->server)),
             ...$this->getChainedBuildJobs($deployment),
         ];
 
         Bus::chain($jobs)
-            ->catch(fn () => $deployment->server->update(['status' => Status::INSTALL_FAILED->value]))
+            ->catch(self::markInstallFailed($deployment->server->id))
             ->dispatch();
 
         $deployment->server->update(['status' => Status::INSTALLING->value]);
     }
 
+    /**
+     * The chain's failure handler. It writes by ID rather than through the
+     * model captured at dispatch: that copy is serialized with the chain, and
+     * if it already read install_failed -- as it does when retrying a failed
+     * install -- Eloquent sees nothing dirty and skips the write, leaving the
+     * server stuck on "installing".
+     */
+    private static function markInstallFailed(int $serverId): \Closure
+    {
+        return static fn () => Server::whereKey($serverId)
+            ->update(['status' => Status::INSTALL_FAILED->value]);
+    }
+
     private function getChainedBuildJobs(ServerDeploymentData $deployment): array
     {
         // Base jobs: either create a new server or sync an existing one

database/seeders/E2eSeeder.php

@@ -3,7 +3,9 @@
 namespace Database\Seeders;
 
 use Convoy\Models\Location;
+use Convoy\Enums\Server\Status;
 use Convoy\Models\Node;
+use Convoy\Models\Server;
 use Convoy\Models\Template;
 use Convoy\Models\TemplateGroup;
 use Convoy\Models\User;
@@ -23,7 +25,7 @@ class E2eSeeder extends Seeder
 {
     public function run(): void
     {
-        User::factory()->create([
+        $admin = User::factory()->create([
             'name' => 'E2E Admin',
             'email' => 'admin@e2e.test',
             'root_admin' => true,
@@ -46,5 +48,24 @@ public function run(): void
             'vmid' => 9000,
             'hidden' => false,
         ]);
+
+        // A server whose install failed, for the retry screen. The UUID is
+        // fixed so the specs can open /servers/e2e00000.
+        Server::create([
+            'uuid' => 'e2e00000-0000-4000-8000-000000000001',
+            'uuid_short' => 'e2e00000',
+            'status' => Status::INSTALL_FAILED->value,
+            'name' => 'E2E Failed Install',
+            'hostname' => 'e2e-failed',
+            'user_id' => $admin->id,
+            'node_id' => $node->id,
+            'vmid' => 9100,
+            'cpu' => 1,
+            'memory' => 512 * 1024 * 1024,
+            'disk' => 4 * 1024 * 1024 * 1024,
+            'snapshot_limit' => 0,
+            'backup_limit' => null,
+            'bandwidth_limit' => null,
+        ]);
     }
 }

lang/en_US/server/settings.php

@@ -16,6 +16,13 @@
             'description' => 'Are you sure you want to reinstall this server? All data will be lost.',
         ],
     ],
+    'retry_install' => [
+        'failed_title' => 'Install failed',
+        'failed_message' => 'Your server failed to install. Pick a template to try again, or contact your administrator if it keeps failing.',
+        'title' => 'Retry Installation',
+        'description' => 'Installs the server again from the template you pick, with a new OS password.',
+        'button' => 'Retry Installation',
+    ],
     'isos' => [
         'title' => 'Mountable ISOs',
         'empty' => 'There are no ISOs',

resources/scripts/components/elements/ScreenBlock.tsx

@@ -14,6 +14,8 @@ interface BaseProps {
     message: string
     full?: boolean
     center?: boolean
+    /** Spin the icon, for screens that update themselves when the work is done. */
+    spin?: boolean
     children?: ReactNode
     onRetry?: () => void
     onBack?: () => void
@@ -36,6 +38,7 @@ const ScreenBlock = ({
     icon: Icon,
     message,
     center,
+    spin,
     onBack,
     onRetry,
     full,
@@ -54,7 +57,15 @@ const ScreenBlock = ({
                 }`}
             >
                 {Icon && (
-                    <Icon className='w-16 h-16 border dark:border-stone-600 rounded-md p-3 text-black dark:text-stone-400 mx-auto' />
+                    <div className='w-16 h-16 border dark:border-stone-600 rounded-md p-3 mx-auto'>
+                        <Icon
+                            className={`w-full h-full text-black dark:text-stone-400 ${
+                                spin
+                                    ? 'animate-[spin_2s_linear_infinite] motion-reduce:animate-none'
+                                    : ''
+                            }`}
+                        />
+                    </div>
                 )}
 
                 <h2 className='text-stone-900 dark:text-white font-bold text-4xl mt-6'>

resources/scripts/components/servers/InstallFailedScreen.tsx

@@ -0,0 +1,37 @@
+import { ExclamationCircleIcon } from '@heroicons/react/24/outline'
+import { useTranslation } from 'react-i18next'
+
+import PageContentBlock from '@/components/elements/PageContentBlock'
+
+import ReinstallServerCard from '@/components/servers/settings/partials/general/ReinstallServerCard'
+
+
+// A failed install used to be a dead end ("contact your administrator"). The
+// server can be reinstalled from here instead: the retry is a reinstall, and
+// the API allows it for install_failed servers.
+const InstallFailedScreen = () => {
+    const { t } = useTranslation('server.settings')
+
+    return (
+        <PageContentBlock title={t('retry_install.failed_title')}>
+            <div className='mx-auto w-full max-w-xl space-y-6'>
+                <div className='text-center'>
+                    <ExclamationCircleIcon className='w-16 h-16 border dark:border-stone-600 rounded-md p-3 text-black dark:text-stone-400 mx-auto' />
+                    <h2 className='text-stone-900 dark:text-white font-bold text-4xl mt-6'>
+                        {t('retry_install.failed_title')}
+                    </h2>
+                    <p className='description-small mt-3'>
+                        {t('retry_install.failed_message')}
+                    </p>
+                </div>
+                <ReinstallServerCard
+                    title={t('retry_install.title')}
+                    description={t('retry_install.description')}
+                    submitLabel={t('retry_install.button')}
+                />
+            </div>
+        </PageContentBlock>
+    )
+}
+
+export default InstallFailedScreen

resources/scripts/components/servers/settings/partials/general/ReinstallServerCard.tsx

@@ -19,7 +19,13 @@ import TextInputForm from '@/components/elements/forms/TextInputForm'
 import TemplatesSelectForm from '@/components/servers/settings/TemplatesSelectForm'
 
 
-const ReinstallServerCard = () => {
+interface Props {
+    title?: string
+    description?: string
+    submitLabel?: string
+}
+
+const ReinstallServerCard = ({ title, description, submitLabel }: Props) => {
     const { t } = useTranslation('server.settings')
     const { t: tStrings } = useTranslation('strings')
     const [isModalOpen, setIsModalOpen] = useState(false)
@@ -87,13 +93,15 @@ const ReinstallServerCard = () => {
             <FormProvider {...form}>
                 <FormCard className='w-full'>
                     <FormCard.Body>
-                        <FormCard.Title>{t('reinstall.title')}</FormCard.Title>
+                        <FormCard.Title>
+                            {title ?? t('reinstall.title')}
+                        </FormCard.Title>
                         <FlashMessageRender
                             className='mt-3'
                             byKey={`servers.${server.uuid}.settings.general.reinstall`}
                         />
                         <p className='description-small mt-3'>
-                            {t('reinstall.description')}
+                            {description ?? t('reinstall.description')}
                         </p>
                         <div className='flex flex-col space-y-3 mt-3'>
                             <TemplatesSelectForm />
@@ -119,7 +127,7 @@ const ReinstallServerCard = () => {
                             color='danger'
                             size='sm'
                         >
-                            {tStrings('reinstall')}
+                            {submitLabel ?? tStrings('reinstall')}
                         </Button>
                     </FormCard.Footer>
                 </FormCard>

resources/scripts/routers/ServerRouter.tsx

@@ -17,6 +17,8 @@ import ScreenBlock, { ErrorMessage } from '@/components/elements/ScreenBlock'
 import Spinner from '@/components/elements/Spinner'
 import { NavigationBarContext } from '@/components/elements/navigation/NavigationBar'
 
+import InstallFailedScreen from '@/components/servers/InstallFailedScreen'
+
 
 export const routes: Route[] = [
     {
@@ -167,6 +169,26 @@ const ServerRouter = () => {
         }
     }, [id])
 
+    // These statuses are only ever cleared by a queued job in the background, so
+    // poll while the server is in one and let the screen move on by itself
+    // instead of waiting for the user to refresh.
+    const isBusy =
+        server?.status === 'installing' ||
+        server?.status === 'restoring_backup' ||
+        server?.status === 'restoring_snapshot'
+
+    useEffect(() => {
+        if (!isBusy) return
+
+        const interval = setInterval(() => {
+            // A failed poll is retried on the next tick; the screen keeps
+            // showing the last known status meanwhile.
+            getServer(id as string).catch(() => {})
+        }, 5000)
+
+        return () => clearInterval(interval)
+    }, [id, isBusy])
+
     const getScreenBlock = (status: EloquentStatus) => {
         switch (status) {
             case 'suspended':
@@ -183,6 +205,7 @@ const ServerRouter = () => {
                     <ScreenBlock
                         center
                         icon={ArrowPathIcon}
+                        spin
                         message='Your server is being installed. This can take from 1-15 minutes.'
                         title='Installing'
                     />
@@ -192,6 +215,7 @@ const ServerRouter = () => {
                     <ScreenBlock
                         center
                         icon={ArrowPathIcon}
+                        spin
                         message='Your server is being restored from a backup. This can take from 1-15 minutes.'
                         title='Restoring Backup'
                     />
@@ -201,19 +225,13 @@ const ServerRouter = () => {
                     <ScreenBlock
                         center
                         icon={ArrowPathIcon}
+                        spin
                         message='Your server is being restored from a snapshot. This can take from 1-15 minutes.'
                         title='Restoring Snapshot'
                     />
                 )
             case 'install_failed':
-                return (
-                    <ScreenBlock
-                        center
-                        icon={ExclamationCircleIcon}
-                        message='Your server failed to install. Please contact your administrator.'
-                        title='Install failed'
-                    />
-                )
+                return <InstallFailedScreen />
             case null:
                 return null
             default:

routes/api-client.php

@@ -31,8 +31,9 @@
         Route::post('/rename', [Client\Servers\SettingsController::class, 'rename']);
         Route::get(
             '/template-groups', [Client\Servers\SettingsController::class, 'getTemplateGroups'],
-        );
-        Route::post('/reinstall', [Client\Servers\SettingsController::class, 'reinstall']);
+        )->name('servers.settings.template-groups');
+        Route::post('/reinstall', [Client\Servers\SettingsController::class, 'reinstall'])
+             ->name('servers.settings.reinstall');
 
         Route::get(
             '/hardware/boot-order', [Client\Servers\SettingsController::class, 'getBootOrder'],