diff --git a/app/Actions/Server/InstallServer.php b/app/Actions/Server/InstallServer.php index 058fefbe5..968353be2 100644 --- a/app/Actions/Server/InstallServer.php +++ b/app/Actions/Server/InstallServer.php @@ -18,6 +18,8 @@ class InstallServer { + private const MAX_WAIT_SECONDS = 180; + protected Server $server; /** @@ -27,20 +29,33 @@ public function run(Server $server): void { $this->server = $server; - $maxWait = 180; + $maxWait = self::MAX_WAIT_SECONDS; + $connected = false; + $lastError = null; + while ($maxWait > 0) { - if (! $this->server->provider()->isRunning()) { - continue; - } - try { - $this->server->ssh()->connect(); - break; - } catch (SSHConnectionError) { - // ignore + if ($this->server->provider()->isRunning()) { + try { + $this->server->ssh()->connect(); + $connected = true; + + break; + } catch (SSHConnectionError $e) { + $lastError = $e; + } } + Sleep::sleep(10); $maxWait -= 10; } + + if (! $connected) { + throw new SSHConnectionError( + 'The server did not become reachable within '.self::MAX_WAIT_SECONDS.' seconds.', + previous: $lastError, + ); + } + $this->install(); $this->server->update([ 'status' => ServerStatus::READY, diff --git a/tests/Feature/Jobs/ServerInstallJobTest.php b/tests/Feature/Jobs/ServerInstallJobTest.php index e951f71b5..9734112d9 100644 --- a/tests/Feature/Jobs/ServerInstallJobTest.php +++ b/tests/Feature/Jobs/ServerInstallJobTest.php @@ -1,9 +1,17 @@ 'server-installation-failed', ]); }); + +test('waiting for the provider is bounded and paced when the instance never becomes active', function () { + SSH::fake(); + + $polls = 0; + Http::fake([ + 'api.digitalocean.com/v2/droplets/*' => function () use (&$polls) { + $polls++; + + if ($polls > 50) { + throw new Error('isRunning() was polled '.$polls.' times: the wait loop is unbounded.'); + } + + return Http::response([ + 'droplet' => [ + 'id' => 599103218, + 'status' => 'new', + 'networks' => ['v4' => []], + ], + ]); + }, + '*' => Http::response([]), + ]); + + $serverProvider = ServerProvider::factory()->create([ + 'user_id' => $this->user->id, + 'provider' => DigitalOcean::id(), + 'credentials' => ['token' => 'secret-token'], + ]); + + $this->server->update([ + 'provider' => DigitalOcean::id(), + 'provider_id' => $serverProvider->id, + 'provider_data' => ['plan' => 's-1vcpu-512mb-10gb', 'region' => 'nyc1', 'droplet_id' => 599103218], + 'ip' => '', + 'status' => ServerStatus::INSTALLING, + ]); + + expect(fn () => app(InstallServer::class)->run($this->server->refresh())) + ->toThrow(SSHConnectionError::class, 'The server did not become reachable within 180 seconds.'); + + expect($polls)->toBe(18); + + Sleep::assertSleptTimes(18); + Sleep::assertSlept(fn (Duration $duration) => (int) $duration->totalSeconds === 10, 18); + + $this->server->refresh(); + + expect($this->server->status)->toEqual(ServerStatus::INSTALLING); +}); + +test('the underlying ssh failure is preserved when the server never becomes reachable', function () { + SSH::fake()->connectionWillFail(); + + Http::fake([ + 'api.digitalocean.com/v2/droplets/*' => Http::response([ + 'droplet' => [ + 'id' => 599103218, + 'status' => 'active', + 'networks' => ['v4' => [['type' => 'public', 'ip_address' => '164.92.1.1']]], + ], + ]), + '*' => Http::response([]), + ]); + + $serverProvider = ServerProvider::factory()->create([ + 'user_id' => $this->user->id, + 'provider' => DigitalOcean::id(), + 'credentials' => ['token' => 'secret-token'], + ]); + + $this->server->update([ + 'provider' => DigitalOcean::id(), + 'provider_id' => $serverProvider->id, + 'provider_data' => ['plan' => 's-1vcpu-512mb-10gb', 'region' => 'nyc1', 'droplet_id' => 599103218], + 'ip' => '', + 'status' => ServerStatus::INSTALLING, + ]); + + try { + app(InstallServer::class)->run($this->server->refresh()); + + $this->fail('Expected InstallServer to throw.'); + } catch (SSHConnectionError $e) { + expect($e->getMessage())->toBe('The server did not become reachable within 180 seconds.') + ->and($e->getPrevious())->toBeInstanceOf(SSHConnectionError::class) + ->and($e->getPrevious()?->getMessage())->toBe('Connection failed'); + } +});