fix: add ELO calculation retry and dedicated server state recovery

Author: Flegma

src/dedicated-servers/dedicated-servers.service.ts

@@ -345,11 +345,17 @@ export class DedicatedServersService {
             await this.pingDedicatedServer(serverId);
           }, 10000);
         })
-        .catch((error) => {
+        .catch(async (error) => {
           this.logger.error(
-            `[${serverId}] error waiting for pod to be ready`,
+            `[${serverId}] error waiting for pod to be ready, recovering server state`,
             error,
           );
+          await this.removeDedicatedServer(serverId).catch((removeError) => {
+            this.logger.error(
+              `[${serverId}] failed to clean up server after pod readiness failure`,
+              removeError,
+            );
+          });
         });
 
       return true;

src/matches/jobs/EloCalculation.ts

@@ -1,23 +1,34 @@
 import { Job } from "bullmq";
 import { WorkerHost } from "@nestjs/bullmq";
+import { Logger } from "@nestjs/common";
 import { MatchQueues } from "../enums/MatchQueues";
 import { UseQueue } from "../../utilities/QueueProcessors";
 import { PostgresService } from "../../postgres/postgres.service";
 
 @UseQueue("Matches", MatchQueues.EloCalculation)
 export class EloCalculation extends WorkerHost {
+  private readonly logger = new Logger(EloCalculation.name);
+
   constructor(private readonly postgres: PostgresService) {
     super();
   }
 
   async process(job: Job): Promise<void> {
     const { matchId } = job.data;
 
-    await this.postgres.query(
-      `
-      SELECT generate_player_elo_for_match($1)
-    `,
-      [matchId],
-    );
+    try {
+      await this.postgres.query(
+        `
+        SELECT generate_player_elo_for_match($1)
+      `,
+        [matchId],
+      );
+    } catch (error) {
+      this.logger.error(
+        `ELO calculation failed for match ${matchId} (attempt ${job.attemptsMade + 1})`,
+        error,
+      );
+      throw error;
+    }
   }
 }

src/matches/matches.module.ts

@@ -75,6 +75,10 @@ import { DiscordTournamentVoiceModule } from "../discord-bot/discord-tournament-
       },
       {
         name: MatchQueues.EloCalculation,
+        defaultJobOptions: {
+          attempts: 3,
+          backoff: { type: "exponential", delay: 5000 },
+        },
       },
     ),
     BullBoardModule.forFeature(