test(mothership): cover a job dispatch retried after a usage outage and ended at its last attempt

Author: waleedlatif1

apps/sim/lib/mothership/jobs/lane.integration.ts

@@ -272,6 +272,13 @@ async function runsOf(scheduleId: string) {
     .where(eq(jobExecutionLogs.scheduleId, scheduleId))
 }
 
+/** The usage gate's answer when the payer's usage can't be read. */
+const usageOutage = {
+  isExceeded: true,
+  message: 'Usage could not be checked.',
+  reason: 'usage_unavailable' as const,
+}
+
 /** Makes a schedule due again now, as its next occurrence arriving would. */
 async function dueAgain(scheduleId: string) {
   const dueAt = new Date(Date.now() - 1_000)
@@ -647,26 +654,48 @@ describe.runIf(Boolean(redisUrl))('the scheduled job lane', () => {
     await reconcileUntilSettled(occurrenceId)
   })
 
-  it('retries an occurrence whose usage could not be read, and skips one whose billing is blocked', async () => {
+  it('retries an occurrence whose usage could not be read, and runs it once usage reads again', async () => {
     setEnvFlags({ isHosted: true })
     const { scheduleId, occurrenceId } = await seedJob()
-    vi.mocked(checkAttributedUsageLimits).mockResolvedValueOnce({
-      isExceeded: true,
-      message: 'Usage could not be checked.',
-      reason: 'usage_unavailable',
-    })
+    vi.mocked(checkAttributedUsageLimits).mockResolvedValueOnce(usageOutage)
     await runJobLane(new Date())
     expect(await runsOf(scheduleId)).toEqual([
       expect.objectContaining({ status: 'queued', outcome: null, dispatchFailures: 1 }),
     ])
 
+    vi.mocked(checkAttributedUsageLimits).mockResolvedValueOnce({ isExceeded: false })
+    await reconcileJobRuns(new Date(Date.now() + JOB_DISPATCH_GRACE_MS + 1_000))
+    await reconcileUntilSettled(occurrenceId)
+    expect(legsFor(occurrenceId)).toHaveLength(1)
+    expect(await runsOf(scheduleId)).toEqual([
+      expect.objectContaining({ status: 'complete', outcome: 'not_met', dispatchFailures: 1 }),
+    ])
+  })
+
+  it('ends an occurrence whose usage stays unreadable at its last attempt, and skips one whose billing is blocked', async () => {
+    setEnvFlags({ isHosted: true })
+    const outage = await seedJob()
+    await claimDueJobOccurrences(new Date(), 10)
+    for (let attempt = 1; attempt <= 5; attempt++) {
+      vi.mocked(checkAttributedUsageLimits).mockResolvedValueOnce(usageOutage)
+      expect(await dispatchJobOccurrence(outage.occurrenceId)).toBe(
+        attempt < 5 ? 'retry' : 'failed'
+      )
+    }
+    expect(await runsOf(outage.scheduleId)).toEqual([
+      expect.objectContaining({ outcome: 'failed', summary: 'The run could not be started.' }),
+    ])
+    expect(legsFor(outage.occurrenceId)).toHaveLength(0)
+
+    const blocked = await seedJob()
+    await claimDueJobOccurrences(new Date(), 10)
     vi.mocked(checkAttributedUsageLimits).mockResolvedValueOnce({
       isExceeded: true,
       message: 'Billing is blocked for this account.',
       reason: 'billing_blocked',
     })
-    expect(await dispatchJobOccurrence(occurrenceId)).toBe('skipped_budget')
-    expect(await runsOf(scheduleId)).toEqual([
+    expect(await dispatchJobOccurrence(blocked.occurrenceId)).toBe('skipped_budget')
+    expect(await runsOf(blocked.scheduleId)).toEqual([
       expect.objectContaining({
         outcome: 'skipped_budget',
         summary: 'Billing is blocked for this account.',