improvement(mothership): let a code call run for up to fifteen minutes (#9588)

* improvement(mothership): let a code call run for up to fifteen minutes

A render or build that needs more than five minutes was cut off at the
ceiling and failed, and the model started over in smaller pieces or
escaped the call with a background process the paused workbench then
stalled. Mothership code calls now run up to 900 seconds. The worker's
schema still decides what a model may request, so this is inert until
the worker raises its own maximum.

* fix(mothership): renew a running code call's sandbox CLI scope past its expiry

* fix(mothership): keep a long code call's upload bindings for its whole lifetime

Author: waleedlatif1

apps/sim/lib/mothership/tools/handlers/function-execute-session.test.ts

@@ -146,7 +146,7 @@ describe('executeFunctionExecute session plumbing', () => {
     expect(mockExecuteTool.mock.calls[1][1].timeout).toBe(90_000)
 
     await executeFunctionExecute({ code: 'x', language: 'python', timeout: 45_000 }, BASE_CONTEXT)
-    expect(mockExecuteTool.mock.calls[2][1].timeout).toBe(300_000)
+    expect(mockExecuteTool.mock.calls[2][1].timeout).toBe(900_000)
   })
 })
 
@@ -187,8 +187,9 @@ describe.each([
     { timeout: 7, expected: 7_000 },
     { timeout: '90', expected: 90_000 },
     { timeout: 300, expected: 300_000 },
-    { timeout: 601, expected: 300_000 },
-    { timeout: 45_000, expected: 300_000 },
+    { timeout: 900, expected: 900_000 },
+    { timeout: 901, expected: 900_000 },
+    { timeout: 45_000, expected: 900_000 },
     { timeout: 0, expected: 60_000 },
     { timeout: 'invalid', expected: 60_000 },
   ])('maps $timeout seconds to $expected milliseconds once', async ({ timeout, expected }) => {

apps/sim/lib/mothership/tools/handlers/function-execute.ts

@@ -76,7 +76,11 @@ const logger = createLogger('CopilotFunctionExecute')
 const MAX_MOUNTED_FILES = 500
 /** The call budget includes cold allocation and mounting, before the user's code starts. */
 const DEFAULT_TIMEOUT_SECONDS = 60
-const MAX_TIMEOUT_SECONDS = 300
+/**
+ * The longest call the worker may request. Long renders and builds need it whole: cut off at
+ * five minutes, they fail and start over. The worker's schema decides what a model may ask for.
+ */
+const MAX_TIMEOUT_SECONDS = 900
 
 /** An input the run asked for that was not mounted, and why; paths are the model's own. */
 export interface ExcludedInput {

apps/sim/lib/mothership/tools/sandbox-project-files.ts

@@ -17,6 +17,7 @@ import {
 } from '@/lib/mothership/tools/sandbox-callback'
 import {
   recordSandboxResourceEffects,
+  SANDBOX_SCOPE_STATE_TTL_SECONDS,
   type SandboxResourceScope,
 } from '@/lib/mothership/tools/sandbox-resources'
 import { chatSandboxSessionKey } from '@/lib/mothership/tools/sandbox-session-key'
@@ -66,10 +67,11 @@ export async function proxySandboxProjectFileRequest(
     uploadBinding: {
       async record(uploadId) {
         await redis().eval(
-          "redis.call('SADD', KEYS[1], ARGV[1]); redis.call('EXPIRE', KEYS[1], 600); return 1",
+          "redis.call('SADD', KEYS[1], ARGV[1]); redis.call('EXPIRE', KEYS[1], ARGV[2]); return 1",
           1,
           bindingKey,
-          uploadId
+          uploadId,
+          SANDBOX_SCOPE_STATE_TTL_SECONDS
         )
       },
       async contains(uploadId) {

apps/sim/lib/mothership/tools/sandbox-resources.test.ts

@@ -11,19 +11,33 @@ vi.mock('@/lib/core/config/feature-flags', () => featureFlagsMock)
 
 const { values, lists, seen, persist, redis } = vi.hoisted(() => {
   const values = new Map<string, string>()
+  const expiresAt = new Map<string, number>()
   const lists = new Map<string, string[]>()
   const seen = new Map<string, Set<string>>()
+  const live = (key: string) => {
+    if ((expiresAt.get(key) ?? Number.POSITIVE_INFINITY) <= Date.now()) {
+      values.delete(key)
+      expiresAt.delete(key)
+    }
+    return values.has(key)
+  }
   return {
     values,
     lists,
     seen,
     persist: vi.fn(async () => {}),
     redis: {
-      set: vi.fn(async (key: string, value: string) => {
+      set: vi.fn(async (key: string, value: string, _mode?: 'EX', ttlSeconds?: number) => {
         values.set(key, value)
+        if (ttlSeconds) expiresAt.set(key, Date.now() + ttlSeconds * 1000)
+      }),
+      expire: vi.fn(async (key: string, ttlSeconds: number) => {
+        if (!live(key)) return 0
+        expiresAt.set(key, Date.now() + ttlSeconds * 1000)
+        return 1
       }),
-      get: vi.fn(async (key: string) => values.get(key) ?? null),
-      exists: vi.fn(async (key: string) => (values.has(key) ? 1 : 0)),
+      get: vi.fn(async (key: string) => (live(key) ? (values.get(key) ?? null) : null)),
+      exists: vi.fn(async (key: string) => (live(key) ? 1 : 0)),
       del: vi.fn(async (key: string) => {
         values.delete(key)
         lists.delete(key)
@@ -183,6 +197,36 @@ describe('sandbox resource ownership and publication', () => {
     })
   })
 
+  it('keeps the callback scope live for a call that outlasts its expiry, and revokes it after', async () => {
+    vi.useFakeTimers()
+    try {
+      let release = () => {}
+      const finished = new Promise<void>((resolve) => {
+        release = resolve
+      })
+      let token = ''
+      const call = withSandboxResourceScope(
+        identity,
+        new AbortController().signal,
+        undefined,
+        async () => {
+          token =
+            (await sandboxResourceEndpoint('https://sim.test', args, 'secret')).split('/').at(-1) ??
+            ''
+          await finished
+        }
+      )
+      await vi.advanceTimersByTimeAsync(15 * 60_000)
+      expect(await readSandboxResourceScope(token, 'secret')).not.toBeNull()
+      release()
+      await vi.advanceTimersByTimeAsync(1_000)
+      await call
+      expect(await readSandboxResourceScope(token, 'secret')).toBeNull()
+    } finally {
+      vi.useRealTimers()
+    }
+  })
+
   it('revokes on cancellation and never emits an abandoned callback', async () => {
     const controller = new AbortController()
     const emit = vi.fn()

apps/sim/lib/mothership/tools/sandbox-resources.ts

@@ -16,6 +16,13 @@ import { isProjectsEnabled } from '@/lib/projects/rollout.server'
 
 const logger = createLogger('MothershipSandboxResources')
 const CONTEXT_TTL_SECONDS = 600
+/** A running call renews its scope well inside the TTL; the TTL only bounds a crashed owner. */
+const CONTEXT_RENEW_MS = (CONTEXT_TTL_SECONDS * 1000) / 3
+/**
+ * Outlives the longest code call (15 minutes plus its settle margin). State keyed by a scope token
+ * is reachable only through that token's live context, so outliving the context widens nothing.
+ */
+export const SANDBOX_SCOPE_STATE_TTL_SECONDS = 20 * 60
 const POLL_MS = 200
 const scopeSchema = z
   .object({
@@ -233,10 +240,23 @@ export async function withSandboxResourceScope<T>(
     () => {},
     () => {}
   )
+  const renew = async () => {
+    if (!active.token || signal.aborted) return
+    try {
+      await redisClient().expire(contextKey(active.token), CONTEXT_TTL_SECONDS)
+    } catch {
+      logger.warn('Sandbox resource scope renewal deferred', { toolCallId: identity.toolCallId })
+    }
+  }
+  let renewedAt = Date.now()
   try {
     while (running) {
       await Promise.race([settled, sleep(POLL_MS)])
       await drain()
+      if (Date.now() - renewedAt >= CONTEXT_RENEW_MS) {
+        renewedAt = Date.now()
+        await renew()
+      }
     }
     return await execution
   } finally {