fix(execution): re-queue work selected onto a retiring worker; stop E2E hosts with their workers

- dispatchToWorker re-queues when the selected worker retired during a target
  switch (not only when it was removed), and cleans it up if idle.
- E2E hosts run in their own process group so cleanup also stops their
  worker children.
- Sandbox rule notes the URL needs its token.

Author: waleedlatif1

.claude/rules/sim-sandbox.md

@@ -88,8 +88,10 @@ payload or `ivm.Reference` wrapper in the worker:
 ## Where the worker runs
 
 The pool in `isolated-vm.ts` opens each worker either as a local child process
-or, when `IVM_REMOTE_WORKER_URL` is set (and the `isolated-vm-local-workers`
-kill switch is off), as a WebSocket connection to `isolated-vm-host.cjs`, which runs the
+or, when `IVM_REMOTE_WORKER_URL` is set (with `IVM_REMOTE_WORKER_TOKEN`; a URL
+without a valid token fails executions rather than falling back) and the
+`isolated-vm-local-workers` kill switch is off, as a WebSocket connection to
+`isolated-vm-host.cjs`, which runs the
 same worker in a dedicated service that holds no platform credentials. Every
 rule in this file applies in both modes.
 

.cursor/rules/sim-sandbox.mdc

@@ -84,8 +84,10 @@ payload or `ivm.Reference` wrapper in the worker:
 ## Where the worker runs
 
 The pool in `isolated-vm.ts` opens each worker either as a local child process
-or, when `IVM_REMOTE_WORKER_URL` is set (and the `isolated-vm-local-workers`
-kill switch is off), as a WebSocket connection to `isolated-vm-host.cjs`, which runs the
+or, when `IVM_REMOTE_WORKER_URL` is set (with `IVM_REMOTE_WORKER_TOKEN`; a URL
+without a valid token fails executions rather than falling back) and the
+`isolated-vm-local-workers` kill switch is off, as a WebSocket connection to
+`isolated-vm-host.cjs`, which runs the
 same worker in a dedicated service that holds no platform credentials. Every
 rule in this file applies in both modes.
 

apps/sim/lib/execution/isolated-vm.ts

@@ -1155,8 +1155,7 @@ function spawnWorker(): Promise<WorkerInfo> {
               return
             }
             workerInfo.ready = true
-            // The target changed while this worker was starting: it serves at most the
-            // execution awaiting it, then retires.
+            // The target changed while this worker was starting, so it takes no work.
             workerInfo.retiring = workerInfo.targetKey !== workerTargetKey(workerTarget)
             clearTimeout(startTimeout)
             proc.off('message', readyHandler)
@@ -1265,8 +1264,10 @@ function dispatchToWorker(
     return
   }
 
-  // A target change can retire and remove the worker between its selection and this dispatch.
-  if (workers.get(workerInfo.id) !== workerInfo) {
+  // A target change can retire (or retire and remove) the worker between its selection and this
+  // dispatch; the execution then waits for a worker on the current target.
+  if (workers.get(workerInfo.id) !== workerInfo || workerInfo.retiring) {
+    if (workerInfo.retiring && workerInfo.activeExecutions === 0) cleanupWorker(workerInfo.id)
     enqueueExecution(ownerState, req, resolve, state, brokers)
     return
   }

apps/sim/scripts/test-isolated-vm-remote-e2e.ts

@@ -90,6 +90,8 @@ async function startHost(maxWorkers: number): Promise<Host> {
       SIM_HOST_ONLY_SETTING: HOST_ONLY_VALUE,
     },
     stdio: ['ignore', 'pipe', 'pipe'],
+    // Its own process group, so cleanup can stop the host and its worker children together.
+    detached: true,
   })
   const relay = (data: Buffer) => logger.info(`host:${port} ${data.toString().trim()}`)
   child.stdout?.on('data', relay)
@@ -410,7 +412,11 @@ run()
     process.exitCode = 1
   })
   .finally(async () => {
-    for (const host of hosts) if (host.process.exitCode === null) host.process.kill('SIGKILL')
+    for (const host of hosts) {
+      try {
+        if (host.process.pid) process.kill(-host.process.pid, 'SIGKILL')
+      } catch {}
+    }
     const report = {
       suite: 'isolated-vm-remote',
       startedAt,