fix(file-search): stop requeueing skipped artifacts, and fix public share, live-doc persist and cleanup edge cases (#9110)

* fix(file-search): keep a skipped build published so rendered reads stop requeueing it

A skipped publication retired its build and cleared the revision's build id,
so a compiled document whose index build skipped never matched its rendered
artifact key: every rendered read requeued it, and the worker downloaded,
compiled and skipped it again. Skipped builds now stay published with their
bound artifact key (and no text), and the artifact-current checks accept a
skipped revision. Failed revisions still requeue.

* fix(public-shares): answer a shared Project page with a foreign embed as not found

renderSimPageDocument throws ProjectPageForeignEmbedError for a Project page
that embeds another owner's file. The authenticated artifact read maps it to
not_found, but the public content and inline reads let it escape as a 500.
Map it the same way at both public render sites.

* fix(file-doc): let the workspace document persist take a full snapshot

The relay's workspace persist ran behind the proxy with the default 10 MB body
cap, so a collaborative state above ~7.5 MiB (12 MiB is allowed) was refused
with 413 while the Project persist route took it. Exclude the workspace
persist from the proxy matcher and give it the same bypassesProxy body cap,
now shared from collab-doc state, and the same off-app-host refusal.

* chore(knip): drop the stale ignore for the v2 Project file search contract

Its only export is now imported by the v2 route and the OpenAPI spec.

* fix(memory): let a rejected key spend memory cleanup attempts

A 401 from the worker waited hourly without consuming an attempt, like a
missing route, so a permanently revoked self-hosted key kept its cleanup rows
pending forever. A 401 still waits an hour but now spends an attempt, so it
dead-letters after the default attempts (about ten hours), past a worker
rollout. 404, 405 and 501 keep waiting for free.

* fix(public-shares): log expected public file share client errors as warnings

The public file share routes logged every caught error at ERROR, including
classified client failures such as a missing share (404) or a password
exchange against a non-password share (400). Log a classified 4xx as a
warning and keep ERROR for unclassified failures.

* fix(public-shares): import the error logger in the OTP route

* test(file-search): skip on a corrupt compiled document the parsers reject

Author: waleedlatif1

apps/sim/app/api/files/public/[token]/content/route.ts

@@ -5,6 +5,7 @@ import { parseRequest } from '@/lib/api/server'
 import { getClientIp } from '@/lib/core/utils/request'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
 import {
+  logPublicFileError,
   publicFileAuthDenied,
   publicFileBinaryErrorResponse,
   publicFileShareCredential,
@@ -58,7 +59,7 @@ export const GET = withRouteHandler(
         request.headers.get('if-none-match')
       )
     } catch (error) {
-      logger.error('Error serving public shared file:', error)
+      logPublicFileError(logger, 'Error serving public shared file:', error)
       return publicFileBinaryErrorResponse(error, 'Failed to serve file')
     }
   }

apps/sim/app/api/files/public/[token]/inline/route.ts

@@ -5,6 +5,7 @@ import { parseRequest } from '@/lib/api/server'
 import { getClientIp } from '@/lib/core/utils/request'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
 import {
+  logPublicFileError,
   publicFileAuthDenied,
   publicFileBinaryErrorResponse,
   publicFileShareCredential,
@@ -48,7 +49,7 @@ export const GET = withRouteHandler(
         request.headers.get('if-none-match')
       )
     } catch (error) {
-      logger.error('Error serving public inline image:', error)
+      logPublicFileError(logger, 'Error serving public inline image:', error)
       return publicFileBinaryErrorResponse(error, 'Failed to serve file')
     }
   }

apps/sim/app/api/files/public/[token]/otp/route.ts

@@ -11,7 +11,11 @@ import { OTP_IP_RATE_LIMIT } from '@/lib/core/security/otp'
 import { afterResponse } from '@/lib/core/utils/after-response'
 import { getClientIp } from '@/lib/core/utils/request'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
-import { publicFileAuthDenied, publicFileErrorResponse } from '@/lib/public-shares/api'
+import {
+  logPublicFileError,
+  publicFileAuthDenied,
+  publicFileErrorResponse,
+} from '@/lib/public-shares/api'
 import {
   requestPublicFileShareOtp,
   verifyPublicFileShareOtp,
@@ -48,7 +52,7 @@ export const POST = withRouteHandler(
       afterResponse(accepted.deliver)
       return NextResponse.json({ message: 'Verification code sent' })
     } catch (error) {
-      logger.error('Error processing OTP request:', error)
+      logPublicFileError(logger, 'Error processing OTP request:', error)
       return publicFileErrorResponse(error, 'Failed to process request')
     }
   }
@@ -69,7 +73,7 @@ export const PUT = withRouteHandler(
       response.cookies.set(auth.cookie)
       return response
     } catch (error) {
-      logger.error('Error verifying OTP:', error)
+      logPublicFileError(logger, 'Error verifying OTP:', error)
       return publicFileErrorResponse(error, 'Failed to process request')
     }
   }

apps/sim/app/api/files/public/[token]/route.ts

@@ -9,6 +9,7 @@ import { parseRequest } from '@/lib/api/server'
 import { getClientIp } from '@/lib/core/utils/request'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
 import {
+  logPublicFileError,
   publicFileAuthDenied,
   publicFileErrorResponse,
   publicFileShareCredential,
@@ -48,7 +49,7 @@ export const GET = withRouteHandler(
         ownerName,
       })
     } catch (error) {
-      logger.error('Error fetching public file metadata:', error)
+      logPublicFileError(logger, 'Error fetching public file metadata:', error)
       return publicFileErrorResponse(error, 'Failed to fetch file')
     }
   }
@@ -70,7 +71,7 @@ export const POST = withRouteHandler(
       response.cookies.set(auth.cookie)
       return response
     } catch (error) {
-      logger.error('Error authenticating public file share:', error)
+      logPublicFileError(logger, 'Error authenticating public file share:', error)
       return publicFileErrorResponse(error, 'Failed to authenticate')
     }
   }

apps/sim/app/api/files/public/[token]/sso/route.ts

@@ -6,7 +6,11 @@ import { parseRequest } from '@/lib/api/server'
 import { RateLimiter, type TokenBucketConfig } from '@/lib/core/rate-limiter'
 import { getClientIp } from '@/lib/core/utils/request'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
-import { publicFileAuthDenied, publicFileErrorResponse } from '@/lib/public-shares/api'
+import {
+  logPublicFileError,
+  publicFileAuthDenied,
+  publicFileErrorResponse,
+} from '@/lib/public-shares/api'
 import { getPublicFileShareSsoEligibility } from '@/lib/public-shares/application'
 
 export const dynamic = 'force-dynamic'
@@ -47,7 +51,7 @@ export const POST = withRouteHandler(
         ? NextResponse.json({ eligible: result.eligible })
         : publicFileAuthDenied(result)
     } catch (error) {
-      logger.error('Error checking public file SSO eligibility:', error)
+      logPublicFileError(logger, 'Error checking public file SSO eligibility:', error)
       return publicFileErrorResponse(error, 'Failed to process request')
     }
   }

apps/sim/app/api/internal/file-doc/persist/route.ts

@@ -3,8 +3,10 @@ import { getErrorMessage } from '@sim/utils/errors'
 import type { NextRequest } from 'next/server'
 import { NextResponse } from 'next/server'
 import { persistFileDocContract } from '@/lib/api/contracts/file-doc'
+import { isOffAppHost } from '@/lib/api/mcp/host-routing'
 import { parseRequest } from '@/lib/api/server'
 import { persistFileDoc } from '@/lib/collab-doc/persist'
+import { MAX_COLLAB_DOC_PERSIST_BODY_BYTES } from '@/lib/collab-doc/state'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
 import { checkInternalApiKey, createUnauthorizedResponse } from '@/lib/mothership/request/http'
 
@@ -16,12 +18,19 @@ const logger = createLogger('FileDocPersistAPI')
  * the shared `x-api-key: INTERNAL_API_SECRET` secret, matching the header the realtime relay sends.
  * The relay owns the live doc but not the conversion engine or blob/DB access, so it ships the current
  * doc state here — the server-authoritative durable path that replaces the editor's client autosave.
+ * Excluded from the proxy matcher, so the route refuses the dedicated MCP host before auth itself.
  */
 export const POST = withRouteHandler(async (request: NextRequest) => {
+  if (isOffAppHost(request)) return new NextResponse(null, { status: 404 })
   const auth = checkInternalApiKey(request)
   if (!auth.success) return createUnauthorizedResponse()
 
-  const parsed = await parseRequest(persistFileDocContract, request, {})
+  const parsed = await parseRequest(
+    persistFileDocContract,
+    request,
+    {},
+    { maxBodyBytes: MAX_COLLAB_DOC_PERSIST_BODY_BYTES, bypassesProxy: true }
+  )
   if (!parsed.success) return parsed.response
   const { workspaceId, fileId, userId, docState, expectedVersion } = parsed.data.body
 

apps/sim/app/api/internal/project-file-doc/[projectId]/[fileId]/persist/route.ts

@@ -7,12 +7,9 @@ import {
   internalRateLimits,
 } from '@/lib/api/server/routes'
 import { realtimeProjectFileAuth } from '@/lib/auth/realtime-file-delegation'
-import { MAX_COLLAB_DOC_STATE_BYTES } from '@/lib/collab-doc/state'
+import { MAX_COLLAB_DOC_PERSIST_BODY_BYTES } from '@/lib/collab-doc/state'
 import { persistProjectFileDoc } from '@/lib/projects/files/application/documents'
 
-/** A base64-encoded maximum snapshot plus 1 MiB for its JSON envelope (17 MiB). */
-const PERSIST_MAX_BODY_BYTES = Math.ceil(MAX_COLLAB_DOC_STATE_BYTES / 3) * 4 + 1024 * 1024
-
 const persist = defineInternalJsonRoute({
   contract: projectFileDocPersistContract,
   operation: persistProjectFileDoc.operation,
@@ -27,7 +24,7 @@ const persist = defineInternalJsonRoute({
     expectedVersion: body.expectedVersion,
   }),
   useCase: persistProjectFileDoc,
-  parseOptions: { maxBodyBytes: PERSIST_MAX_BODY_BYTES, bypassesProxy: true },
+  parseOptions: { maxBodyBytes: MAX_COLLAB_DOC_PERSIST_BODY_BYTES, bypassesProxy: true },
 })
 
 /** Excluded from the proxy matcher, so the route refuses the dedicated MCP host before auth itself. */

apps/sim/lib/collab-doc/state.ts

@@ -7,6 +7,13 @@ import type { DbOrTx, DbTransaction } from '@/lib/db/types'
 /** Matches the decoded size of the persist endpoint's 16 MiB base64 snapshot limit. */
 export const MAX_COLLAB_DOC_STATE_BYTES = 12 * 1024 * 1024
 
+/**
+ * Body cap of the relay's persist routes: a base64-encoded maximum snapshot plus 1 MiB for its JSON
+ * envelope (17 MiB). Both routes bypass the proxy, whose 10 MB buffer would truncate it.
+ */
+export const MAX_COLLAB_DOC_PERSIST_BODY_BYTES =
+  Math.ceil(MAX_COLLAB_DOC_STATE_BYTES / 3) * 4 + 1024 * 1024
+
 /** Reject oversized snapshots before copying, decoding, or writing them. */
 export function assertCollabDocStateSize(docState: Uint8Array): void {
   if (docState.byteLength > MAX_COLLAB_DOC_STATE_BYTES) {

apps/sim/lib/mothership/memories/cleanup-outbox.ts

@@ -29,9 +29,15 @@ const logger = createLogger('MemoryCleanup')
 const UNAVAILABLE_RETRY_MS = 60 * 60 * 1000
 /**
  * Statuses meaning the worker does not serve cleanup yet, not that this request failed: a worker
- * that predates the route answers 404 to the hosted key and 401 to a self-hosted customer key.
+ * that predates the route answers 404 to the hosted key.
  */
-const ENDPOINT_MISSING_STATUSES = new Set([401, 404, 405, 501])
+const ENDPOINT_MISSING_STATUSES = new Set([404, 405, 501])
+/**
+ * A worker that predates the route also answers this to a self-hosted customer key, but so does a
+ * worker that revoked the key for good: it waits like a missing route but spends an attempt, so a
+ * revoked key dead-letters after the default attempts (about ten hours), past a worker rollout.
+ */
+const UNAUTHORIZED_STATUS = 401
 
 /** Non-strict so rows enqueued with the retired `credentialFingerprint` field still parse. */
 const cleanupPayloadSchema = z.object({
@@ -74,16 +80,21 @@ export async function enqueueMemoryCleanup(
   }
 }
 
-function waitForWorker(reason: string, eventId: string, detail?: Record<string, string>) {
+function waitForWorker(
+  reason: string,
+  eventId: string,
+  { consumeAttempt = false, ...detail }: { baseUrl?: string; consumeAttempt?: boolean } = {}
+) {
   logger.warn(`Memory cleanup waiting: ${reason}`, { eventId, ...detail })
-  return deferOutboxHandler(reason, UNAVAILABLE_RETRY_MS, false)
+  return deferOutboxHandler(reason, UNAVAILABLE_RETRY_MS, consumeAttempt)
 }
 
 /**
  * Sends the processor's current key only to a destination the processor itself configures, and
  * replays the enqueuing deployment's source env so the worker erases that deployment's notes.
  * A missing key, destination, or worker endpoint waits without consuming attempts: erasure must
- * not dead-letter while configuration or the worker rollout catches up.
+ * not dead-letter while configuration or the worker rollout catches up. A rejected key waits too,
+ * but within the retry budget.
  */
 export const memoryCleanupOutboxHandlers = {
   [MEMORY_CLEANUP_EVENT]: async (rawPayload, context) => {
@@ -109,6 +120,10 @@ export const memoryCleanupOutboxHandlers = {
       await response.body?.cancel().catch(() => {})
       if (ENDPOINT_MISSING_STATUSES.has(response.status))
         return waitForWorker(`endpoint unavailable (${response.status})`, context.eventId)
+      if (response.status === UNAUTHORIZED_STATUS)
+        return waitForWorker('worker rejected the key (401)', context.eventId, {
+          consumeAttempt: true,
+        })
       throw new Error(`Memory cleanup failed (${response.status})`)
     }
     MemoryCleanupResult.parse(await response.json())

apps/sim/lib/mothership/memories/cleanup.integration.ts

@@ -139,10 +139,10 @@ it('rolls cleanup back with deletion and retries committed account/target erasur
   expect(received.every((request) => request.key === 'memory-cleanup-fixture-key')).toBe(true)
 
   const first = events[0]
-  const replay = async (payload: unknown = first.payload) => {
+  const replay = async (payload: unknown = first.payload, attempts = 9) => {
     await db
       .update(outboxEvent)
-      .set({ status: 'pending', availableAt: new Date(0), attempts: 9, payload })
+      .set({ status: 'pending', availableAt: new Date(0), attempts, payload })
       .where(eq(outboxEvent.id, first.id))
     await runtime.outbox.processOutboxEventById(
       first.id,
@@ -161,11 +161,18 @@ it('rolls cleanup back with deletion and retries committed account/target erasur
   waitsAnHour(await replay())
   expect(received).toHaveLength(before + 1)
 
+  status = 401
+  const rejected = await replay(first.payload, 5)
+  expect(rejected).toMatchObject({ status: 'pending', attempts: 6 })
+  expect(rejected.availableAt.getTime()).toBeGreaterThan(Date.now() + 50 * 60_000)
+  expect(await replay()).toMatchObject({ status: 'dead_letter', attempts: 10 })
+  expect(received).toHaveLength(before + 3)
+
   status = 200
   waitsAnHour(await replay({ ...toRecord(first.payload), baseUrl: 'https://unconfigured.test' }))
-  expect(received).toHaveLength(before + 1)
+  expect(received).toHaveLength(before + 3)
 
   const legacy = await replay({ ...toRecord(first.payload), credentialFingerprint: 'retired' })
   expect(legacy.status).toBe('completed')
-  expect(received).toHaveLength(before + 2)
+  expect(received).toHaveLength(before + 4)
 }, 30_000)