perf(db): index a workspace's live files in upload order (#9434)

Adds workspace_files_workspace_active_uploaded_idx (workspace_id, uploaded_at, id) on the shared live workspace-file predicate, built concurrently with replay-safe recovery, so the file list and its bounded SSR seed read a workspace's rows in upload order instead of matching every workspace's files and sorting. listWorkspaceFiles orders by (uploaded_at, id) for a stable order. Adds @sim/db/testing/explain for plan assertions on the SQL a call actually sends.

Author: waleedlatif1

apps/sim/lib/uploads/contexts/workspace/__integration__/file-list.integration.ts

@@ -0,0 +1,137 @@
+/**
+ * Real PostgreSQL reads of a workspace's whole file list: which rows each scope returns, their
+ * order, and the plan the live list's SQL gets.
+ */
+import { db } from '@sim/db'
+import { user, workspace, workspaceFiles } from '@sim/db/schema'
+import { explainCaptured } from '@sim/db/testing/explain'
+import { deleteWorkspaceFixture, insertWorkspaceFixture } from '@sim/db/testing/workspace-fixtures'
+import { uploadsSetupMock } from '@sim/testing/mocks/uploads-setup.mock'
+import { generateId } from '@sim/utils/id'
+import { eq, inArray } from 'drizzle-orm'
+import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'
+
+vi.mock('@/lib/uploads/core/setup.server', () => uploadsSetupMock)
+
+import { listWorkspaceFiles } from '@/lib/uploads/contexts/workspace/file-manager'
+import { WORKSPACE_FILE_SEED_MAX } from '@/app/workspace/[workspaceId]/lib/seed-workspace-files'
+
+const ownerId = generateId()
+const workspaceId = generateId()
+const otherWorkspaceId = generateId()
+const tick = (seconds: number) => new Date(Date.UTC(2026, 0, 1, 0, 0, seconds))
+
+type FileRow = typeof workspaceFiles.$inferInsert
+
+function fileRow(name: string, values: Partial<FileRow> = {}): FileRow {
+  const id = `${name}-${workspaceId}`
+  return {
+    id,
+    key: `workspace/${workspaceId}/${id}`,
+    userId: ownerId,
+    workspaceId,
+    context: 'workspace',
+    originalName: `${name}.txt`,
+    contentType: 'text/plain',
+    sizeBytes: 1,
+    ...values,
+  }
+}
+
+describe('workspace file list in PostgreSQL', () => {
+  /**
+   * Rows inserted, and deleted, out of id order, with uploaded_at ties inside each scope and
+   * across them, so an order that is not (uploaded_at, id) shows up as a reordered list.
+   */
+  const live = [
+    fileRow('live-d', { uploadedAt: tick(1) }),
+    fileRow('live-c', { uploadedAt: tick(2) }),
+    fileRow('live-b', { uploadedAt: tick(2) }),
+    fileRow('live-a', { uploadedAt: tick(2) }),
+    fileRow('live-e', { uploadedAt: tick(4) }),
+  ]
+  const archived = [
+    fileRow('gone-b', { uploadedAt: tick(2), deletedAt: tick(10) }),
+    fileRow('gone-c', { uploadedAt: tick(3), deletedAt: tick(11) }),
+    fileRow('gone-a', { uploadedAt: tick(3), deletedAt: tick(12) }),
+  ]
+  const neverListed = [
+    fileRow('chat-upload', { context: 'mothership', uploadedAt: tick(2) }),
+    fileRow('execution-output', { context: 'execution', uploadedAt: tick(2) }),
+    fileRow('other-workspace', {
+      workspaceId: otherWorkspaceId,
+      key: `workspace/${otherWorkspaceId}/other`,
+      uploadedAt: tick(2),
+    }),
+  ]
+  const byUploadThenId = (rows: FileRow[]) =>
+    [...rows]
+      .sort(
+        (a, b) =>
+          (a.uploadedAt?.getTime() ?? 0) - (b.uploadedAt?.getTime() ?? 0) ||
+          (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)
+      )
+      .map((row) => row.id)
+
+  beforeAll(async () => {
+    await db.insert(user).values({
+      id: ownerId,
+      email: `${ownerId}@file-list.invalid`,
+      name: 'File list fixture',
+      emailVerified: true,
+      createdAt: new Date(),
+      updatedAt: new Date(),
+    })
+    await insertWorkspaceFixture(
+      db,
+      [workspaceId, otherWorkspaceId].map((id) => ({
+        id,
+        ownerId,
+        billedAccountUserId: ownerId,
+        name: 'File list fixture',
+      }))
+    )
+    await db.insert(workspaceFiles).values([...live, ...archived, ...neverListed])
+  })
+
+  afterAll(async () => {
+    try {
+      await db.delete(workspaceFiles).where(eq(workspaceFiles.userId, ownerId))
+      await deleteWorkspaceFixture(db, inArray(workspace.id, [workspaceId, otherWorkspaceId]))
+      await db.delete(user).where(eq(user.id, ownerId))
+    } finally {
+      await db.$client.end()
+    }
+  })
+
+  it.each([
+    { scope: 'active', label: 'live files', expected: byUploadThenId(live) },
+    { scope: 'archived', label: 'Recently Deleted files', expected: byUploadThenId(archived) },
+    {
+      scope: 'all',
+      label: 'live and deleted files together',
+      expected: byUploadThenId([...live, ...archived]),
+    },
+  ] as const)('lists $label by upload time, ties by id', async ({ scope, expected }) => {
+    const files = await listWorkspaceFiles(workspaceId, { scope, throwOnError: true })
+    expect(files.map((file) => file.id)).toEqual(expected)
+  })
+
+  /**
+   * With every sorting plan ruled out, an ordered plan exists only if an index's predicate is
+   * implied by the list's filter and its key gives the list's order.
+   */
+  it('reads the live list, whole or bounded, in order from its workspace index', async () => {
+    for (const limit of [undefined, WORKSPACE_FILE_SEED_MAX + 1]) {
+      const nodes = await explainCaptured(
+        db,
+        () => listWorkspaceFiles(workspaceId, { limit, throwOnError: true }),
+        { match: /from "workspace_files"/, gucs: { enable_sort: 'off' } }
+      )
+      expect(nodes.map((node) => node['Node Type'])).not.toContain('Sort')
+      expect(nodes.map((node) => node['Index Name'])).toContain(
+        'workspace_files_workspace_active_uploaded_idx'
+      )
+    }
+  })
+})

apps/sim/lib/uploads/contexts/workspace/file-manager.ts

@@ -1794,7 +1794,8 @@ async function hydrateWorkspaceFilePaths(
 }
 
 /**
- * List all files for a workspace
+ * List all files for a workspace, oldest upload first; `id` orders files uploaded at the same
+ * instant, so the order is stable across reads.
  */
 export async function listWorkspaceFiles(
   workspaceId: string,
@@ -1806,7 +1807,7 @@ export async function listWorkspaceFiles(
       .select(workspaceFileListColumns)
       .from(workspaceFiles)
       .where(workspaceFileScopeCondition(workspaceId, scope))
-      .orderBy(workspaceFiles.uploadedAt)
+      .orderBy(workspaceFiles.uploadedAt, workspaceFiles.id)
     const files = await (limit === undefined ? query : query.limit(limit))
 
     return await hydrateWorkspaceFilePaths(files, workspaceId, options)

packages/db/migrations/0442_workspace_files_active_uploaded_idx.sql

@@ -0,0 +1,35 @@
+-- A workspace's live file list, in upload order. No index served the list's filter in that
+-- order, so the planner matched every workspace's live files, filtered to one workspace on the
+-- heap and sorted them, on disk for a large workspace; a bounded read (the SSR seed) paid for
+-- the whole workspace.
+--
+-- The predicate is the one the other live workspace-file indexes share, which the list's filter
+-- implies. The list's owner conditions (project_id and organization_id IS NULL) stay heap checks
+-- that discard nothing: the project and organization binding CHECKs rule both out for
+-- workspace-context rows. The key stays narrow: carrying the listed name and storage key columns
+-- would bring an entry near the btree size limit and fail the write.
+--
+-- Replay-safety: workspace_files takes uploads and edits continuously, so the build is
+-- CONCURRENTLY, below an embedded COMMIT, and a failure there replays the whole file from the
+-- top. A failed concurrent build leaves an INVALID index that IF NOT EXISTS would keep: the replay
+-- moves only that one aside to be dropped, so a valid index built by an earlier attempt is kept.
+DO $$
+BEGIN
+  IF EXISTS (
+    SELECT 1 FROM pg_index WHERE indexrelid = to_regclass('workspace_files_workspace_active_uploaded_idx_failed')
+      AND (indisvalid OR indrelid <> to_regclass('workspace_files'))
+  ) THEN
+    RAISE EXCEPTION 'Refusing to drop unexpected recovery index workspace_files_workspace_active_uploaded_idx_failed';
+  END IF;
+  IF EXISTS (
+    SELECT 1 FROM pg_index WHERE indexrelid = to_regclass('workspace_files_workspace_active_uploaded_idx')
+      AND indrelid = to_regclass('workspace_files') AND NOT indisvalid
+  ) THEN
+    ALTER INDEX "workspace_files_workspace_active_uploaded_idx" RENAME TO "workspace_files_workspace_active_uploaded_idx_failed";
+  END IF;
+END $$;--> statement-breakpoint
+COMMIT;--> statement-breakpoint
+SET lock_timeout = 0;--> statement-breakpoint
+DROP INDEX CONCURRENTLY IF EXISTS "workspace_files_workspace_active_uploaded_idx_failed";--> statement-breakpoint
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "workspace_files_workspace_active_uploaded_idx" ON "workspace_files" USING btree ("workspace_id","uploaded_at","id") WHERE "workspace_files"."deleted_at" IS NULL AND "workspace_files"."context" = 'workspace' AND "workspace_files"."workspace_id" IS NOT NULL;--> statement-breakpoint
+SET lock_timeout = '5s';