Skip to content

Commit fffa150

Browse files
waleedlatif1waleedlatif1
authored
feat(files): page, search, look up and roll up workspace files over the session API (#9435)
Session endpoints for bounded workspace-file pages (folder, sort, cursor), name search, batched lookup and folder rollups, behind one shared page implementation also used by v2 and the mothership VFS; the unpaged list response is unchanged. Folder pages seek the folder-name index via coalesce(folder_id,''); timestamp keysets order on the plain column with microsecond cursors (legacy millisecond cursors still resume). References that aren't valid percent-encoding are read as bare names. Migration 0443 adds a folder upload-order index and a trigram name index (fastupdate off), built concurrently.
1 parent 19b8d53 commit fffa150

23 files changed

Lines changed: 32998 additions & 142 deletions

File tree

‎apps/sim/app/api/workspaces/[id]/files/folder-stats/route.test.ts‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
import { authMockFns } from '@sim/testing'
2+
import { createRouteContext } from '@sim/testing/helpers/http'
3+
import { createMockRequest } from '@sim/testing/mocks/request.mock'
4+
import {
5+
workspaceFilesListMock,
6+
workspaceFilesListMockFns,
7+
} from '@sim/testing/mocks/workspace-files-list.mock'
8+
import { beforeEach, describe, expect, it, vi } from 'vitest'
9+
10+
vi.mock('@/lib/workspace-files/application/list-workspace-files', () => workspaceFilesListMock)
11+
12+
import { GET } from '@/app/api/workspaces/[id]/files/folder-stats/route'
13+
14+
const WORKSPACE_ID = 'workspace-1'
15+
const stats = workspaceFilesListMockFns.mockGetWorkspaceFileFolderStats
16+
17+
describe('GET /api/workspaces/[id]/files/folder-stats', () => {
18+
beforeEach(() => {
19+
authMockFns.mockGetSession.mockResolvedValue({
20+
user: { id: 'user-1', name: 'Test User', email: 'test@sim.ai' },
21+
session: { id: 'session-1' },
22+
})
23+
stats.mockResolvedValue({ folders: [] })
24+
})
25+
26+
it.each([
27+
['omitted', ''],
28+
['root', '?parentId=root'],
29+
])('reads the root children when parentId is %s', async (_label, query) => {
30+
const response = await GET(
31+
createMockRequest({
32+
method: 'GET',
33+
url: `/api/workspaces/${WORKSPACE_ID}/files/folder-stats${query}`,
34+
}),
35+
createRouteContext({ id: WORKSPACE_ID })
36+
)
37+
38+
expect(response.status).toBe(200)
39+
expect(stats.mock.calls[0][0].input).toEqual({ workspaceId: WORKSPACE_ID, parentId: null })
40+
})
41+
})

‎apps/sim/app/api/workspaces/[id]/files/folder-stats/route.ts‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
import { workspaceFileFolderStatsContract } from '@/lib/api/contracts/workspace-files'
2+
import {
3+
defineInternalJsonRoute,
4+
internalRateLimits,
5+
internalSessionAuth,
6+
} from '@/lib/api/server/routes'
7+
import { internalFileErrorPolicies } from '@/lib/workspace-files/api'
8+
import { getWorkspaceFileFolderStats } from '@/lib/workspace-files/application/list-workspace-files'
9+
import { fileOperations } from '@/lib/workspace-files/application/operations'
10+
11+
export const dynamic = 'force-dynamic'
12+
13+
/**
14+
* GET /api/workspaces/[id]/files/folder-stats — live file count and bytes under each child
15+
* folder of `parentId` (default the root), each folder's subtree rolled up.
16+
*/
17+
export const GET = defineInternalJsonRoute({
18+
contract: workspaceFileFolderStatsContract,
19+
auth: internalSessionAuth,
20+
operation: fileOperations.list,
21+
rateLimit: internalRateLimits.user({ bucketName: 'workspace-files.read' }),
22+
errorPolicy: internalFileErrorPolicies.default,
23+
mapInput: ({ params, query }) => ({ workspaceId: params.id, parentId: query.parentId }),
24+
useCase: getWorkspaceFileFolderStats,
25+
present: ({ folders }) => ({ success: true as const, folders }),
26+
})

‎apps/sim/app/api/workspaces/[id]/files/lookup/route.ts‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
import { lookupWorkspaceFilesContract } from '@/lib/api/contracts/workspace-files'
2+
import {
3+
defineInternalJsonRoute,
4+
internalRateLimits,
5+
internalSessionAuth,
6+
} from '@/lib/api/server/routes'
7+
import { internalFileErrorPolicies, internalFilePresenters } from '@/lib/workspace-files/api'
8+
import { lookupWorkspaceFiles } from '@/lib/workspace-files/application/list-workspace-files'
9+
import { fileOperations } from '@/lib/workspace-files/application/operations'
10+
11+
export const dynamic = 'force-dynamic'
12+
13+
/**
14+
* POST /api/workspaces/[id]/files/lookup — resolve a bounded batch of live workspace files by
15+
* id, storage key, or reference without listing the workspace.
16+
*/
17+
export const POST = defineInternalJsonRoute({
18+
contract: lookupWorkspaceFilesContract,
19+
auth: internalSessionAuth,
20+
operation: fileOperations.list,
21+
rateLimit: internalRateLimits.user({ bucketName: 'workspace-files.read' }),
22+
errorPolicy: internalFileErrorPolicies.default,
23+
mapInput: ({ params, body }) => ({
24+
workspaceId: params.id,
25+
ids: body.ids,
26+
keys: body.keys,
27+
references: body.references,
28+
}),
29+
useCase: lookupWorkspaceFiles,
30+
present: ({ files, references }) => ({
31+
...internalFilePresenters.successFiles({ files }),
32+
references,
33+
}),
34+
})

‎apps/sim/app/api/workspaces/[id]/files/route.test.ts‎

Lines changed: 43 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ vi.mock('@/lib/workspace-files/application/list-workspace-files', () => workspac
2828
vi.mock('@/lib/posthog/server', () => posthogServerMock)
2929

3030
import { OrchestrationError } from '@/lib/core/orchestration/types'
31-
import { POST } from '@/app/api/workspaces/[id]/files/route'
31+
import { GET, POST } from '@/app/api/workspaces/[id]/files/route'
3232

3333
const WORKSPACE_ID = 'workspace-1'
3434
const USER = { id: 'user-1', name: 'Test User', email: 'test@sim.ai' }
@@ -62,7 +62,6 @@ describe('/api/workspaces/[id]/files', () => {
6262
authMockFns.mockGetSession.mockResolvedValue({ user: USER, session: { id: 'session-1' } })
6363
mocks.admitCreate.mockResolvedValue(undefined)
6464
mocks.createFile.mockResolvedValue({ file: FILE })
65-
workspaceFilesListMockFns.mockListAllWorkspaceFiles.mockResolvedValue({ files: [FILE] })
6665
})
6766

6867
it('authorizes the asserted workspace before buffering the create body', async () => {
@@ -87,4 +86,46 @@ describe('/api/workspaces/[id]/files', () => {
8786
expect(mocks.admitCreate).toHaveBeenCalled()
8887
expect(mocks.createFile).not.toHaveBeenCalled()
8988
})
89+
90+
describe('GET', () => {
91+
const list = (query: string) =>
92+
GET(
93+
createMockRequest({ method: 'GET', url: `/api/workspaces/${WORKSPACE_ID}/files${query}` }),
94+
context
95+
)
96+
const collection = workspaceFilesListMockFns.mockListWorkspaceFileCollection
97+
98+
it('keeps the unpaged response, without a cursor, when no paging param is sent', async () => {
99+
collection.mockResolvedValueOnce({ files: [FILE], nextKeys: null })
100+
101+
const response = await list('?scope=active')
102+
103+
expect(response.status).toBe(200)
104+
const body = await response.json()
105+
expect(body).toEqual({ success: true, files: [expect.objectContaining({ id: FILE.id })] })
106+
expect(collection.mock.calls[0][0].input).toEqual({
107+
workspaceId: WORKSPACE_ID,
108+
scope: 'active',
109+
})
110+
})
111+
112+
it('resumes from its own cursor and refuses it under another folder', async () => {
113+
collection.mockResolvedValue({ files: [FILE], nextKeys: ['notes.md', FILE.id] })
114+
115+
const first = await (await list('?folderId=root&sortBy=name&limit=1')).json()
116+
expect(collection.mock.calls[0][0].input.page).toMatchObject({
117+
folderId: null,
118+
after: undefined,
119+
})
120+
expect(first.nextCursor).toEqual(expect.any(String))
121+
122+
const resumed = await list(`?folderId=root&sortBy=name&limit=1&cursor=${first.nextCursor}`)
123+
expect(resumed.status).toBe(200)
124+
expect(collection.mock.calls[1][0].input.page.after).toEqual(['notes.md', FILE.id])
125+
126+
const moved = await list(`?folderId=folder-2&sortBy=name&limit=1&cursor=${first.nextCursor}`)
127+
expect(moved.status).toBe(400)
128+
expect(collection).toHaveBeenCalledTimes(2)
129+
})
130+
})
90131
})

‎apps/sim/app/api/workspaces/[id]/files/route.ts‎

Lines changed: 59 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
import {
22
createWorkspaceFileContract,
3+
DEFAULT_WORKSPACE_FILE_PAGE_SIZE,
34
listWorkspaceFilesContract,
5+
WORKSPACE_FILE_PAGE_PARAMS,
46
} from '@/lib/api/contracts/workspace-files'
7+
import { cursorRoute, cursorScopeKey } from '@/lib/api/cursor-binding'
58
import {
69
defineInternalJsonRoute,
710
internalRateLimits,
@@ -17,22 +20,73 @@ import {
1720
admitCreateWorkspaceFile,
1821
createWorkspaceFile,
1922
} from '@/lib/workspace-files/application/create-workspace-file'
20-
import { listAllWorkspaceFiles } from '@/lib/workspace-files/application/list-workspace-files'
23+
import { listWorkspaceFileCollection } from '@/lib/workspace-files/application/list-workspace-files'
2124
import { fileOperations } from '@/lib/workspace-files/application/operations'
2225
import { MAX_WORKSPACE_FILE_INLINE_BODY_BYTES } from '@/lib/workspace-files/orchestration'
26+
import { readSortedCursor, writeSortedCursor } from '@/app/api/v2/lib/response'
2327

2428
export const dynamic = 'force-dynamic'
2529

26-
/** GET /api/workspaces/[id]/files — List workspace files. */
30+
/** The params besides the sort that change which files a page walk returns; the cursor is bound to them. */
31+
function fileCursorFilters(
32+
workspaceId: string,
33+
query: { scope: 'active' | 'archived'; folderId?: string | null }
34+
) {
35+
return cursorScopeKey(cursorRoute(listWorkspaceFilesContract, { id: workspaceId }), {
36+
scope: query.scope,
37+
folderId: query.folderId,
38+
})
39+
}
40+
41+
/**
42+
* GET /api/workspaces/[id]/files — List workspace files: one bounded page with `nextCursor`
43+
* when any paging param is sent, else the whole scope.
44+
*/
2745
export const GET = defineInternalJsonRoute({
2846
contract: listWorkspaceFilesContract,
2947
auth: internalSessionAuth,
3048
operation: fileOperations.list,
49+
// One policy per route: paged requests share the unpaged list's exemption until the builder
50+
// can choose a policy per request.
3151
rateLimit: internalRateLimits.none({ reason: 'Preserve existing internal file-list behavior' }),
3252
errorPolicy: internalFileErrorPolicies.default,
33-
mapInput: ({ params, query }) => ({ workspaceId: params.id, scope: query.scope }),
34-
useCase: listAllWorkspaceFiles,
35-
present: internalFilePresenters.successFiles,
53+
mapInput: ({ params, query }) => {
54+
if (!WORKSPACE_FILE_PAGE_PARAMS.some((param) => query[param] !== undefined)) {
55+
return { workspaceId: params.id, scope: query.scope }
56+
}
57+
const sortBy = query.sortBy ?? 'uploadedAt'
58+
const sortOrder = query.sortOrder ?? 'asc'
59+
return {
60+
workspaceId: params.id,
61+
scope: query.scope,
62+
page: {
63+
folderId: query.folderId,
64+
sortBy,
65+
sortOrder,
66+
limit: query.limit ?? DEFAULT_WORKSPACE_FILE_PAGE_SIZE,
67+
after: readSortedCursor(
68+
query.cursor,
69+
sortBy,
70+
sortOrder,
71+
fileCursorFilters(params.id, query)
72+
),
73+
},
74+
}
75+
},
76+
useCase: listWorkspaceFileCollection,
77+
present: (result, { input }) => {
78+
const listed = internalFilePresenters.successFiles(result)
79+
if (!input.page) return listed
80+
return {
81+
...listed,
82+
nextCursor: writeSortedCursor(
83+
result.nextKeys,
84+
input.page.sortBy,
85+
input.page.sortOrder,
86+
fileCursorFilters(input.workspaceId, { scope: input.scope, folderId: input.page.folderId })
87+
),
88+
}
89+
},
3690
})
3791

3892
/** POST /api/workspaces/[id]/files — Create an authored workspace file. */

‎apps/sim/app/api/workspaces/[id]/files/search/route.ts‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
import { searchWorkspaceFilesContract } from '@/lib/api/contracts/workspace-files'
2+
import {
3+
defineInternalJsonRoute,
4+
internalRateLimits,
5+
internalSessionAuth,
6+
} from '@/lib/api/server/routes'
7+
import { internalFileErrorPolicies, internalFilePresenters } from '@/lib/workspace-files/api'
8+
import { listWorkspaceFileCollection } from '@/lib/workspace-files/application/list-workspace-files'
9+
import { fileOperations } from '@/lib/workspace-files/application/operations'
10+
11+
export const dynamic = 'force-dynamic'
12+
13+
/** Sized for debounced type-ahead: a burst while typing, about two searches a second sustained. */
14+
const SEARCH_TYPE_AHEAD_LIMIT = { maxTokens: 120, refillRate: 120, refillIntervalMs: 60_000 }
15+
16+
/**
17+
* GET /api/workspaces/[id]/files/search — live files whose name contains `q`, by name, each
18+
* carrying its `folderPath`; within `folderId` (and, by default, its subfolders) when given.
19+
*/
20+
export const GET = defineInternalJsonRoute({
21+
contract: searchWorkspaceFilesContract,
22+
auth: internalSessionAuth,
23+
operation: fileOperations.list,
24+
rateLimit: internalRateLimits.user({
25+
bucketName: 'workspace-files.search',
26+
config: SEARCH_TYPE_AHEAD_LIMIT,
27+
}),
28+
errorPolicy: internalFileErrorPolicies.default,
29+
mapInput: ({ params, query }) => ({
30+
workspaceId: params.id,
31+
scope: 'active' as const,
32+
page: {
33+
search: query.q,
34+
folderId: query.folderId,
35+
recursive: query.recursive,
36+
sortBy: 'name' as const,
37+
sortOrder: 'asc' as const,
38+
limit: query.limit,
39+
},
40+
}),
41+
useCase: listWorkspaceFileCollection,
42+
present: internalFilePresenters.successFiles,
43+
})

0 commit comments

Comments
 (0)