Skip to content

Commit e53bf39

Browse files
authored
improvement(search): page Slack Real-time Search to fill a Slack-scoped search's limit (#9590)
* improvement(search): page Slack Real-time Search to fill a Slack-scoped search's limit * fix(search): keep earlier Slack pages on any later-page failure and page past short pages
1 parent 2b77ade commit e53bf39

3 files changed

Lines changed: 248 additions & 3 deletions

File tree

‎apps/sim/lib/sim-search/live/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ Each event must exist under the source's delegated token, be in an allowed calen
5858

5959
### Slack
6060

61-
Member mode only. Search uses the connected user's Slack real-time search grant and provider search syntax. Date bounds are sent as numeric provider timestamps; date ordering selects keyword retrieval and timestamp sort, because Slack semantic retrieval uses relevance ordering. Channels, messages, direct messages, and files are limited by that grant and provider scopes; there is no admin channel filter in member mode. Missing required search scopes produce a reconnect status. Thread/file reads use the same connected identity.
61+
Member mode only. Search uses the connected user's Slack real-time search grant and provider search syntax. Date bounds are sent as numeric provider timestamps; date ordering selects keyword retrieval and timestamp sort, because Slack semantic retrieval uses relevance ordering. Channels, messages, direct messages, and files are limited by that grant and provider scopes; there is no admin channel filter in member mode. Missing required search scopes produce a reconnect status. Thread/file reads use the same connected identity. Real-time Search returns at most 20 results a call and counts every page against the member's rate limit, so a search across every source reads one page; a Slack-scoped or native Slack search follows the cursor for up to three pages to fill its limit, and a later page that fails keeps the earlier pages and returns the unread cursor.
6262

6363
### GitHub
6464

‎apps/sim/lib/sim-search/live/providers.test.ts‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,3 +188,199 @@ describe('native search endpoints', () => {
188188
expect(api.json).not.toHaveBeenCalled()
189189
})
190190
})
191+
192+
describe('Slack RTS pages', () => {
193+
const scopes = ['search:read.public']
194+
const slackPage = (first: number, count: number, nextCursor?: string) => ({
195+
ok: true,
196+
results: {
197+
messages: Array.from({ length: count }, (_, index) => ({
198+
message_ts: `${first + index}.000100`,
199+
channel_id: 'C1',
200+
channel_name: 'team',
201+
content: 'Launch',
202+
permalink: `https://team.slack.com/archives/C1/p${first + index}000100`,
203+
})),
204+
},
205+
response_metadata: { next_cursor: nextCursor ?? '' },
206+
})
207+
const sentBodies = (api: ReturnType<typeof client>) =>
208+
api.json.mock.calls.map(([, options]) => options?.body as Record<string, unknown>)
209+
210+
it('follows the cursor until a native Slack query fills its limit', async () => {
211+
const api = client()
212+
api.json
213+
.mockResolvedValueOnce(slackPage(1, 20, 'page-2'))
214+
.mockResolvedValueOnce(slackPage(21, 20, 'page-3'))
215+
const result = await searchSlack(api, {
216+
...input,
217+
limit: 30,
218+
scopes,
219+
native: { provider: 'slack', query: 'launch' },
220+
})
221+
expect(sentBodies(api).map(({ cursor, limit }) => ({ cursor, limit }))).toEqual([
222+
{ cursor: undefined, limit: 20 },
223+
{ cursor: 'page-2', limit: 10 },
224+
])
225+
expect(result.documents).toHaveLength(40)
226+
expect(result.nextCursor).toBe('page-3')
227+
expect(result.partial).toBeUndefined()
228+
})
229+
230+
it('pages a plain query scoped to the Slack source', async () => {
231+
const api = client()
232+
api.json
233+
.mockResolvedValueOnce(slackPage(1, 20, 'page-2'))
234+
.mockResolvedValueOnce(slackPage(21, 5))
235+
const result = await searchSlack(api, {
236+
...input,
237+
limit: 50,
238+
scopes,
239+
filters: { source: 'slack' },
240+
})
241+
expect(api.json).toHaveBeenCalledTimes(2)
242+
expect(result.documents).toHaveLength(25)
243+
expect(result.nextCursor).toBeUndefined()
244+
})
245+
246+
it('reads one page for a search across every source, whatever its limit', async () => {
247+
const api = client()
248+
api.json.mockResolvedValue(slackPage(1, 20, 'page-2'))
249+
const result = await searchSlack(api, { ...input, limit: 30, scopes })
250+
expect(api.json).toHaveBeenCalledTimes(1)
251+
expect(result.nextCursor).toBe('page-2')
252+
})
253+
254+
it('stops when Slack reports no further page', async () => {
255+
const api = client()
256+
api.json.mockResolvedValue(slackPage(1, 12))
257+
await searchSlack(api, {
258+
...input,
259+
limit: 50,
260+
scopes,
261+
native: { provider: 'slack', query: 'launch' },
262+
})
263+
expect(api.json).toHaveBeenCalledTimes(1)
264+
})
265+
266+
it('reads at most three pages', async () => {
267+
const api = client()
268+
api.json.mockImplementation(async (_path, options) => {
269+
const cursor = (options?.body as { cursor?: string }).cursor
270+
const index = cursor ? Number(cursor) : 0
271+
return slackPage(index * 20 + 1, 1, String(index + 1))
272+
})
273+
const result = await searchSlack(api, {
274+
...input,
275+
limit: 100,
276+
scopes,
277+
native: { provider: 'slack', query: 'launch' },
278+
})
279+
expect(api.json).toHaveBeenCalledTimes(3)
280+
expect(result.nextCursor).toBe('3')
281+
})
282+
283+
it('stops once matched files fill the limit alongside messages', async () => {
284+
const api = client()
285+
const withFiles = slackPage(1, 20, 'page-2')
286+
api.json.mockResolvedValue({
287+
...withFiles,
288+
results: {
289+
...withFiles.results,
290+
files: Array.from({ length: 10 }, (_, index) => ({
291+
file_id: `F${index}`,
292+
title: 'Launch plan',
293+
permalink: `https://team.slack.com/files/F${index}`,
294+
})),
295+
},
296+
})
297+
const result = await searchSlack(api, {
298+
...input,
299+
limit: 30,
300+
scopes: [...scopes, 'search:read.files'],
301+
native: { provider: 'slack', query: 'launch' },
302+
})
303+
expect(api.json).toHaveBeenCalledTimes(1)
304+
expect(result.documents).toHaveLength(30)
305+
})
306+
307+
it('keeps earlier pages and the unread cursor when a later page fails', async () => {
308+
const api = client()
309+
api.json
310+
.mockResolvedValueOnce(slackPage(1, 20, 'page-2'))
311+
.mockResolvedValueOnce({ ok: false, error: 'ratelimited' })
312+
const result = await searchSlack(api, {
313+
...input,
314+
limit: 50,
315+
scopes,
316+
native: { provider: 'slack', query: 'launch' },
317+
})
318+
expect(result.documents).toHaveLength(20)
319+
expect(result.nextCursor).toBe('page-2')
320+
expect(result.partial).toBe(true)
321+
expect(result.message).toContain('nextCursor')
322+
})
323+
324+
it('keeps earlier pages when a later page fails outside the Slack API', async () => {
325+
const api = client()
326+
api.json
327+
.mockResolvedValueOnce(slackPage(1, 20, 'page-2'))
328+
.mockRejectedValueOnce(new Error('socket hang up'))
329+
const result = await searchSlack(api, {
330+
...input,
331+
limit: 50,
332+
scopes,
333+
native: { provider: 'slack', query: 'launch' },
334+
})
335+
expect(result.documents).toHaveLength(20)
336+
expect(result.nextCursor).toBe('page-2')
337+
expect(result.partial).toBe(true)
338+
})
339+
340+
it('stops paging once the caller cancels the search', async () => {
341+
const api = client()
342+
const cancelled = new AbortController()
343+
api.json.mockResolvedValueOnce(slackPage(1, 20, 'page-2')).mockImplementationOnce(async () => {
344+
cancelled.abort()
345+
throw new Error('aborted')
346+
})
347+
await expect(
348+
searchSlack(api, {
349+
...input,
350+
limit: 50,
351+
scopes,
352+
signal: cancelled.signal,
353+
native: { provider: 'slack', query: 'launch' },
354+
})
355+
).rejects.toThrow('aborted')
356+
})
357+
358+
it('keeps paging past short pages that still carry a cursor', async () => {
359+
const api = client()
360+
api.json
361+
.mockResolvedValueOnce(slackPage(1, 10, 'page-2'))
362+
.mockResolvedValueOnce(slackPage(11, 10, 'page-3'))
363+
.mockResolvedValueOnce(slackPage(21, 10))
364+
const result = await searchSlack(api, {
365+
...input,
366+
limit: 30,
367+
scopes,
368+
native: { provider: 'slack', query: 'launch' },
369+
})
370+
expect(api.json).toHaveBeenCalledTimes(3)
371+
expect(result.documents).toHaveLength(30)
372+
})
373+
374+
it('still fails when the first page fails', async () => {
375+
const api = client()
376+
api.json.mockResolvedValue({ ok: false, error: 'ratelimited' })
377+
await expect(
378+
searchSlack(api, {
379+
...input,
380+
limit: 50,
381+
scopes,
382+
native: { provider: 'slack', query: 'launch' },
383+
})
384+
).rejects.toMatchObject({ status: 'rate_limited' })
385+
})
386+
})

‎apps/sim/lib/sim-search/live/slack.ts‎

Lines changed: 51 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { dateSortDirection, nativeDateBounds, nativeText } from '@/lib/sim-search/live/dates'
22
import { array, NativeSearchError, object, string } from '@/lib/sim-search/live/http'
3+
import { joinMessages } from '@/lib/sim-search/live/pages'
34
import {
45
slackConversationName,
56
slackConversationUrl,
@@ -35,6 +36,20 @@ function slackResult(value: unknown) {
3536
return data
3637
}
3738

39+
/** Results one Real-time Search call returns at most. */
40+
const SLACK_PAGE_SIZE = 20
41+
/**
42+
* Real-time Search pages a search aimed at Slack reads to fill its limit. Each page counts
43+
* against the member's RTS rate limit of about 10 calls a minute.
44+
*/
45+
const MAX_SLACK_SEARCH_PAGES = 3
46+
47+
/**
48+
* A search across every source reads one page, so Slack shares the result list with the other
49+
* sources. A Slack-scoped or native Slack search follows the cursor until it fills its limit.
50+
* A later page that fails keeps the earlier pages and returns the cursor it could not read,
51+
* unless the caller cancelled the search.
52+
*/
3853
export async function searchSlack(
3954
client: NativeClient,
4055
input: NativeSearchInput
@@ -44,6 +59,40 @@ export async function searchSlack(
4459
'reconnect',
4560
'Reconnect your Slack account to grant permission to search.'
4661
)
62+
const pages = input.native || input.filters?.source === 'slack' ? MAX_SLACK_SEARCH_PAGES : 1
63+
const first = await searchSlackPage(client, input, input.native?.cursor, input.limit)
64+
const documents = [...first.documents]
65+
let nextCursor = first.nextCursor
66+
for (let page = 1; page < pages && nextCursor && documents.length < input.limit; page++) {
67+
try {
68+
const more = await searchSlackPage(client, input, nextCursor, input.limit - documents.length)
69+
documents.push(...more.documents)
70+
nextCursor = more.nextCursor
71+
} catch (error) {
72+
if (input.signal?.aborted) throw error
73+
return {
74+
documents,
75+
nextCursor,
76+
partial: true,
77+
message: joinMessages([
78+
first.message,
79+
error instanceof NativeSearchError
80+
? error.message
81+
: 'Slack could not return more results.',
82+
'Continue with nextCursor for more.',
83+
]),
84+
}
85+
}
86+
}
87+
return { documents, nextCursor, message: first.message }
88+
}
89+
90+
async function searchSlackPage(
91+
client: NativeClient,
92+
input: NativeSearchInput,
93+
cursor: string | undefined,
94+
limit: number
95+
): Promise<NativePage> {
4796
const channels = [
4897
['public_channel', 'search:read.public'],
4998
['private_channel', 'search:read.private'],
@@ -80,13 +129,13 @@ export async function searchSlack(
80129
? ['messages', 'files']
81130
: ['messages'],
82131
include_context_messages: true,
83-
limit: Math.min(input.limit, 20),
132+
limit: Math.min(limit, SLACK_PAGE_SIZE),
84133
...(dates.start ? { after: Math.floor(Date.parse(dates.start) / 1000) - 1 } : {}),
85134
...(dates.end ? { before: Math.ceil(Date.parse(dates.end) / 1000) } : {}),
86135
...(dateSortDirection(input.filters)
87136
? { sort: 'timestamp', sort_dir: dateSortDirection(input.filters) }
88137
: {}),
89-
...(input.native?.cursor ? { cursor: input.native.cursor } : {}),
138+
...(cursor ? { cursor } : {}),
90139
...(input.native?.termClauses ? { term_clauses: input.native.termClauses } : {}),
91140
...(modifiers ? { modifiers } : {}),
92141
...(input.native?.keywordOnly || !nativeText(input)

0 commit comments

Comments
 (0)