Skip to content

Commit f67d024

Browse files
authored
improvement(search): fall back to Jira summaries holding half the terms of a longer query (#9115)
* improvement(search): fall back to summaries holding half the terms of a longer query * fix(search): count Jira half-term summaries by stemmed term, from a wider pool, most terms first * improvement(search): share the half-terms threshold and convert Jira descriptions only when filtered
1 parent 5eb0b33 commit f67d024

3 files changed

Lines changed: 193 additions & 68 deletions

File tree

‎apps/sim/lib/sim-search/live/atlassian.test.ts‎

Lines changed: 88 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -334,16 +334,32 @@ describe('Confluence plain search ranking', () => {
334334

335335
describe('Jira plain search ranking', () => {
336336
const search = '/ex/jira/cloud/rest/api/3/search/jql'
337-
/** Answers the all-terms query with `exact` and the summary query with `titled`, or its failure. */
338-
function jira(exact: { ids: string[]; more: boolean }, titled: string[] | (() => never)) {
339-
const issues = (ids: string[]) => ids.map((key) => ({ key, fields: { summary: key } }))
340-
const json = vi.fn(async (path: string, options?: { body?: { jql?: string } }) => {
341-
if (path === '/oauth/token/accessible-resources') return [SITE]
342-
if (path !== search) throw new Error(`Unexpected request: ${path}`)
343-
if (options?.body?.jql?.includes('summary ~'))
344-
return { issues: issues(typeof titled === 'function' ? titled() : titled) }
345-
return { issues: issues(exact.ids), ...(exact.more ? { nextPageToken: 'page-2' } : {}) }
346-
})
337+
type Issue = string | { key: string; summary: string }
338+
/**
339+
* Answers the all-terms query with `exact`, the all-terms summary query with `titled`, and the
340+
* half-the-terms summary query with `half`, or their failures.
341+
*/
342+
function jira(
343+
exact: { ids: string[]; more: boolean },
344+
titled: Issue[] | (() => never),
345+
half: Issue[] | (() => never) = []
346+
) {
347+
const toRow = (row: Issue) => (typeof row === 'string' ? { key: row, summary: row } : row)
348+
const issues = (rows: Issue[]) =>
349+
rows.map(toRow).map(({ key, summary }) => ({ key, fields: { summary } }))
350+
const json = vi.fn(
351+
async (path: string, options?: { body?: { jql?: string; maxResults?: number } }) => {
352+
if (path === '/oauth/token/accessible-resources') return [SITE]
353+
if (path !== search) throw new Error(`Unexpected request: ${path}`)
354+
const jql = options?.body?.jql ?? ''
355+
if (jql.includes('summary ~')) {
356+
const rows = jql.includes(' OR ') ? half : titled
357+
const picked = typeof rows === 'function' ? rows() : rows
358+
return { issues: issues(picked).slice(0, options?.body?.maxResults) }
359+
}
360+
return { issues: issues(exact.ids), ...(exact.more ? { nextPageToken: 'page-2' } : {}) }
361+
}
362+
)
347363
return { json, text: vi.fn(), bytes: vi.fn() } satisfies NativeClient
348364
}
349365
const query = { query: 'merged results pipelines', limit: 3, scopes: [] }
@@ -411,4 +427,66 @@ describe('Jira plain search ranking', () => {
411427
expect(api.json).toHaveBeenCalledTimes(2)
412428
}
413429
})
430+
431+
it('leads with issues whose summary holds at least half the terms when none holds them all', async () => {
432+
const api = jira(
433+
overflowing,
434+
[],
435+
[
436+
{ key: 'LOOSE-1', summary: 'Pipelines dashboard' },
437+
{ key: 'HALF-1', summary: 'Merged results broke the dashboard' },
438+
]
439+
)
440+
const page = await searchAtlassian(api, 'jira', query)
441+
expect(page.documents.map((document) => document.id)).toEqual(['HALF-1', 'NEW-3', 'NEW-2'])
442+
expect(page.nextCursor).toBeUndefined()
443+
})
444+
445+
it('keeps a two-term query to summaries holding every term', async () => {
446+
const api = jira(overflowing, [], [{ key: 'HALF-1', summary: 'Merged pipelines' }])
447+
const page = await searchAtlassian(api, 'jira', { ...query, query: 'merged pipelines' })
448+
expect(page.documents.map((document) => document.id)).toEqual(['NEW-3', 'NEW-2', 'NEW-1'])
449+
expect(api.json).toHaveBeenCalledTimes(3)
450+
})
451+
452+
it('keeps the all-terms page and its cursor when the half-the-terms summary search fails', async () => {
453+
const api = jira(overflowing, [], () => {
454+
throw new NativeSearchError('rate_limited', 'Rate limited.', 30)
455+
})
456+
const page = await searchAtlassian(api, 'jira', query)
457+
expect(page.documents.map((document) => document.id)).toEqual(['NEW-3', 'NEW-2', 'NEW-1'])
458+
expect(page.nextCursor).toBe('page-2')
459+
})
460+
461+
it('counts a word and its plural as one term before allowing the half-the-terms fallback', async () => {
462+
const api = jira(overflowing, [], [{ key: 'ONE-1', summary: 'Rotation schedule' }])
463+
const page = await searchAtlassian(api, 'jira', { ...query, query: 'token tokens rotation' })
464+
expect(page.documents.map((document) => document.id)).toEqual(['NEW-3', 'NEW-2', 'NEW-1'])
465+
})
466+
467+
it('orders half-the-terms summaries by how many terms they hold', async () => {
468+
const api = jira(
469+
overflowing,
470+
[],
471+
[
472+
{ key: 'TWO-1', summary: 'Merged results dashboard' },
473+
{ key: 'THREE-1', summary: 'Merged results in release pipelines' },
474+
]
475+
)
476+
const page = await searchAtlassian(api, 'jira', {
477+
...query,
478+
query: 'merged results release pipelines',
479+
})
480+
expect(page.documents.map((document) => document.id)).toEqual(['THREE-1', 'TWO-1', 'NEW-3'])
481+
})
482+
483+
it('finds half-the-terms summaries beyond the first page of pair matches', async () => {
484+
const pairs = Array.from({ length: 10 }, (_, index) => ({
485+
key: `PAIR-${index}`,
486+
summary: 'Pipelines dashboard',
487+
}))
488+
const api = jira(overflowing, [], [...pairs, { key: 'HALF-9', summary: 'Merged pipelines' }])
489+
const page = await searchAtlassian(api, 'jira', query)
490+
expect(page.documents.map((document) => document.id)[0]).toBe('HALF-9')
491+
})
414492
})

‎apps/sim/lib/sim-search/live/atlassian.ts‎

Lines changed: 67 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,8 @@ import { providerText } from '@/lib/sim-search/live/text'
1212
import {
1313
confluenceTextClause,
1414
holdsHalfTheTerms,
15-
jiraSummaryClause,
1615
jiraTextClause,
16+
stemmedTerms,
1717
} from '@/lib/sim-search/live/text-match'
1818
import type {
1919
NativeClient,
@@ -117,6 +117,13 @@ async function sites(client: NativeClient) {
117117
return array(await client.json('/oauth/token/accessible-resources', { memo: true }))
118118
}
119119

120+
/**
121+
* Pair matches fetched for the half-the-terms summary fallback. Jira returns them newest first,
122+
* not by relevance, and the page is then filtered to summaries holding enough terms, so it starts
123+
* from a wider pool than the caller's limit or the any-term fallback's.
124+
*/
125+
const HALF_TERM_CANDIDATES = 50
126+
120127
/**
121128
* One Atlassian site's page; `ranking` marks a page whose order or completeness differs from the
122129
* all-terms search.
@@ -176,55 +183,82 @@ export async function searchAtlassian(
176183
const cloudId = string(site.id)
177184
const origin = string(site.url).replace(/\/$/, '')
178185
if (provider === 'jira') {
179-
const searchIssues = async (jql: string, cursor?: string): Promise<SitePage> => {
186+
/** One JQL page, each issue with its summary and raw fields. */
187+
const searchIssues = async (
188+
jql: string,
189+
{ cursor, maxResults = input.limit }: { cursor?: string; maxResults?: number } = {}
190+
) => {
180191
const data = object(
181192
await client.json(`/ex/jira/${segment(cloudId)}/rest/api/3/search/jql`, {
182193
body: {
183194
jql: scopeAtlassianQuery(jql, scope, order),
184-
maxResults: input.limit,
195+
maxResults,
185196
fields: ['summary', 'description', 'updated', 'creator', 'status'],
186197
...(cursor ? { nextPageToken: cursor } : {}),
187198
},
188199
signal: input.signal,
189200
})
190201
)
191202
return {
192-
documents: array(data.issues)
193-
.filter((row) => {
194-
const fields = object(row.fields)
195-
return (
196-
!relevant ||
197-
relevant(`${string(fields.summary)}\n${textContent(fields.description)}`)
198-
)
199-
})
200-
.map((row) => issue(row, cloudId, origin)),
201-
excluded: false,
203+
issues: array(data.issues).map((row) => {
204+
const fields = object(row.fields)
205+
return {
206+
document: issue(row, cloudId, origin),
207+
summary: string(fields.summary),
208+
fields,
209+
}
210+
}),
202211
next: string(data.nextPageToken) || undefined,
203212
}
204213
}
205214
const exact = await searchIssues(
206215
input.native?.query || (text ? jiraTextClause(text, input.textMatch) : ''),
207-
input.native?.cursor && single ? input.native.cursor : undefined
216+
{ cursor: input.native?.cursor && single ? input.native.cursor : undefined }
208217
)
218+
const exactPage: SitePage = {
219+
documents: exact.issues
220+
.filter(
221+
({ summary, fields }) =>
222+
!relevant || relevant(`${summary}\n${textContent(fields.description)}`)
223+
)
224+
.map(({ document }) => document),
225+
excluded: false,
226+
next: exact.next,
227+
}
209228
// JQL has no relevance order, so an all-terms match larger than a page comes back newest
210-
// first; issues whose summary holds every term lead it instead.
211-
if (!text || input.native || order || input.textMatch === 'any' || !exact.next) return exact
212-
try {
213-
const titled = await searchIssues(jiraSummaryClause(text))
214-
if (!titled.documents.length) return exact
215-
const leading = new Set(titled.documents.map((document) => document.id))
216-
return {
217-
documents: [
218-
...titled.documents,
219-
...exact.documents.filter((document) => !leading.has(document.id)),
220-
].slice(0, input.limit),
221-
excluded: false,
222-
ranking: 'relevance',
223-
}
224-
} catch {
225-
// A cancelled search stops here; any other failure keeps the all-terms page.
226-
input.signal?.throwIfAborted()
227-
return exact
229+
// first; issues whose summary holds every term, or else at least half of three or more
230+
// terms (most terms first), lead it instead.
231+
if (!text || input.native || order || input.textMatch === 'any' || !exact.next)
232+
return exactPage
233+
const terms = stemmedTerms(text)
234+
const [every, half] = await Promise.allSettled([
235+
searchIssues(jiraTextClause(text, 'all', 'summary')),
236+
terms.count >= 3
237+
? searchIssues(jiraTextClause(text, 'any', 'summary'), {
238+
maxResults: HALF_TERM_CANDIDATES,
239+
})
240+
: undefined,
241+
])
242+
// A cancelled search stops here; any other failure keeps the all-terms page.
243+
input.signal?.throwIfAborted()
244+
let titled =
245+
every.status === 'fulfilled' ? every.value.issues.map(({ document }) => document) : []
246+
if (!titled.length && half.status === 'fulfilled' && half.value) {
247+
titled = half.value.issues
248+
.map(({ document, summary }) => ({ document, held: terms.held(summary) }))
249+
.filter(({ held }) => held >= terms.required)
250+
.sort((left, right) => right.held - left.held)
251+
.map(({ document }) => document)
252+
}
253+
if (!titled.length) return exactPage
254+
const leading = new Set(titled.map((document) => document.id))
255+
return {
256+
documents: [
257+
...titled,
258+
...exactPage.documents.filter((document) => !leading.has(document.id)),
259+
].slice(0, input.limit),
260+
excluded: false,
261+
ranking: 'relevance',
228262
}
229263
}
230264
const searchPages = async (clause: string, cursor?: string) => {
@@ -289,7 +323,7 @@ export async function searchAtlassian(
289323
: '') +
290324
(pages.some((result) => result.ranking === 'relevance')
291325
? provider === 'jira'
292-
? ' More issues contained every word than fit on one page, so issues whose summary holds every word come first; refine the query, or page with a native JQL query.'
326+
? ' More issues contained every word than fit on one page, so issues whose summary holds every word, or else most of them, come first; refine the query, or page with a native JQL query.'
293327
: ' More pages contained every word than fit on one page, so this page is ordered by Confluence relevance search; refine the query, or page with a native CQL query.'
294328
: '') +
295329
(pages.some((result) => result.ranking === 'unavailable')

‎apps/sim/lib/sim-search/live/text-match.ts‎

Lines changed: 38 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -89,14 +89,19 @@ const escapeSearchPhrase = (value: string) =>
8989
JSON.stringify(value.replace(/[+\-&|!(){}[\]^"~*?:\\/]/g, '\\$&'))
9090

9191
/**
92-
* The `text ~` clause for a plain query in Jira JQL. `any` ORs every pair of the query's longest
93-
* terms, so a match contains at least two of them somewhere in the issue, or either term of a
94-
* two-term query. Jira stems text search, so a word and its plural share one term.
92+
* The clause for a plain query in Jira JQL, over all text fields or the summary alone. `any` ORs
93+
* every pair of the query's longest terms, so a match contains at least two of them in that field,
94+
* or either term of a two-term query. Jira stems text search, so a word and its plural share one
95+
* term.
9596
*/
96-
export function jiraTextClause(text: string, match: TextMatch = 'all'): string {
97+
export function jiraTextClause(
98+
text: string,
99+
match: TextMatch = 'all',
100+
field: 'text' | 'summary' = 'text'
101+
): string {
97102
const terms = match === 'any' ? distinctTerms(text, singularKey) : []
98-
if (terms.length < 2) return `text ~ ${escapeSearchPhrase(text)}`
99-
const clause = (term: string) => `text ~ ${escapeSearchPhrase(term)}`
103+
if (terms.length < 2) return `${field} ~ ${escapeSearchPhrase(text)}`
104+
const clause = (term: string) => `${field} ~ ${escapeSearchPhrase(term)}`
100105
const groups =
101106
terms.length === 2
102107
? terms.map(clause)
@@ -106,14 +111,6 @@ export function jiraTextClause(text: string, match: TextMatch = 'all'): string {
106111
return `(${groups.join(' OR ')})`
107112
}
108113

109-
/**
110-
* The `summary ~` clause for a plain query in Jira JQL: issues whose summary holds every word.
111-
* JQL has no relevance order, so this stands in for a title boost.
112-
*/
113-
export function jiraSummaryClause(text: string): string {
114-
return `summary ~ ${escapeSearchPhrase(text)}`
115-
}
116-
117114
/**
118115
* The text clause for a plain query in Confluence CQL. `any` uses `siteSearch`, Confluence's own
119116
* relevance search, which ranks a page with the query's rarer words above the many pages sharing
@@ -129,19 +126,35 @@ export function termCoverage(text: string, terms: readonly string[]): number {
129126
}
130127

131128
/**
132-
* A test for whether a text holds at least half the query's distinct terms. Query and text words
133-
* are compared by their singular key, so a word and its regular plural count as one term as a
134-
* stemmed search matches them; a term with punctuation inside it is matched as written.
129+
* The query's distinct terms as a stemmed search counts them, and how many of them a text holds.
130+
* Query and text words are compared by their singular key, so a word and its regular plural count
131+
* as one term; a term with punctuation inside it is matched as written.
135132
*/
136-
export function holdsHalfTheTerms(query: string): (text: string) => boolean {
133+
export function stemmedTerms(query: string): {
134+
count: number
135+
/** At least half the terms, the minimum a partial match must hold. */
136+
required: number
137+
held: (text: string) => number
138+
} {
137139
const terms = distinctTerms(query, singularKey)
138140
const keys = terms.map(singularKey)
139-
const required = Math.ceil(terms.length / 2)
140-
return (text) => {
141-
const words = new Set(textWords(text).map(singularKey))
142-
const present = terms.filter(
143-
(term, index) => words.has(keys[index]) || findTermMatches(text, [term]).length > 0
144-
)
145-
return present.length >= required
141+
return {
142+
count: terms.length,
143+
required: Math.ceil(terms.length / 2),
144+
held: (text) => {
145+
const words = new Set(textWords(text).map(singularKey))
146+
return terms.filter(
147+
(term, index) => words.has(keys[index]) || findTermMatches(text, [term]).length > 0
148+
).length
149+
},
146150
}
147151
}
152+
153+
/**
154+
* A test for whether a text holds at least half the query's distinct terms, as `stemmedTerms`
155+
* counts them.
156+
*/
157+
export function holdsHalfTheTerms(query: string): (text: string) => boolean {
158+
const { required, held } = stemmedTerms(query)
159+
return (text) => held(text) >= required
160+
}

0 commit comments

Comments
 (0)