Skip to content

Commit 1faa8db

Browse files
committed
fix(files): preserve active artifact indexing and narrow helpers
1 parent a6d8ab9 commit 1faa8db

10 files changed

Lines changed: 134 additions & 80 deletions

File tree

‎apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ import {
2020
getErrorMessage,
2121
getPostgresConstraintName,
2222
getPostgresErrorCode,
23-
isPostgresCommitRejection,
2423
} from '@sim/utils/errors'
2524
import { generateShortId } from '@sim/utils/id'
2625
import { omit } from '@sim/utils/object'
@@ -582,6 +581,30 @@ async function discardStagedFileContent(staged: StagedFileContent): Promise<void
582581
})
583582
}
584583

584+
const POSTGRES_COMMIT_REJECTION_CODES = new Set([
585+
'23000',
586+
'23001',
587+
'23502',
588+
'23503',
589+
'23505',
590+
'23514',
591+
'23P01',
592+
'40000',
593+
'40001',
594+
'40002',
595+
'40P01',
596+
])
597+
598+
/**
599+
* Recognizes explicit PostgreSQL constraint/rollback rejections at COMMIT. Use only on the
600+
* transaction's rejection after its callback completed, never on errors from post-commit work.
601+
* Unknown outcomes, including 40003 and connection failures, are not safe cleanup authority.
602+
*/
603+
function isPostgresCommitRejection(error: unknown): boolean {
604+
const code = getPostgresErrorCode(error)
605+
return code !== undefined && POSTGRES_COMMIT_REJECTION_CODES.has(code)
606+
}
607+
585608
/** Discards known rollbacks while retaining bytes whose COMMIT outcome is uncertain. */
586609
async function finalizeStagedFileContent<T>(
587610
staged: StagedFileContent,

‎apps/sim/lib/uploads/documents/references.ts‎

Lines changed: 48 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { tokenizer } from 'acorn'
1+
import ts from '@typescript/typescript6'
22
import { DocCompileUserError } from '@/lib/uploads/documents/compile-error'
33

44
export type DocumentSourceLanguage = 'javascript' | 'python'
@@ -18,31 +18,55 @@ interface SourceToken {
1818
const FILE_HELPERS = new Set(['getFileBase64', 'addImage', 'drawImage', 'input_path'])
1919

2020
function* javascriptTokens(source: string): Generator<SourceToken> {
21-
try {
22-
const tokens = tokenizer(source, { ecmaVersion: 'latest', allowHashBang: true })
23-
for (let token = tokens.getToken(); token.type.label !== 'eof'; token = tokens.getToken()) {
24-
const label = token.type.label
25-
if (label === 'string') {
26-
yield {
27-
kind: 'string',
28-
value: source.slice(token.start + 1, token.end - 1),
29-
start: token.start + 1,
30-
}
31-
} else if (label === 'template') {
32-
if (source[token.start - 1] === '`' && source[token.end] === '`') {
33-
yield { kind: 'string', value: source.slice(token.start, token.end), start: token.start }
34-
}
35-
} else {
36-
yield {
37-
kind: label === 'name' ? 'word' : 'punctuation',
38-
value: source.slice(token.start, token.end),
39-
start: token.start,
40-
}
21+
const fileName = 'document.js'
22+
const sourceFile = ts.createSourceFile(
23+
fileName,
24+
source,
25+
ts.ScriptTarget.Latest,
26+
true,
27+
ts.ScriptKind.JS
28+
)
29+
const program = ts.createProgram(
30+
[fileName],
31+
{ allowJs: true, noLib: true, noResolve: true },
32+
{
33+
getSourceFile: (name) => (name === fileName ? sourceFile : undefined),
34+
getDefaultLibFileName: () => '',
35+
writeFile: () => {},
36+
getCurrentDirectory: () => '',
37+
getDirectories: () => [],
38+
fileExists: (name) => name === fileName,
39+
readFile: (name) => (name === fileName ? source : undefined),
40+
getCanonicalFileName: (name) => name,
41+
useCaseSensitiveFileNames: () => true,
42+
getNewLine: () => '\n',
43+
}
44+
)
45+
const diagnostic = program.getSyntacticDiagnostics(sourceFile)[0]
46+
if (diagnostic) {
47+
throw new DocCompileUserError(ts.flattenDiagnosticMessageText(diagnostic.messageText, '\n'))
48+
}
49+
50+
const pending: ts.Node[] = [sourceFile]
51+
while (pending.length > 0) {
52+
const node = pending.pop()
53+
if (!node) break
54+
if (ts.isStringLiteralLike(node)) {
55+
const start = node.getStart(sourceFile) + 1
56+
yield { kind: 'string', value: source.slice(start, node.end - 1), start }
57+
continue
58+
}
59+
const children = node.getChildren(sourceFile)
60+
if (children.length > 0) {
61+
for (let index = children.length - 1; index >= 0; index--) pending.push(children[index])
62+
} else {
63+
const start = node.getStart(sourceFile)
64+
yield {
65+
kind: ts.isIdentifier(node) ? 'word' : 'punctuation',
66+
value: source.slice(start, node.end),
67+
start,
4168
}
4269
}
43-
} catch (error) {
44-
if (error instanceof SyntaxError) throw new DocCompileUserError(error.message)
45-
throw error
4670
}
4771
}
4872

‎apps/sim/lib/workspace-files/search/artifact-ready.ts‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,11 +51,18 @@ export async function markFileSearchArtifactReadyInTx(
5151
const [state] = await tx.select().from(workspaceFileSearchRevision).where(predicate).limit(1)
5252
if (state?.buildId) {
5353
const [build] = await tx
54-
.select({ artifactKey: workspaceFileSearchBuild.artifactKey })
54+
.select({
55+
artifactKey: workspaceFileSearchBuild.artifactKey,
56+
live: sql<boolean>`${workspaceFileSearchBuild.expiresAt} > clock_timestamp()`,
57+
})
5558
.from(workspaceFileSearchBuild)
5659
.where(eq(workspaceFileSearchBuild.id, state.buildId))
5760
.for('update')
58-
if (state.status === 'ready' && build?.artifactKey === input.artifactKey) return false
61+
if (
62+
build?.artifactKey === input.artifactKey &&
63+
(state.status === 'ready' || (state.status === 'pending' && build.live))
64+
)
65+
return false
5966
await tx
6067
.update(workspaceFileSearchBuild)
6168
.set({ expiresAt: sql`clock_timestamp()` })

‎apps/sim/lib/workspace-files/search/index-state.ts‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,24 @@ export async function beginFileSearchBuild(
201201
})
202202
}
203203

204+
/** Records the selected artifact before extraction so repeated render receipts preserve this build. */
205+
export async function bindFileSearchBuildArtifact(
206+
build: FileSearchBuild,
207+
artifactKey: string,
208+
dependencies: readonly FileSearchDependencyIdentity[]
209+
): Promise<boolean> {
210+
return db.transaction(async (tx) => {
211+
await configureFileSearchTransaction(tx, FILE_SEARCH_INDEX_TRANSACTION_LIMITS)
212+
if (!(await lockFileSearchInputs(tx, build, dependencies)) || !(await lockBuild(tx, build)))
213+
return false
214+
await tx
215+
.update(workspaceFileSearchBuild)
216+
.set({ artifactKey })
217+
.where(eq(workspaceFileSearchBuild.id, build.id))
218+
return true
219+
})
220+
}
221+
204222
/** Each batch is fenced and work-bounded; no file or parser work runs inside this transaction. */
205223
export async function appendFileSearchChunks(
206224
build: FileSearchBuild,

‎apps/sim/lib/workspace-files/search/indexing.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import {
2828
import {
2929
appendFileSearchChunks,
3030
beginFileSearchBuild,
31+
bindFileSearchBuildArtifact,
3132
type FileSearchBuild,
3233
type FileSearchRevision,
3334
failFileSearchRevision,
@@ -117,6 +118,8 @@ export async function indexWorkspaceFileForSearch(
117118
const source = await loadFileSearchSource(revision, signal)
118119
if (!source) return
119120
const { file, bytes, dependencies, artifactKey } = source
121+
if (artifactKey && !(await bindFileSearchBuildArtifact(build, artifactKey, dependencies)))
122+
return
120123
const extracted = await extractIndexText(bytes, file.originalName, signal)
121124
if (!extracted) {
122125
await publishFileSearchBuild(

‎apps/sim/lib/workspace-files/search/owners.integration.ts‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -324,7 +324,38 @@ describe('owner-scoped indexed file search on PostgreSQL and stored bytes', () =
324324
})
325325
expect(served.buffer).toEqual(artifact)
326326
expect(served.contributingFiles?.map((file) => file.context)).toEqual([context])
327-
await indexWorkspaceFileForSearch({ owner, ...(await claim(generated.id)) }, signal)
327+
const extraction = await import('@/lib/workspace-files/search/extract')
328+
const extract = extraction.extractIndexText
329+
const entered = createDeferred<void>()
330+
const release = createDeferred<void>()
331+
const pause = vi
332+
.spyOn(extraction, 'extractIndexText')
333+
.mockImplementationOnce(async (...args) => {
334+
entered.resolve()
335+
await release.promise
336+
return extract(...args)
337+
})
338+
const indexing = indexWorkspaceFileForSearch(
339+
{ owner, ...(await claim(generated.id)) },
340+
signal
341+
)
342+
await entered.promise
343+
try {
344+
for (let repeat = 0; repeat < 2; repeat++) {
345+
const cached = await fetchAuthorizedServableWorkspaceFileBuffer(
346+
generated,
347+
f.principal,
348+
{
349+
maxBytes: 1024 * 1024,
350+
}
351+
)
352+
expect(cached.buffer).toEqual(artifact)
353+
}
354+
} finally {
355+
release.resolve()
356+
await indexing
357+
pause.mockRestore()
358+
}
328359
const read = () =>
329360
searchWorkspaceFileContent.execute({
330361
principal: f.principal,

‎apps/sim/package.json‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,6 @@
168168
"@xterm/addon-webgl": "0.19.0",
169169
"@xterm/xterm": "6.0.0",
170170
"@xyflow/react": "12.11.3",
171-
"acorn": "8.17.0",
172171
"ajv": "8.18.0",
173172
"archiver": "8.0.0",
174173
"better-auth": "1.6.27",

‎bun.lock‎

Lines changed: 0 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎packages/utils/src/errors.test.ts‎

Lines changed: 0 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ import {
44
getPostgresCancellationReason,
55
getPostgresErrorCode,
66
getTransientDatabaseFailure,
7-
isPostgresCommitRejection,
87
} from '@sim/utils/errors'
98
import { describe, expect, it } from 'vitest'
109

@@ -237,28 +236,3 @@ describe('describeError', () => {
237236
expect(described?.causeChain?.length).toBeLessThanOrEqual(10)
238237
})
239238
})
240-
241-
describe('isPostgresCommitRejection', () => {
242-
it('recognizes a deferred constraint rejection through a transaction wrapper', () => {
243-
const rejection = Object.assign(new Error('deferred constraint failed'), { code: '23514' })
244-
expect(isPostgresCommitRejection(new Error('transaction failed', { cause: rejection }))).toBe(
245-
true
246-
)
247-
})
248-
249-
it.each(['40003', '08007', 'CONNECTION_CLOSED', 'ECONNRESET', undefined])(
250-
'does not treat uncertain outcome %s as permission to discard staged data',
251-
(code) => {
252-
const error = Object.assign(new Error('commit outcome unavailable'), { code })
253-
expect(isPostgresCommitRejection(error)).toBe(false)
254-
}
255-
)
256-
257-
it('does not replace an outer connection failure with a nested rejection code', () => {
258-
const rejection = Object.assign(new Error('earlier constraint failure'), { code: '23514' })
259-
const connection = Object.assign(new Error('connection lost', { cause: rejection }), {
260-
code: 'CONNECTION_CLOSED',
261-
})
262-
expect(isPostgresCommitRejection(connection)).toBe(false)
263-
})
264-
})

‎packages/utils/src/errors.ts‎

Lines changed: 0 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -30,30 +30,6 @@ export function getPostgresErrorCode(error: unknown): string | undefined {
3030
return readPgErrorField(error, 'code')
3131
}
3232

33-
const POSTGRES_COMMIT_REJECTION_CODES = new Set([
34-
'23000',
35-
'23001',
36-
'23502',
37-
'23503',
38-
'23505',
39-
'23514',
40-
'23P01',
41-
'40000',
42-
'40001',
43-
'40002',
44-
'40P01',
45-
])
46-
47-
/**
48-
* Recognizes explicit PostgreSQL constraint/rollback rejections at COMMIT. Use only on the
49-
* transaction's rejection after its callback completed, never on errors from post-commit work.
50-
* Unknown outcomes, including 40003 and connection failures, are not safe cleanup authority.
51-
*/
52-
export function isPostgresCommitRejection(error: unknown): boolean {
53-
const code = getPostgresErrorCode(error)
54-
return code !== undefined && POSTGRES_COMMIT_REJECTION_CODES.has(code)
55-
}
56-
5733
const POSTGRES_CANCELLATION_REASONS = [
5834
['57014', 'canceling statement due to statement timeout', 'statement_timeout'],
5935
['57014', 'canceling statement due to user request', 'user_cancel'],

0 commit comments

Comments
 (0)