@@ -15,6 +15,14 @@ const LOCK_TIMEOUT_MS = 1_000
1515/** A probe may walk the dead index entries a purge leaves until VACUUM, so it gets longer. */
1616const PROBE_STATEMENT_TIMEOUT_MS = 30_000
1717const PURGE_LOCK = 'sim:search-projection-purge'
18+ /**
19+ * Whether this backend holds the session-level purge lock. A bigint advisory key appears in
20+ * `pg_locks` split into `classid` (high 32 bits) and `objid` (low 32 bits) with `objsubid` 1.
21+ */
22+ const HOLDS_PURGE_LOCK = `EXISTS (
23+ SELECT 1 FROM pg_locks WHERE locktype = 'advisory' AND pid = pg_backend_pid() AND granted
24+ AND objsubid = 1 AND ((classid::bigint << 32) | objid::bigint) = hashtextextended('${ PURGE_LOCK } ', 0)
25+ )`
1826/**
1927 * Keeps every scan on `embedding_search_kb_idx`: a sequential scan would restart from the heap's
2028 * start each batch, over the dead rows earlier batches left, and a bitmap scan would collect a
@@ -108,6 +116,35 @@ export function parseOperatorDatabaseUrl(rawUrl: string | undefined): OperatorDa
108116 return { url, hosted }
109117}
110118
119+ /**
120+ * The purge's single-backend client for a validated operator URL. The TLS query parameters are
121+ * enforced here as `ssl` and removed from the URL: postgres.js forwards `sslrootcert` to the
122+ * server as a startup parameter, which PostgreSQL rejects as unrecognized. The search path is
123+ * pinned because the purge reads and deletes unqualified tables. Connects on the first query.
124+ */
125+ export function connectOperatorDatabase ( rawUrl : string | undefined ) : Sql {
126+ const { url, hosted } = parseOperatorDatabaseUrl ( rawUrl )
127+ const clientUrl = new URL ( url )
128+ clientUrl . searchParams . delete ( 'sslmode' )
129+ clientUrl . searchParams . delete ( 'sslrootcert' )
130+ return postgres ( clientUrl . toString ( ) , {
131+ port : Number ( url . port || '5432' ) ,
132+ ssl : hosted ? 'verify-full' : false ,
133+ max : 1 ,
134+ prepare : false ,
135+ connect_timeout : 5 ,
136+ max_lifetime : null ,
137+ connection : {
138+ application_name : 'sim-search-projection-purge' ,
139+ search_path : 'public, pg_temp' ,
140+ statement_timeout : STATEMENT_TIMEOUT_MS ,
141+ lock_timeout : LOCK_TIMEOUT_MS ,
142+ idle_in_transaction_session_timeout : 5_000 ,
143+ } ,
144+ onnotice : ( ) => undefined ,
145+ } )
146+ }
147+
111148/** Search knowledge bases in id order; the table is small and read once per run. */
112149async function searchKnowledgeBases ( sql : Sql ) : Promise < string [ ] > {
113150 const rows = await sql < { id : string } [ ] > `
@@ -167,10 +204,18 @@ interface SearchProjectionPurgeResult {
167204 * Deletes one batch of a Search knowledge base's candidate rows. The rows are chosen through
168205 * `embedding_search_kb_idx` and deleted by tuple id, re-checking the knowledge base so a row
169206 * moved or reassigned since the scan is left alone. Deletes nothing when the knowledge base is gone.
207+ * Refuses when this backend no longer holds the purge lock: the client reconnects silently after a
208+ * dropped connection, and the new backend does not inherit the old one's session lock.
170209 */
171210async function purgeBatch ( sql : Sql , knowledgeBaseId : string , batchSize : number ) : Promise < number > {
172211 return sql . begin ( async ( tx ) => {
173- await tx `SELECT ${ tx . unsafe ( INDEX_SCANS_ONLY ) } `
212+ const [ session ] = await tx < { locked : boolean } [ ] > `
213+ SELECT ${ tx . unsafe ( HOLDS_PURGE_LOCK ) } AS locked, ${ tx . unsafe ( INDEX_SCANS_ONLY ) } `
214+ if ( ! session . locked ) {
215+ throw new SearchProjectionPurgeError (
216+ 'The database connection was replaced and no longer holds the purge lock; rerun to continue'
217+ )
218+ }
174219 const [ base ] = await tx < { is_search_index : boolean } [ ] > `
175220 SELECT is_search_index FROM knowledge_base WHERE id = ${ knowledgeBaseId } FOR SHARE`
176221 if ( ! base ) return 0
@@ -194,9 +239,10 @@ async function purgeBatch(sql: Sql, knowledgeBaseId: string, batchSize: number):
194239}
195240
196241/**
197- * Deletes the candidate rows of every Search knowledge base, batch by batch, pausing after each.
198- * Stateless: whatever a stopped run leaves, the next run finds and deletes. Canonical
199- * `embedding` and `document` rows are never touched.
242+ * Deletes the candidate rows of every Search knowledge base, batch by batch, pausing after each,
243+ * while holding a session lock that keeps a second run out. Stateless: whatever a stopped run
244+ * leaves, the next run finds and deletes. Canonical `embedding` and `document` rows are never
245+ * touched.
200246 */
201247export async function purgeSearchProjection (
202248 sql : Sql ,
@@ -205,20 +251,28 @@ export async function purgeSearchProjection(
205251 const batchSize = options . batchSize ?? DEFAULT_BATCH_SIZE
206252 const pauseMs = options . pauseMs ?? DEFAULT_PAUSE_MS
207253 const maxBatches = options . maxBatches ?? Number . POSITIVE_INFINITY
254+ const [ { acquired } ] = await sql < { acquired : boolean } [ ] > `
255+ SELECT pg_try_advisory_lock(hashtextextended(${ PURGE_LOCK } , 0)) AS acquired`
256+ if ( ! acquired ) throw new SearchProjectionPurgeError ( 'Another purge is running' )
208257 let deleted = 0
209258 let batches = 0
210- for ( const knowledgeBaseId of await searchKnowledgeBases ( sql ) ) {
211- for ( ; ; ) {
212- if ( batches >= maxBatches ) return { deleted, batches, remaining : true }
213- const count = await purgeBatch ( sql , knowledgeBaseId , batchSize )
214- if ( count === 0 ) break
215- batches += 1
216- deleted += count
217- await options . onBatch ?.( { knowledgeBaseId, deleted : count } )
218- if ( pauseMs > 0 ) await sleep ( pauseMs )
259+ try {
260+ for ( const knowledgeBaseId of await searchKnowledgeBases ( sql ) ) {
261+ for ( ; ; ) {
262+ if ( batches >= maxBatches ) return { deleted, batches, remaining : true }
263+ const count = await purgeBatch ( sql , knowledgeBaseId , batchSize )
264+ if ( count === 0 ) break
265+ batches += 1
266+ deleted += count
267+ await options . onBatch ?.( { knowledgeBaseId, deleted : count } )
268+ if ( pauseMs > 0 ) await sleep ( pauseMs )
269+ }
219270 }
271+ return { deleted, batches, remaining : false }
272+ } finally {
273+ // On a replaced connection this unlocks nothing; the old backend's lock ended with it.
274+ await sql `SELECT pg_advisory_unlock(hashtextextended(${ PURGE_LOCK } , 0))`
220275 }
221- return { deleted, batches, remaining : false }
222276}
223277
224278function integerOption (
@@ -253,23 +307,7 @@ async function main(): Promise<void> {
253307 const pauseMs = integerOption ( values [ 'pause-ms' ] , DEFAULT_PAUSE_MS , 0 , MAX_PAUSE_MS )
254308 const maxBatches =
255309 values [ 'max-batches' ] === undefined ? undefined : integerOption ( values [ 'max-batches' ] , 1 , 1 )
256- const rawUrl = process . env . MIGRATION_DATABASE_URL
257- const { url, hosted } = parseOperatorDatabaseUrl ( rawUrl )
258- const sql = postgres ( url . toString ( ) , {
259- port : Number ( url . port || '5432' ) ,
260- ssl : hosted ? 'verify-full' : false ,
261- max : 1 ,
262- prepare : false ,
263- connect_timeout : 5 ,
264- max_lifetime : null ,
265- connection : {
266- application_name : 'sim-search-projection-purge' ,
267- statement_timeout : STATEMENT_TIMEOUT_MS ,
268- lock_timeout : LOCK_TIMEOUT_MS ,
269- idle_in_transaction_session_timeout : 5_000 ,
270- } ,
271- onnotice : ( ) => undefined ,
272- } )
310+ const sql = connectOperatorDatabase ( process . env . MIGRATION_DATABASE_URL )
273311 try {
274312 if ( ! values . execute ) {
275313 const states = await inspectSearchProjection ( sql )
@@ -288,9 +326,6 @@ async function main(): Promise<void> {
288326 throw new SearchProjectionPurgeError (
289327 'A shadow-copy Search retirement job is still installed; abort it with the release that shipped it first'
290328 )
291- const [ { acquired } ] = await sql < { acquired : boolean } [ ] > `
292- SELECT pg_try_advisory_lock(hashtextextended(${ PURGE_LOCK } , 0)) AS acquired`
293- if ( ! acquired ) throw new SearchProjectionPurgeError ( 'Another purge is running' )
294329 const result = await purgeSearchProjection ( sql , {
295330 batchSize,
296331 pauseMs,
0 commit comments