|
9 | 9 | document, |
10 | 10 | knowledgeConnector, |
11 | 11 | knowledgeConnectorMember, |
| 12 | + knowledgeConnectorMemberSyncLog, |
| 13 | + knowledgeDocumentObservation, |
12 | 14 | organization, |
13 | 15 | resourcePolicy, |
14 | 16 | user, |
@@ -47,6 +49,7 @@ import { |
47 | 49 | materializeDocumentAcls, |
48 | 50 | recordMemberObservations, |
49 | 51 | rewriteConnectorAcls, |
| 52 | + sweepStaleMemberObservations, |
50 | 53 | } from '@/lib/knowledge/connectors/member-observations' |
51 | 54 | import { |
52 | 55 | executeMemberSync, |
@@ -395,6 +398,83 @@ describe('connector lease ACL pages in PostgreSQL', () => { |
395 | 398 | }) |
396 | 399 | }) |
397 | 400 |
|
| 401 | + describe('stale member sweep', () => { |
| 402 | + it('defers a connector whose row a member run holds and still sweeps the others', async () => { |
| 403 | + const busy = members |
| 404 | + const idle = await seedKnowledgeMemberFixture(ids) |
| 405 | + for (const fixture of [busy, idle]) { |
| 406 | + await db |
| 407 | + .update(knowledgeConnector) |
| 408 | + .set({ |
| 409 | + status: 'active', |
| 410 | + memberSyncStatus: 'idle', |
| 411 | + memberSyncLockToken: null, |
| 412 | + syncIntervalMinutes: 60, |
| 413 | + lastMemberSyncAt: new Date(), |
| 414 | + }) |
| 415 | + .where(eq(knowledgeConnector.id, fixture.connectorId)) |
| 416 | + /** Enrolled long enough ago that never having listed makes them stale. */ |
| 417 | + await db |
| 418 | + .update(knowledgeConnectorMember) |
| 419 | + .set({ createdAt: new Date(Date.now() - 3 * 24 * 60 * 60 * 1000) }) |
| 420 | + .where(eq(knowledgeConnectorMember.connectorId, fixture.connectorId)) |
| 421 | + const seeded = await seedDocuments( |
| 422 | + fixture.connectorId, |
| 423 | + fixture.members.map((member) => member.subjectToken).sort(), |
| 424 | + 3 |
| 425 | + ) |
| 426 | + for (const member of fixture.members) |
| 427 | + await recordMemberObservations( |
| 428 | + db, |
| 429 | + member.id, |
| 430 | + seeded.map((row) => row.id), |
| 431 | + fixture.runId |
| 432 | + ) |
| 433 | + } |
| 434 | + const observed = async (connectorId: string) => |
| 435 | + ( |
| 436 | + await db |
| 437 | + .select({ id: knowledgeDocumentObservation.memberId }) |
| 438 | + .from(knowledgeDocumentObservation) |
| 439 | + .innerJoin( |
| 440 | + knowledgeConnectorMember, |
| 441 | + eq(knowledgeConnectorMember.id, knowledgeDocumentObservation.memberId) |
| 442 | + ) |
| 443 | + .where(eq(knowledgeConnectorMember.connectorId, connectorId)) |
| 444 | + ).length |
| 445 | + |
| 446 | + /** A member page of a running sync holds the busy connector's row for longer than a sweep page waits. */ |
| 447 | + let release!: () => void |
| 448 | + let locked!: () => void |
| 449 | + const held = new Promise<void>((resolve) => { |
| 450 | + locked = resolve |
| 451 | + }) |
| 452 | + const holder = db.transaction(async (tx) => { |
| 453 | + await tx |
| 454 | + .select({ id: knowledgeConnector.id }) |
| 455 | + .from(knowledgeConnector) |
| 456 | + .where(eq(knowledgeConnector.id, busy.connectorId)) |
| 457 | + .for('update') |
| 458 | + locked() |
| 459 | + await new Promise<void>((resolve) => { |
| 460 | + release = resolve |
| 461 | + }) |
| 462 | + }) |
| 463 | + await held |
| 464 | + try { |
| 465 | + const result = await sweepStaleMemberObservations(new Date(Date.now() + 1_000)) |
| 466 | + expect(result.members).toBe(idle.members.length) |
| 467 | + } finally { |
| 468 | + release() |
| 469 | + await holder |
| 470 | + } |
| 471 | + |
| 472 | + expect(await observed(busy.connectorId)).toBe(busy.members.length * 3) |
| 473 | + expect(await observed(idle.connectorId)).toBe(0) |
| 474 | + expect((await storedAcls(idle.connectorId)).every((acl) => acl.length === 0)).toBe(true) |
| 475 | + }, 30_000) |
| 476 | + }) |
| 477 | + |
398 | 478 | describe('resumeMembershipRewrites', () => { |
399 | 479 | it('rematerialises a changed member one page per lease transaction', async () => { |
400 | 480 | const seeded = await seedDocuments(members.connectorId, []) |
@@ -649,5 +729,45 @@ describe('connector lease ACL pages in PostgreSQL', () => { |
649 | 729 | ) |
650 | 730 | expect(granted).toBe(0) |
651 | 731 | }) |
| 732 | + |
| 733 | + it('closes a run whose disable of a removed option fails as an ordinary failure', async () => { |
| 734 | + /** The option id is set but no longer exists, which membership reconciliation reports. */ |
| 735 | + await db |
| 736 | + .update(knowledgeConnector) |
| 737 | + .set({ credentialGroupOptionId: generateId() }) |
| 738 | + .where(eq(knowledgeConnector.id, members.connectorId)) |
| 739 | + await seedDocuments( |
| 740 | + members.connectorId, |
| 741 | + members.members.map((member) => member.subjectToken).sort() |
| 742 | + ) |
| 743 | + await db.execute( |
| 744 | + sql.raw(`CREATE OR REPLACE FUNCTION fail_after_acl_writes() RETURNS trigger |
| 745 | + LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'fixture statement failure'; END $$`) |
| 746 | + ) |
| 747 | + await db.execute( |
| 748 | + sql`CREATE TRIGGER fail_after_acl_writes BEFORE UPDATE OF acl ON document FOR EACH ROW |
| 749 | + WHEN (NEW.connector_id = ${sql.raw(`'${members.connectorId}'`)}) |
| 750 | + EXECUTE FUNCTION fail_after_acl_writes()` |
| 751 | + ) |
| 752 | + |
| 753 | + const failed = await executeMemberSync(members.connectorId, { |
| 754 | + billingAttribution: await billing(), |
| 755 | + }) |
| 756 | + |
| 757 | + expect(failed.error).toBeTruthy() |
| 758 | + const state = await connectorState() |
| 759 | + expect(state?.memberSyncStatus).toBe('error') |
| 760 | + expect(state?.memberSyncLockToken).toBeNull() |
| 761 | + const [log] = await db |
| 762 | + .select({ status: knowledgeConnectorMemberSyncLog.status }) |
| 763 | + .from(knowledgeConnectorMemberSyncLog) |
| 764 | + .where(eq(knowledgeConnectorMemberSyncLog.connectorId, members.connectorId)) |
| 765 | + expect(log?.status).toBe('failed') |
| 766 | + |
| 767 | + await db.execute(sql`DROP TRIGGER fail_after_acl_writes ON document`) |
| 768 | + await executeMemberSync(members.connectorId, { billingAttribution: await billing() }) |
| 769 | + expect((await connectorState())?.memberSyncStatus).toBe('disabled') |
| 770 | + expect((await storedAcls(members.connectorId)).every((acl) => acl.length === 0)).toBe(true) |
| 771 | + }) |
652 | 772 | }) |
653 | 773 | }) |
0 commit comments