fix(scim): close provisioning races and refresh policy state

Author: waleedlatif1

apps/sim/ee/access-control/components/group-detail.tsx

@@ -1588,7 +1588,9 @@ export function GroupDetail({
                   <div className='flex items-center justify-between gap-3'>
                     <span className='text-[var(--text-muted)] text-small'>
                       {members.length === 0
-                        ? 'Applies to all members of its workspaces. Add members to restrict it to specific people.'
+                        ? group.membershipMode === 'inherit'
+                          ? 'Applies to all members of its workspaces. Add members to restrict it to specific people.'
+                          : 'No members assigned.'
                         : `Restricted to ${members.length} member${members.length === 1 ? '' : 's'}`}
                     </span>
                     <Chip

apps/sim/ee/scim/TESTING.md

@@ -74,6 +74,8 @@ timeout.
   by conflict recovery and reconciliation.
 - A concurrent membership-lock change while a group removal waits on the
   organization lock; withdrawal uses the committed setting.
+- Concurrent name-matching changes while POST, PUT, and PATCH wait on the
+  organization lock; mapping uses the committed setting in both toggle directions.
 - Managed-membership enforcement through the actual batch invitation and
   workspace permission APIs, including successful edits to unmanaged members.
 - Session revocation on deactivation and permanent OAuth grant revocation on
@@ -125,9 +127,10 @@ the admission operation and Enterprise entitlement reads through PostgreSQL.
 
 The real PostgreSQL/Redis suite
 `lib/mothership/request/application/removal-stops-runs.integration.ts` verifies
-that deactivation writes durable Stops for admitted Chat runs, reactivation cannot
-revive those runs, an existing account first provisioned inactive is stopped,
-and a rolled-back suspension emits no durable or delivered Stop.
+that deactivation writes durable Stops for admitted Chat runs, including concurrent
+interactive and headless admission. Reactivation cannot revive those runs or a
+headless run admitted during suspension. An existing account first provisioned
+inactive is stopped, and a rolled-back suspension emits no durable or delivered Stop.
 
 ## Verification dates and limits
 

apps/sim/ee/scim/hooks/scim.ts

@@ -140,7 +140,7 @@ export function useUpsertScimGroupMapping() {
       requestJson(upsertScimGroupMappingContract, { params: { id: organizationId }, body }),
     onSettled: (_data, _error, { organizationId }) => {
       queryClient.invalidateQueries({ queryKey: scimKeys.mappings(organizationId) })
-      queryClient.invalidateQueries({ queryKey: permissionGroupKeys.list(organizationId) })
+      queryClient.invalidateQueries({ queryKey: permissionGroupKeys.all })
     },
   })
 }
@@ -157,7 +157,7 @@ export function useDeleteScimGroupMapping() {
       requestJson(deleteScimGroupMappingContract, { params: { id: organizationId, mappingId } }),
     onSettled: (_data, _error, { organizationId }) => {
       queryClient.invalidateQueries({ queryKey: scimKeys.mappings(organizationId) })
-      queryClient.invalidateQueries({ queryKey: permissionGroupKeys.list(organizationId) })
+      queryClient.invalidateQueries({ queryKey: permissionGroupKeys.all })
     },
   })
 }
@@ -170,7 +170,7 @@ export function useReconcileScimConnection() {
     onSettled: (_data, _error, organizationId) => {
       queryClient.invalidateQueries({ queryKey: scimKeys.connection(organizationId) })
       queryClient.invalidateQueries({ queryKey: scimKeys.mappings(organizationId) })
-      queryClient.invalidateQueries({ queryKey: permissionGroupKeys.list(organizationId) })
+      queryClient.invalidateQueries({ queryKey: permissionGroupKeys.all })
     },
   })
 }

apps/sim/ee/scim/lib/application/groups/manage-groups.test.ts

@@ -77,6 +77,7 @@ describe('replaceScimGroup', () => {
     queueTableRows(scimConnection, [
       { id: 'conn-1', organizationId: 'org-1', status: 'active', settings: {} },
     ])
+    queueTableRows(scimConnection, [{ status: 'active', settings: {} }])
     let group = { ...initialGroup }
     const memberIds = new Set(['user-1'])
     mocks.findGroup.mockImplementation(async () => group)

apps/sim/ee/scim/lib/application/groups/manage-groups.ts

@@ -1,6 +1,6 @@
 import { AuditAction, AuditResourceType } from '@sim/audit'
 import { db } from '@sim/db'
-import { scimGroup } from '@sim/db/schema'
+import { type ScimConnectionSettings, scimConnection, scimGroup } from '@sim/db/schema'
 import { and, eq, ne } from 'drizzle-orm'
 import type { ScimPatchOperation } from '@/lib/api/contracts/scim'
 import { acquireOrganizationMutationLock } from '@/lib/billing/organizations/membership'
@@ -57,11 +57,26 @@ import {
  */
 async function withGroupWrite<T extends { stoppedRuns: StoppedRun[] }>(
   context: ScimUseCaseContext,
-  work: (tx: DbTransaction) => Promise<T>
+  work: (tx: DbTransaction, settings: ScimConnectionSettings) => Promise<T>
 ): Promise<Omit<T, 'stoppedRuns'>> {
   const { stoppedRuns, ...result } = await db.transaction(async (tx) => {
     await acquireOrganizationMutationLock(tx, context.organizationId)
-    return work(tx)
+    const [connection] = await tx
+      .select({ status: scimConnection.status, settings: scimConnection.settings })
+      .from(scimConnection)
+      .where(
+        and(
+          eq(scimConnection.id, context.connection.id),
+          eq(scimConnection.organizationId, context.organizationId)
+        )
+      )
+      .limit(1)
+    if (!connection || connection.status !== 'active') {
+      throw new ScimError(401, undefined, 'Invalid SCIM token', {
+        'WWW-Authenticate': 'Bearer realm="SCIM"',
+      })
+    }
+    return work(tx, connection.settings)
   })
   deliverRunStops(stoppedRuns)
   return result
@@ -185,7 +200,7 @@ export const createScimGroup = defineAuthorizedScimUseCase({
     context,
   }: ScimUseCaseArgs<CreateScimGroupInput>): Promise<ScimGroupWriteResult> {
     const { group } = input
-    return withGroupWrite(context, async (tx) => {
+    return withGroupWrite(context, async (tx, settings) => {
       await assertDisplayNameAvailable(tx, {
         connectionId: context.connection.id,
         displayName: group.displayName,
@@ -202,7 +217,7 @@ export const createScimGroup = defineAuthorizedScimUseCase({
       }
       await assertMemberCount(tx, created.id)
 
-      const mapped = context.connection.settings.autoMapPermissionGroupsByName
+      const mapped = settings.autoMapPermissionGroupsByName
         ? await autoMapPermissionGroupByName(tx, {
             organizationId: context.organizationId,
             scimGroupId: created.id,
@@ -253,7 +268,7 @@ export const replaceScimGroup = defineAuthorizedScimUseCase({
     input,
     context,
   }: ScimUseCaseArgs<ReplaceScimGroupInput>): Promise<ScimGroupWriteResult> {
-    return withGroupWrite(context, async (tx) => {
+    return withGroupWrite(context, async (tx, settings) => {
       const current = await findScimGroupById(tx, context.connection.id, input.groupId)
       if (!current) throw notFound('SCIM Group not found')
 
@@ -277,7 +292,7 @@ export const replaceScimGroup = defineAuthorizedScimUseCase({
         })
       }
       let adopted = false
-      if (renamed && context.connection.settings.autoMapPermissionGroupsByName) {
+      if (renamed && settings.autoMapPermissionGroupsByName) {
         const mapped = await autoMapPermissionGroupByName(tx, {
           organizationId: context.organizationId,
           scimGroupId: current.id,
@@ -358,7 +373,7 @@ export const patchScimGroup = defineAuthorizedScimUseCase({
   }: ScimUseCaseArgs<PatchScimGroupInput>): Promise<PatchScimGroupResult> {
     const patch = parseGroupPatch(input.operations)
 
-    return withGroupWrite(context, async (tx) => {
+    return withGroupWrite(context, async (tx, settings) => {
       const current = await findScimGroupById(tx, context.connection.id, input.groupId)
       if (!current) throw notFound('SCIM Group not found')
 
@@ -402,11 +417,7 @@ export const patchScimGroup = defineAuthorizedScimUseCase({
             ...(externalIdChanged ? { externalId: patch.externalId } : {}),
           })
         }
-        if (
-          renamed &&
-          patch.displayName &&
-          context.connection.settings.autoMapPermissionGroupsByName
-        ) {
+        if (renamed && patch.displayName && settings.autoMapPermissionGroupsByName) {
           const mapped = await autoMapPermissionGroupByName(tx, {
             organizationId: context.organizationId,
             scimGroupId: current.id,

apps/sim/lib/mothership/async-runs/repository.test.ts

@@ -42,14 +42,15 @@ describe('run admission and early Stop', () => {
     expect(await requestRunStop(scope)).toBeNull()
     expect(dbChainMockFns.values).toHaveBeenCalledWith(scope)
     expect(dbChainMockFns.onConflictDoNothing).toHaveBeenCalled()
-    expect(dbChainMockFns.execute).toHaveBeenCalledTimes(2)
     expect(
-      hasMockCondition(
-        dbChainMockFns.where.mock.calls[0]?.[0],
-        (condition) =>
-          condition.type === 'eq' &&
-          condition.left === copilotRuns.userId &&
-          condition.right === scope.userId
+      dbChainMockFns.where.mock.calls.some(([predicate]) =>
+        hasMockCondition(
+          predicate,
+          (condition) =>
+            condition.type === 'eq' &&
+            condition.left === copilotRuns.userId &&
+            condition.right === scope.userId
+        )
       )
     ).toBe(true)
   })
@@ -148,12 +149,13 @@ describe('run admission and early Stop', () => {
       [copilotRequestStops.workspaceId, scope.workspaceId],
       [copilotRequestStops.streamId, scope.streamId],
     ]) {
-      /** The chat row is read first; the Stop lookup follows it. */
       expect(
-        hasMockCondition(
-          dbChainMockFns.where.mock.calls[1]?.[0],
-          (condition) =>
-            condition.type === 'eq' && condition.left === column && condition.right === value
+        dbChainMockFns.where.mock.calls.some(([predicate]) =>
+          hasMockCondition(
+            predicate,
+            (condition) =>
+              condition.type === 'eq' && condition.left === column && condition.right === value
+          )
         )
       ).toBe(true)
     }
@@ -167,12 +169,14 @@ describe('run admission and early Stop', () => {
       expect.objectContaining({ workspaceId: scope.workspaceId, status: 'active' })
     )
     expect(
-      hasMockCondition(
-        dbChainMockFns.where.mock.calls[0]?.[0],
-        (condition) =>
-          condition.type === 'eq' &&
-          condition.left === copilotChats.userId &&
-          condition.right === scope.userId
+      dbChainMockFns.where.mock.calls.some(([predicate]) =>
+        hasMockCondition(
+          predicate,
+          (condition) =>
+            condition.type === 'eq' &&
+            condition.left === copilotChats.userId &&
+            condition.right === scope.userId
+        )
       )
     ).toBe(true)
   })

apps/sim/lib/mothership/async-runs/repository.ts

@@ -9,6 +9,7 @@ import {
   copilotOrganizationRequestStops,
   copilotRequestStops,
   copilotRuns,
+  user,
 } from '@sim/db/schema'
 import { createLogger } from '@sim/logger'
 import { filterUndefined, toRecordOrNull } from '@sim/utils/object'
@@ -154,14 +155,16 @@ export interface CreateRunSegmentInput {
 
 type RunAdmissionTransaction = Parameters<Parameters<typeof db.transaction>[0]>[0]
 
-/** Serializes Stop and admission, including when no run row exists to lock yet. */
+/** Serializes account suspension, Stop and admission, even before a run row exists. */
 export async function withRunAdmissionLock<T>(
   userId: string,
   streamId: string,
   action: (tx: RunAdmissionTransaction) => Promise<T>
 ): Promise<T> {
   return db.transaction(async (tx) => {
     await tx.execute(sql`SET LOCAL statement_timeout = '10s'`)
+    /** Lock the account before any chat/run writes; suspension updates it before scanning runs. */
+    await tx.select({ id: user.id }).from(user).where(eq(user.id, userId)).for('share')
     const key = JSON.stringify(['copilot-run-admission', userId, streamId])
     await acquireAdvisoryXactLock(tx, 'copilot_run_admission', key)
     return action(tx)
@@ -462,8 +465,10 @@ export async function insertRunSegment(tx: RunAdmissionTransaction, input: Creat
       workspaceId: copilotChats.workspaceId,
       organizationId: copilotChats.organizationId,
       deletedAt: copilotChats.deletedAt,
+      suspendedAt: user.suspendedAt,
     })
     .from(copilotChats)
+    .innerJoin(user, eq(user.id, copilotChats.userId))
     .where(and(eq(copilotChats.id, input.chatId), eq(copilotChats.userId, input.userId)))
     .for('share')
   const inheritsOwner = !input.workspaceId && !input.organizationId
@@ -475,7 +480,10 @@ export async function insertRunSegment(tx: RunAdmissionTransaction, input: Creat
     .from(table)
     .where(predicate)
     .limit(1)
-  const closedAt = stop?.stoppedAt ?? chat?.deletedAt
+  const closedAt = stop?.stoppedAt ?? chat?.deletedAt ?? chat?.suspendedAt
+  if (chat?.suspendedAt && !stop) {
+    await recordRunStopTx(tx, { ...input, workspaceId, organizationId }, false)
+  }
   const [run] = await tx
     .insert(copilotRuns)
     .values({