fix(audit-logs): list an organization's audit log through its single scope predicate again (#9438)

#9419 read a listing's page ids as one keyset scan per organization workspace plus the
organization-level rows. That bounds an unfiltered listing, but a filter the workspace
keyset index cannot apply in order (an action, a resource, an actor) turns each
workspace's scan into a bitmap AND of that workspace's whole history with the filter's
index, then a sort, once per workspace. For a busy organization a filtered listing went
from one scan of the filter's index to many times that, and slow replica reads are what
recovery conflicts cancel.

Restore the listing to the single organization-scope predicate it used before, which lets
the planner pick the filter's index, keep the org-level index from 0444 (the predicate's
organization-level rows can use it), and keep the scope integration test against the
restored signature. The per-workspace plan test described the reverted shape and goes.

Author: waleedlatif1

apps/sim/app/api/v1/audit-logs/route.ts

@@ -25,7 +25,12 @@ import { generateId } from '@sim/utils/id'
 import { type NextRequest, NextResponse } from 'next/server'
 import { v1ListAuditLogsContract } from '@/lib/api/contracts/v1/audit-logs'
 import { parseRequest } from '@/lib/api/server'
-import { buildFilterConditions, getOrgWorkspaceIds, queryAuditLogs } from '@/lib/audit-logs/query'
+import {
+  buildFilterConditions,
+  buildOrgScopeCondition,
+  getOrgWorkspaceIds,
+  queryAuditLogs,
+} from '@/lib/audit-logs/query'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
 import { validateV1EnterpriseAuditAccess } from '@/app/api/v1/audit-logs/auth'
 import { formatAuditLogEntry } from '@/app/api/v1/audit-logs/format'
@@ -94,12 +99,12 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
       )
     }
 
-    const scope = {
+    const scopeCondition = buildOrgScopeCondition({
       organizationId,
       orgWorkspaceIds,
       orgMemberIds,
       includeDeparted: params.includeDeparted,
-    }
+    })
     const filterConditions = buildFilterConditions({
       action: params.action,
       resourceType: params.resourceType,
@@ -111,8 +116,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
     })
 
     const { data, nextCursor } = await queryAuditLogs(
-      scope,
-      filterConditions,
+      [scopeCondition, ...filterConditions],
       params.limit,
       params.cursor
     )

apps/sim/lib/audit-logs/application/export-audit-logs.ts

@@ -5,6 +5,7 @@ import { AUDIT_LOG_CSV_HEADER, toAuditLogCsvRow } from '@/lib/audit-logs/csv'
 import {
   type AuditLogFilterParams,
   buildFilterConditions,
+  buildOrgScopeCondition,
   getOrgWorkspaceIds,
   queryAuditLogs,
 } from '@/lib/audit-logs/query'
@@ -36,13 +37,13 @@ export const exportAuditLogs = defineAuthorizedAuditLogUseCase({
     ) {
       throw new OrchestrationError('validation', 'actorId is not a member of your organization')
     }
-    const scope = {
+    const scope = buildOrgScopeCondition({
       organizationId: context.organizationId,
       orgWorkspaceIds: workspaceIds,
       orgMemberIds: context.orgMemberIds,
       includeDeparted: input.includeDeparted,
-    }
-    const conditions = buildFilterConditions(input.filters)
+    })
+    const conditions = [scope, ...buildFilterConditions(input.filters)]
     const lines = [AUDIT_LOG_CSV_HEADER]
     let rowCount = 0
     let cursor: string | undefined
@@ -51,7 +52,6 @@ export const exportAuditLogs = defineAuthorizedAuditLogUseCase({
     pages: while (rowCount < EXPORT_MAX_ROWS && bytes < EXPORT_MAX_BYTES) {
       request?.signal?.throwIfAborted()
       const page = await queryAuditLogs(
-        scope,
         conditions,
         Math.min(EXPORT_PAGE_ROWS, EXPORT_MAX_ROWS - rowCount),
         cursor,

apps/sim/lib/audit-logs/application/list-audit-logs.ts

@@ -3,6 +3,7 @@ import { auditLogOperations } from '@/lib/audit-logs/application/operations'
 import {
   type AuditLogFilterParams,
   buildFilterConditions,
+  buildOrgScopeCondition,
   decodeAuditLogCursor,
   getOrgWorkspaceIds,
   queryAuditLogs,
@@ -31,12 +32,12 @@ export const listAuditLogs = defineAuthorizedAuditLogUseCase({
     if (input.filters.workspaceId && !orgWorkspaceIds.includes(input.filters.workspaceId)) {
       throw new OrchestrationError('validation', 'workspaceId does not belong to your organization')
     }
-    const scope = {
+    const scopeCondition = buildOrgScopeCondition({
       organizationId: context.organizationId,
       orgWorkspaceIds,
       orgMemberIds: context.orgMemberIds,
       includeDeparted: input.includeDeparted,
-    }
+    })
     if (
       context.workspaceId &&
       input.filters.workspaceId &&
@@ -47,6 +48,6 @@ export const listAuditLogs = defineAuthorizedAuditLogUseCase({
       ...input.filters,
       ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
     })
-    return queryAuditLogs(scope, filterConditions, input.limit, input.cursor)
+    return queryAuditLogs([scopeCondition, ...filterConditions], input.limit, input.cursor)
   },
 })

apps/sim/lib/audit-logs/list-plans.integration.ts

@@ -4,11 +4,7 @@ import { deleteWorkspaceFixture, insertWorkspaceFixture } from '@sim/db/testing/
 import { generateId } from '@sim/utils/id'
 import { and, desc, eq, inArray, type SQL, sql } from 'drizzle-orm'
 import { afterAll, beforeAll, describe, expect, it } from 'vitest'
-import {
-  type AuditLogListScope,
-  buildOrgScopeCondition,
-  queryAuditLogs,
-} from '@/lib/audit-logs/query'
+import { buildOrgScopeCondition, type OrgScopeParams, queryAuditLogs } from '@/lib/audit-logs/query'
 
 const organizationId = generateId()
 const otherOrganizationId = generateId()
@@ -68,7 +64,7 @@ const fixtureIds = rows.map((row) => row.id)
 const auditLogTimestamp = sql`date_trunc('milliseconds', ${auditLog.createdAt})`
 
 /** The listing's contract as one predicate: every row in scope, newest first. */
-async function referenceOrder(scope: AuditLogListScope, filters: SQL<unknown>[]) {
+async function referenceOrder(scope: OrgScopeParams, filters: SQL<unknown>[]) {
   const matching = await db
     .select({ id: auditLog.id })
     .from(auditLog)
@@ -78,15 +74,20 @@ async function referenceOrder(scope: AuditLogListScope, filters: SQL<unknown>[])
 }
 
 async function paginate(
-  scope: AuditLogListScope,
+  scope: OrgScopeParams,
   filters: SQL<unknown>[],
   limit: number,
   maxBytes?: number
 ) {
   const seen: string[] = []
   let cursor: string | undefined
   for (let page = 0; page <= rows.length; page++) {
-    const result = await queryAuditLogs(scope, filters, limit, cursor, maxBytes)
+    const result = await queryAuditLogs(
+      [buildOrgScopeCondition(scope), ...filters],
+      limit,
+      cursor,
+      maxBytes
+    )
     expect(result.truncated ?? false).toBe(false)
     seen.push(...result.data.map((row) => row.id))
     if (!result.nextCursor) return seen
@@ -146,7 +147,7 @@ describe('queryAuditLogs scope against real Postgres', () => {
   ])(
     'pages through exactly the single-predicate result (%o)',
     async ({ includeDeparted, action }) => {
-      const scope: AuditLogListScope = {
+      const scope: OrgScopeParams = {
         organizationId,
         orgWorkspaceIds: [workspaceA, workspaceB, workspaceA],
         orgMemberIds: [memberId],
@@ -164,7 +165,7 @@ describe('queryAuditLogs scope against real Postgres', () => {
   )
 
   it('reads only organization-level rows when the organization has no workspaces', async () => {
-    const scope: AuditLogListScope = {
+    const scope: OrgScopeParams = {
       organizationId,
       orgWorkspaceIds: [],
       orgMemberIds: [],
@@ -185,7 +186,7 @@ describe('queryAuditLogs scope against real Postgres', () => {
   })
 
   it('narrows to one workspace when the filter names it', async () => {
-    const scope: AuditLogListScope = {
+    const scope: OrgScopeParams = {
       organizationId,
       orgWorkspaceIds: [workspaceA, workspaceB],
       orgMemberIds: [memberId],

apps/sim/lib/audit-logs/list-scope.integration.ts

@@ -53,13 +53,12 @@ describe('Audit pagination and drain tenant coverage in PostgreSQL', () => {
     { limit: 1, maxBytes: undefined, boundary: 'row limit' },
     { limit: 10, maxBytes: 1000, boundary: 'byte limit' },
   ])(
-    'returns every organization event once across a $boundary within one PostgreSQL millisecond',
+    'returns every event once across a $boundary within one PostgreSQL millisecond',
     async ({ limit, maxBytes }) => {
       const seen: string[] = []
       let cursor: string | undefined
       for (let pageIndex = 0; pageIndex < ids.length + 1; pageIndex++) {
         const page = await runtime.query.queryAuditLogs(
-          { organizationId, orgWorkspaceIds: [], orgMemberIds: [], includeDeparted: true },
           [runtime.inArray(runtime.auditLog.id, ids)],
           limit,
           cursor,
@@ -73,7 +72,8 @@ describe('Audit pagination and drain tenant coverage in PostgreSQL', () => {
         if (!page.nextCursor) break
         cursor = page.nextCursor
       }
-      expect(seen).toEqual(ids.slice(0, 3).reverse())
+      expect(seen).toHaveLength(ids.length)
+      expect(new Set(seen)).toEqual(new Set(ids))
     }
   )