feat(webapp): isolate the runs list ClickHouse read pool

Give the runs-list ClickHouse pool server-side query protection
(max_execution_time, thread and memory caps, a per-user concurrency
breaker, readonly) so one tenant expensive query cannot saturate the
shared read service, and cap the runs list created_at lower bound to a
bounded window so an unbounded filter cannot scan every partition.

Billing and bulk count reads move to the read pool, off the ingestion
writer. Count queries are never date-capped so billing keeps counting
runs of any age.

Author: ericallam

.server-changes/runs-list-read-isolation.md

@@ -0,0 +1,6 @@
+---
+area: webapp
+type: improvement
+---
+
+The runs list and the runs.list API are more resilient: a single expensive query can no longer slow the runs list down for everyone. The list now loads from a bounded recent time window, which keeps it fast at scale.

apps/webapp/app/env.server.ts

@@ -2233,6 +2233,22 @@ const EnvironmentSchema = z
       .enum(["log", "error", "warn", "info", "debug"])
       .default("info"),
     RUNS_LIST_CLICKHOUSE_COMPRESSION_REQUEST: z.string().default("1"),
+    RUNS_LIST_CLICKHOUSE_REQUEST_TIMEOUT_MS: z.coerce.number().int().default(30_000),
+    RUNS_LIST_CLICKHOUSE_MAX_EXECUTION_TIME: z.coerce.number().int().default(35),
+    RUNS_LIST_CLICKHOUSE_MAX_THREADS: z.coerce.number().int().optional(),
+    RUNS_LIST_CLICKHOUSE_MAX_MEMORY_USAGE: z.coerce.number().int().optional(),
+    RUNS_LIST_CLICKHOUSE_MAX_MEMORY_USAGE_FOR_USER: z.coerce.number().int().optional(),
+    RUNS_LIST_CLICKHOUSE_MAX_CONCURRENT_QUERIES_FOR_USER: z.coerce.number().int().optional(),
+    RUNS_LIST_CLICKHOUSE_READONLY: z.enum(["0", "1", "2"]).default("2"),
+    /**
+     * Hard cap on how far back the runs list / runs.list API `created_at` lower bound may reach,
+     * in milliseconds. The display list adds `created_at >= now - this` so an unbounded filter
+     * can't scan all partitions. `0` disables the cap. Does not apply to count queries.
+     */
+    RUNS_LIST_MAX_CREATED_AT_AGE_MS: z.coerce
+      .number()
+      .int()
+      .default(30 * 24 * 60 * 60 * 1000),
     /**
      * Dedicated ClickHouse service for queue metrics: the ingestion consumer's inserts and every
      * queue-metrics read (dashboards, queue pages, run inspector, health report) go through it, so

apps/webapp/app/presenters/v3/CreateBulkActionPresenter.server.ts

@@ -26,7 +26,7 @@ export class CreateBulkActionPresenter extends BasePresenter {
 
     const clickhouse = await clickhouseFactory.getClickhouseForOrganization(
       organizationId,
-      "standard"
+      "runsList"
     );
     const runsRepository = new RunsRepository({
       clickhouse,

apps/webapp/app/presenters/v3/NextRunListPresenter.server.ts

@@ -256,6 +256,7 @@ export class NextRunListPresenter {
     const runsRepository = new RunsRepository({
       clickhouse: this.clickhouse,
       prisma: this.replica as PrismaClient,
+      maxCreatedAtAgeMs: env.RUNS_LIST_MAX_CREATED_AT_AGE_MS,
       readThrough: this.readThroughDeps
         ? {
             newClient: this.readThroughDeps.newClient ?? this.replica,

apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts

@@ -1,4 +1,4 @@
-import { ClickHouse } from "@internal/clickhouse";
+import { ClickHouse, type ClickHouseSettings } from "@internal/clickhouse";
 import { createHash } from "crypto";
 import { ClickhouseEventRepository } from "~/v3/eventRepository/clickhouseEventRepository.server";
 import { env } from "~/env.server";
@@ -292,6 +292,40 @@ function initializeRealtimeClickhouseClient(): ClickHouse {
   });
 }
 
+/**
+ * Server-side query protection for the runs-list read pool. Safe as client-level settings ONLY
+ * because this pool is read-only (no inserts); a client-level `max_execution_time` on a mixed
+ * read+write pool would also kill slow inserts. `readonly=2` enforces read-only while still
+ * allowing these settings to apply (`readonly=1` rejects them). `max_concurrent_queries_for_user`
+ * is a per-ClickHouse-user (`default`) fail-fast circuit breaker, not per-tenant isolation.
+ */
+function getRunsListClickhouseSettings(): ClickHouseSettings {
+  const settings: ClickHouseSettings = {
+    max_execution_time: env.RUNS_LIST_CLICKHOUSE_MAX_EXECUTION_TIME,
+    timeout_before_checking_execution_speed: 0,
+  };
+
+  if (env.RUNS_LIST_CLICKHOUSE_READONLY !== "0") {
+    settings.readonly = env.RUNS_LIST_CLICKHOUSE_READONLY;
+  }
+  if (env.RUNS_LIST_CLICKHOUSE_MAX_THREADS !== undefined) {
+    settings.max_threads = env.RUNS_LIST_CLICKHOUSE_MAX_THREADS;
+  }
+  if (env.RUNS_LIST_CLICKHOUSE_MAX_MEMORY_USAGE !== undefined) {
+    settings.max_memory_usage = env.RUNS_LIST_CLICKHOUSE_MAX_MEMORY_USAGE.toString();
+  }
+  if (env.RUNS_LIST_CLICKHOUSE_MAX_MEMORY_USAGE_FOR_USER !== undefined) {
+    settings.max_memory_usage_for_user =
+      env.RUNS_LIST_CLICKHOUSE_MAX_MEMORY_USAGE_FOR_USER.toString();
+  }
+  if (env.RUNS_LIST_CLICKHOUSE_MAX_CONCURRENT_QUERIES_FOR_USER !== undefined) {
+    settings.max_concurrent_queries_for_user =
+      env.RUNS_LIST_CLICKHOUSE_MAX_CONCURRENT_QUERIES_FOR_USER;
+  }
+
+  return settings;
+}
+
 /** Runs list reads — dashboard + API (`RUNS_LIST_CLICKHOUSE_URL`);
  *  falls back to the default client if unset. */
 const defaultRunsListClickhouseClient = singleton(
@@ -319,6 +353,8 @@ function initializeRunsListClickhouseClient(): ClickHouse {
       request: env.RUNS_LIST_CLICKHOUSE_COMPRESSION_REQUEST === "1",
     },
     maxOpenConnections: env.RUNS_LIST_CLICKHOUSE_MAX_OPEN_CONNECTIONS,
+    requestTimeoutMs: env.RUNS_LIST_CLICKHOUSE_REQUEST_TIMEOUT_MS,
+    clickhouseSettings: getRunsListClickhouseSettings(),
   });
 }
 
@@ -550,10 +586,25 @@ function buildOrgClickhouseClient(url: string, clientType: ClientType): ClickHou
         },
         maxOpenConnections: env.REALTIME_BACKEND_NATIVE_CLICKHOUSE_MAX_OPEN_CONNECTIONS,
       });
+    case "runsList":
+      return new ClickHouse({
+        url: parsed.toString(),
+        name,
+        keepAlive: {
+          enabled: env.RUNS_LIST_CLICKHOUSE_KEEP_ALIVE_ENABLED === "1",
+          idleSocketTtl: env.RUNS_LIST_CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS,
+        },
+        logLevel: env.RUNS_LIST_CLICKHOUSE_LOG_LEVEL,
+        compression: {
+          request: env.RUNS_LIST_CLICKHOUSE_COMPRESSION_REQUEST === "1",
+        },
+        maxOpenConnections: env.RUNS_LIST_CLICKHOUSE_MAX_OPEN_CONNECTIONS,
+        requestTimeoutMs: env.RUNS_LIST_CLICKHOUSE_REQUEST_TIMEOUT_MS,
+        clickhouseSettings: getRunsListClickhouseSettings(),
+      });
     case "standard":
     case "query":
     case "admin":
-    case "runsList":
       return new ClickHouse({
         url: parsed.toString(),
         name,

apps/webapp/app/services/runsRepository/clickhouseRunsRepository.server.ts

@@ -127,7 +127,8 @@ export class ClickHouseRunsRepository implements IRunsRepository {
         options,
         this.options.prisma,
         this.options.runStore ?? runStore
-      )
+      ),
+      this.options.maxCreatedAtAgeMs
     );
 
     const forward = options.page.direction === "forward" || !options.page.direction;
@@ -335,6 +336,12 @@ export class ClickHouseRunsRepository implements IRunsRepository {
     };
   }
 
+  /**
+   * Deliberately NOT passed `maxCreatedAtAgeMs`: the only callers are billing limit checks and
+   * bulk actions, which must count runs of any age (a queued/delayed run older than the window
+   * still counts). Clamping here would undercount. Runaway counts are bounded instead by the
+   * read pool's server-side `max_execution_time`, not by a date cap.
+   */
   async countRuns(options: RunListInputOptions) {
     const queryBuilder = this.options.clickhouse.taskRuns.countQueryBuilder();
     applyRunFiltersToQueryBuilder(
@@ -411,9 +418,15 @@ export class ClickHouseRunsRepository implements IRunsRepository {
   }
 }
 
+/**
+ * Builds the shared WHERE clauses for the runs list. `maxCreatedAtAgeMs` (when > 0) floors the
+ * `created_at` lower bound to `now - maxCreatedAtAgeMs`; it is ANDed with any period/from filter,
+ * so the tighter bound wins, and it keeps an unbounded filter from scanning every partition.
+ */
 function applyRunFiltersToQueryBuilder<T>(
   queryBuilder: ClickhouseQueryBuilder<T>,
-  options: FilterRunsOptions
+  options: FilterRunsOptions,
+  maxCreatedAtAgeMs?: number
 ) {
   queryBuilder
     .where("organization_id = {organizationId: String}", {
@@ -426,6 +439,12 @@ function applyRunFiltersToQueryBuilder<T>(
       environmentId: options.environmentId,
     });
 
+  if (typeof maxCreatedAtAgeMs === "number" && maxCreatedAtAgeMs > 0) {
+    queryBuilder.where("created_at >= fromUnixTimestamp64Milli({createdAtFloor: Int64})", {
+      createdAtFloor: Date.now() - maxCreatedAtAgeMs,
+    });
+  }
+
   if (options.tasks && options.tasks.length > 0) {
     queryBuilder.where("task_identifier IN {tasks: Array(String)}", { tasks: options.tasks });
   }

apps/webapp/app/services/runsRepository/runsRepository.server.ts

@@ -33,6 +33,14 @@ export type RunsRepositoryOptions = {
     // Resolved boot constant; when false the split branch is never entered.
     splitEnabled?: boolean;
   };
+
+  /**
+   * Hard cap on how far back the run-listing `created_at` lower bound may reach, in ms. When set
+   * and > 0, the list queries add `created_at >= now - maxCreatedAtAgeMs` so an unbounded filter
+   * can't scan every partition. Omitted / 0 => no cap. Applies to `listRuns`/`listRunIds` only,
+   * never to `countRuns` (billing and bulk counts must count runs of any age).
+   */
+  maxCreatedAtAgeMs?: number;
 };
 
 const RunStatus = z.enum(Object.values(TaskRunStatus) as [TaskRunStatus, ...TaskRunStatus[]]);

apps/webapp/app/v3/services/billingLimit/billingLimitQueuedRuns.server.ts

@@ -39,7 +39,7 @@ export async function getBillableEnvironmentsForBillingLimit(
 export async function createBillingLimitRunsRepository(organizationId: string) {
   const clickhouse = await clickhouseFactory.getClickhouseForOrganization(
     organizationId,
-    "standard"
+    "runsList"
   );
 
   return new RunsRepository({
@@ -95,7 +95,7 @@ export async function countBillableQueuedRunsForOrganization(
 ): Promise<number> {
   const client =
     clickhouse ??
-    (await clickhouseFactory.getClickhouseForOrganization(organizationId, "standard"));
+    (await clickhouseFactory.getClickhouseForOrganization(organizationId, "runsList"));
 
   const queryBuilder = client.taskRuns.countQueryBuilder({
     settings: { max_execution_time: BILLING_LIMIT_QUEUED_COUNT_MAX_EXECUTION_S },

apps/webapp/app/v3/services/bulk/BulkActionV2.server.ts

@@ -115,7 +115,7 @@ export class BulkActionService extends BaseService {
     // Count the runs that will be affected by the bulk action
     const clickhouse = await clickhouseFactory.getClickhouseForOrganization(
       organizationId,
-      "standard"
+      "runsList"
     );
     const runsRepository = new RunsRepository({
       clickhouse,
@@ -275,7 +275,7 @@ export class BulkActionService extends BaseService {
 
     const clickhouse = await clickhouseFactory.getClickhouseForOrganization(
       group.project.organizationId,
-      "standard"
+      "runsList"
     );
     const runsRepository = new RunsRepository({
       clickhouse,

apps/webapp/test/runsListClickhouseSettings.test.ts

@@ -0,0 +1,66 @@
+import { ClickHouse } from "@internal/clickhouse";
+import { clickhouseTest } from "@internal/testcontainers";
+import { describe, expect, vi } from "vitest";
+import { z } from "zod";
+
+vi.setConfig({ testTimeout: 60_000 });
+
+describe("runs-list ClickHouse protection settings", () => {
+  clickhouseTest(
+    "server-side max_execution_time kills a slow read, and readonly=2 does not block the caps",
+    async ({ clickhouseContainer }) => {
+      const clickhouse = new ClickHouse({
+        url: clickhouseContainer.getConnectionUrl(),
+        name: "runs-list-settings-test",
+        requestTimeoutMs: 30_000,
+        clickhouseSettings: {
+          max_execution_time: 1,
+          timeout_before_checking_execution_speed: 0,
+          max_threads: 2,
+          readonly: "2",
+        },
+      });
+
+      const slow = clickhouse.reader.query({
+        name: "slow-read",
+        query: "SELECT sum(number) AS total FROM numbers(1000000000000)",
+        schema: z.object({ total: z.number() }),
+      });
+      const [slowError] = await slow({});
+
+      expect(slowError).not.toBeNull();
+      expect(slowError?.message.toLowerCase()).toMatch(/timeout|exceeded/);
+
+      const fast = clickhouse.reader.query({
+        name: "fast-read",
+        query: "SELECT 1 AS one",
+        schema: z.object({ one: z.number() }),
+      });
+      const [fastError, rows] = await fast({});
+
+      expect(fastError).toBeNull();
+      expect(rows).toEqual([{ one: 1 }]);
+    }
+  );
+
+  clickhouseTest(
+    "readonly=2 rejects writes while permitting reads",
+    async ({ clickhouseContainer }) => {
+      const clickhouse = new ClickHouse({
+        url: clickhouseContainer.getConnectionUrl(),
+        name: "runs-list-readonly-test",
+        clickhouseSettings: { readonly: "2" },
+      });
+
+      const write = clickhouse.reader.query({
+        name: "write-under-readonly",
+        query: "CREATE TABLE trigger_dev.runs_list_readonly_probe (id UInt8) ENGINE = Memory",
+        schema: z.object({}),
+      });
+      const [writeError] = await write({});
+
+      expect(writeError).not.toBeNull();
+      expect(writeError?.message.toLowerCase()).toMatch(/readonly|read-only|read only/);
+    }
+  );
+});