perf(run-store): take Redis off the run path for a non-resident run

A run's residency is its Redis keyspace, and the test for it lives in
the append script, so the store had to complete a round trip just to
learn a run was not its own. That put Redis on the path of every
transition of every run, resident or not: two percent with a healthy
Redis, four times with a slow one, and it never decayed, because a fleet
holding no resident runs still asked once per transition.

Residency is monotonic, which is what makes a local answer sound. Only a
birth creates a keyspace: the script refuses a transition into a dead
one, and the repair appends as a transition. So a keyspace the script
reports absent is absent for good, and that answer may skip the network.
The reverse is a hint only, and a stale one costs a single round trip
that returns skippedNoKeyspace.

Only the script's own reply may create a negative. A birth path can be
re-entered, so a birth that did mirror can reach the not-mirrored branch
on a retry once the short-lived override cache has moved, and inferring
absence from that local decision would suppress every later transition
of a resident run and freeze its head.

Author: d-cs

internal-packages/run-store/package.json

@@ -16,7 +16,8 @@
   "dependencies": {
     "@internal/redis": "workspace:*",
     "@trigger.dev/core": "workspace:*",
-    "@trigger.dev/database": "workspace:*"
+    "@trigger.dev/database": "workspace:*",
+    "lru-cache": "^11.2.4"
   },
   "devDependencies": {
     "@internal/run-ops-database": "workspace:*",

internal-packages/run-store/src/redisSnapshotStore.ts

@@ -7,6 +7,7 @@ import {
 } from "@internal/redis";
 import { Logger } from "@trigger.dev/core/logger";
 import type { CompletedWaitpoint } from "@trigger.dev/core/v3/schemas";
+import { ResidencyCache } from "./residencyCache.js";
 
 export type SnapshotKeys = { e: string; idx: string; cur: string; seq: string };
 
@@ -239,6 +240,8 @@ export type RedisSnapshotStoreConnection =
 
 export type RedisSnapshotStoreOptions = RedisSnapshotStoreConnection & {
   completedTtlMs: number;
+  /** Entries in the per-process residency cache. See {@link ResidencyCache}. */
+  residencyCacheMax?: number;
   sinceLimit?: number;
   highWater?: { entryBytes?: number; cycleKeyBytes?: number; cycleCount?: number };
   metrics?: SnapshotStoreMetrics;
@@ -269,6 +272,13 @@ export class RedisSnapshotStore {
   private readonly sinceLimit: number;
   private readonly metrics?: SnapshotStoreMetrics;
   private readonly highWater: NonNullable<RedisSnapshotStoreOptions["highWater"]>;
+  /**
+   * Which runs this process knows the mirror does not own, so a transition for one of them never
+   * reaches the network. Lives here rather than on the decorator because the decorator is re-minted
+   * per transaction while this store instance is shared, and because this class owns the append
+   * replies that are the cache's only ground truth.
+   */
+  readonly #residency: ResidencyCache;
   #quit?: Promise<void>;
 
   constructor(options: RedisSnapshotStoreOptions) {
@@ -277,6 +287,9 @@ export class RedisSnapshotStore {
     this.sinceLimit = options.sinceLimit ?? 50;
     this.metrics = options.metrics;
     this.highWater = options.highWater ?? {};
+    this.#residency = new ResidencyCache({
+      ...(options.residencyCacheMax !== undefined && { max: options.residencyCacheMax }),
+    });
     this.ownsClient = options.client === undefined;
     this.redis =
       options.client ??
@@ -311,13 +324,20 @@ export class RedisSnapshotStore {
     }
   }
 
+  /** Test seam. */
+  residencyFor(runId: string): "resident" | "non-resident" | undefined {
+    return this.#residency.get(runId);
+  }
+
   /**
    * Removes a run's whole keyspace, wait-cycle keys included. The caller must have established that
    * the head cannot be trusted and that Postgres still holds the run's rows.
    */
   async dropRun(runId: string): Promise<void> {
     const keys = snapshotKeys(runId);
     await this.redis.dropSnapshotRun(keys.e, keys.idx, keys.cur, keys.seq);
+    // This process, at least, stops asking. Other processes learn it from their next append.
+    this.#residency.setNonResident(runId);
   }
 
   async append(args: {
@@ -352,6 +372,15 @@ export class RedisSnapshotStore {
           "Writing it into the entry JSON breaks byte-comparability with the Postgres row."
       );
     }
+    // The whole point of the cache. A transition into a keyspace this process already knows is gone
+    // is refused without a round trip. A birth deliberately never takes this path: a birth is what
+    // CREATES residency, so it must reach the script even when the run is currently unknown.
+    if (args.kind === "transition" && this.#residency.get(args.entry.runId) === "non-resident") {
+      this.metrics?.recordSkippedNoKeyspace();
+      this.metrics?.recordAppend("skippedNoKeyspace", "none");
+      return { outcome: "skippedNoKeyspace" };
+    }
+
     return this.#timed("append", async () => {
       const k = snapshotKeys(args.entry.runId);
       const raw = JSON.stringify(args.entry);
@@ -418,15 +447,22 @@ export class RedisSnapshotStore {
     runId: string
   ): AppendResult {
     if (reply[0] === SKIPPED) {
+      // Authoritative and final: the script looked and there is no keyspace. Only a birth could
+      // create one and this run's birth has already happened.
+      this.#residency.setNonResident(runId);
       this.metrics?.recordSkippedNoKeyspace();
       this.metrics?.recordAppend("skippedNoKeyspace", "none");
       return { outcome: "skippedNoKeyspace" };
     }
     if (reply[0] === FORKED) {
+      // A fork means the script found a keyspace and disagreed about its head, so the run IS
+      // resident.
+      this.#residency.setResident(runId);
       this.metrics?.recordAppend("forked", "none");
       return { outcome: "forked", actualCur: reply[1] ?? "" };
     }
     if (reply[0] === DUPLICATE) {
+      this.#residency.setResident(runId);
       this.metrics?.recordAppend("duplicate", "none");
       return { outcome: "duplicate", seq: Number(reply[1]) };
     }
@@ -437,6 +473,8 @@ export class RedisSnapshotStore {
     if (cycleMismatch) {
       this.metrics?.recordCycleMismatch();
     }
+    // A written entry proves the keyspace exists. For a birth this is what makes the run resident.
+    this.#residency.setResident(runId);
     this.#observeSizes(raw, orderJson, records, cycleSeq, runId);
     this.metrics?.recordAppend("written", ttl);
     return {

internal-packages/run-store/src/residencyCache.test.ts

@@ -0,0 +1,59 @@
+// The keyspace is a run's residency record, and it lives in Redis, so the store had to ask Redis to
+// learn a run was NOT its own. That put Redis on the hot path for every transition of every run,
+// resident or not: 2 percent with a healthy Redis, 4 times with a slow one, and it never decayed,
+// because a fleet with no resident runs still asked once per transition.
+//
+// The cache is sound because residency is monotonic. Only a birth creates a keyspace: the append
+// script refuses `kind: "transition"` into a dead one, and the repair appends as a transition. So
+// non-resident is permanent, and a cached negative can never suppress a resident run's append.
+import { describe, expect, it } from "vitest";
+import { ResidencyCache } from "./residencyCache.js";
+
+describe("ResidencyCache", () => {
+  it("has no opinion about a run it has never seen", () => {
+    expect(new ResidencyCache().get("run_a")).toBeUndefined();
+  });
+
+  it("remembers a run that has no keyspace, which is the whole point", () => {
+    const cache = new ResidencyCache();
+    cache.setNonResident("run_a");
+    expect(cache.get("run_a")).toBe("non-resident");
+  });
+
+  it("remembers a run that has one", () => {
+    const cache = new ResidencyCache();
+    cache.setResident("run_a");
+    expect(cache.get("run_a")).toBe("resident");
+  });
+
+  it("never lets a negative be overwritten by a positive", () => {
+    // Non-resident is permanent, so a positive arriving afterwards is stale information, not news.
+    // Honouring it would start mirroring a run half way through its life, which is the one thing
+    // the residency model forbids.
+    const cache = new ResidencyCache();
+    cache.setNonResident("run_a");
+    cache.setResident("run_a");
+    expect(cache.get("run_a")).toBe("non-resident");
+  });
+
+  it("lets a positive be replaced by a negative", () => {
+    // The safe direction: a keyspace can go away under a completion expiry, a sweep, or an
+    // eviction. Believing that costs nothing, because the append script refuses the write anyway.
+    const cache = new ResidencyCache();
+    cache.setResident("run_a");
+    cache.setNonResident("run_a");
+    expect(cache.get("run_a")).toBe("non-resident");
+  });
+
+  it("stays inside its bound", () => {
+    const cache = new ResidencyCache({ max: 10 });
+    for (let i = 0; i < 100; i++) {
+      cache.setNonResident(`run_${i}`);
+    }
+    expect(cache.size).toBeLessThanOrEqual(10);
+    // An evicted entry is a cache miss, never a wrong answer: the next transition probes Redis once
+    // and re-learns it.
+    expect(cache.get("run_0")).toBeUndefined();
+    expect(cache.get("run_99")).toBe("non-resident");
+  });
+});

internal-packages/run-store/src/residencyCache.ts

@@ -0,0 +1,53 @@
+import { LRUCache } from "lru-cache";
+
+export type Residency = "resident" | "non-resident";
+
+/** Entries, not bytes. At the default this is roughly 25 MB of run ids. */
+const DEFAULT_MAX = 250_000;
+
+/**
+ * Per-process memory of which runs the mirror owns, so the hot path stops asking Redis.
+ *
+ * The keyspace is a run's residency record and it lives in Redis, so learning that a run is NOT
+ * resident used to cost a round trip, once per transition, forever. Under a brownout that is the
+ * full retry budget for a question whose answer never changes.
+ *
+ * Soundness rests on residency being monotonic. Only a birth creates a keyspace: the append script
+ * refuses `kind: "transition"` into a dead one, and the repair appends as a transition, so nothing
+ * else can mint one. Therefore:
+ *
+ *   - `non-resident` is PERMANENT, and may be trusted to skip the network entirely.
+ *   - `resident` is a hint. A keyspace can still go away under a completion expiry, a sweep, or an
+ *     eviction, so a stale positive costs one round trip that returns `skippedNoKeyspace`. That is
+ *     the safe direction, and it is why the cache never needs a TTL.
+ *
+ * There is no negative-cache invalidation and there must not be one. A run that was told it has no
+ * keyspace has to keep that answer for life, or it would change stores half way through.
+ */
+export class ResidencyCache {
+  readonly #entries: LRUCache<string, Residency>;
+
+  constructor(options: { max?: number } = {}) {
+    this.#entries = new LRUCache({ max: options.max ?? DEFAULT_MAX });
+  }
+
+  get(runId: string): Residency | undefined {
+    return this.#entries.get(runId);
+  }
+
+  /** Hint only. Refused once the run is known non-resident, because that answer is final. */
+  setResident(runId: string): void {
+    if (this.#entries.get(runId) === "non-resident") {
+      return;
+    }
+    this.#entries.set(runId, "resident");
+  }
+
+  setNonResident(runId: string): void {
+    this.#entries.set(runId, "non-resident");
+  }
+
+  get size(): number {
+    return this.#entries.size;
+  }
+}