fix(dashboard-agent): tell a queue that can't be read apart from one that isn't there

The queue's live row read collapsed every non-ok response into "no live row",
so a 401, a 429 or a 5xx reached the model as exists:false — the queue does not
exist. Only a 404 is evidence of absence now; anything else reports
exists:"unknown" with the status, and the prompt says unknown is never missing.

Author: kathiekiwi

internal-packages/dashboard-agent/src/__snapshots__/prompt-prefix.test.ts.snap

@@ -26,6 +26,7 @@ import {
   isEnvUnavailable,
   NO_AUTH,
   type DashboardAgentApiClient,
+  type EnvFetchResult,
   type EnvUnavailable,
 } from "./tool-api-client";
 import type { DashboardAgentToolContext } from "./tool-context";
@@ -105,18 +106,66 @@ export function consumerTasksForQueue(workers: unknown, queueName: string): stri
   return [...slugs].sort();
 }
 
+/**
+ * What the queue's live row read came back with. "The route said 404" and "the read never
+ * landed" are different answers: only the first is evidence about the queue, and collapsing
+ * them turns an expired token or a 5xx into "that queue doesn't exist".
+ */
+export type QueueLiveRead =
+  | { kind: "row"; row: Record<string, unknown> }
+  | { kind: "missing" }
+  | { kind: "unknown"; status?: number };
+
+/** Reads the live-row response into those three states. */
+export function readQueueLiveState(result: EnvFetchResult | null): QueueLiveRead {
+  // No current environment, or a read that never landed: nothing is known either way.
+  if (!result) return { kind: "unknown" };
+  if (isEnvUnavailable(result)) {
+    return {
+      kind: "unknown",
+      status: result.envUnavailable === "unknown" ? result.status : undefined,
+    };
+  }
+  if (!result.ok) {
+    // A request that never landed carries no status, and says nothing about the queue.
+    if (!("status" in result)) return { kind: "unknown" };
+    return result.status === 404 ? { kind: "missing" } : { kind: "unknown", status: result.status };
+  }
+  const row = (result.data as { data?: Record<string, unknown> })?.data ?? result.data;
+  if (!row || typeof row !== "object") return { kind: "unknown" };
+  return { kind: "row", row: row as Record<string, unknown> };
+}
+
+/**
+ * The better of two live reads of the same queue name under either kind. A row wins; failing
+ * that a failed read wins over a 404, since one 404 with the other read broken is not proof.
+ */
+export function pickQueueLiveState(first: QueueLiveRead, second: QueueLiveRead): QueueLiveRead {
+  if (first.kind === "row") return first;
+  if (second.kind === "row") return second;
+  if (first.kind === "unknown") return first;
+  return second;
+}
+
 /**
  * Metrics plus the queue's live row. `paused` is the part the model must lead with: a queue
  * someone stopped explains its own emptiness, and every metric below it is a consequence
- * rather than a finding.
+ * rather than a finding. When the read failed, `exists` is `"unknown"` rather than `false`,
+ * because an unreachable queue is not an absent one.
  */
-function withLiveState(
-  metrics: unknown,
-  queueType: "task" | "custom",
-  state: Record<string, unknown> | undefined
-) {
-  const row = (state as { data?: Record<string, unknown> })?.data ?? state;
-  if (!row) return { ...(metrics as object), queueType, exists: false };
+export function withLiveState(metrics: unknown, queueType: "task" | "custom", live: QueueLiveRead) {
+  if (live.kind === "missing") return { ...(metrics as object), queueType, exists: false };
+  if (live.kind === "unknown") {
+    return {
+      ...(metrics as object),
+      queueType,
+      exists: "unknown" as const,
+      liveStateError: live.status
+        ? `Couldn't read the queue's live row (status ${live.status}).`
+        : "Couldn't read the queue's live row.",
+    };
+  }
+  const { row } = live;
   return {
     ...(metrics as object),
     queueType: (row.type as string) ?? queueType,
@@ -422,18 +471,12 @@ export function buildApiTools(args: {
           const result = await envApiGet(
             `/api/v1/queues/${encodeURIComponent(queue)}?type=${kind}`
           );
-          return !isEnvUnavailable(result) && result.ok
-            ? (result.data as Record<string, unknown>)
-            : undefined;
+          return readQueueLiveState(result);
         };
 
         // Only a custom queue needs this read: a task queue's consumer is the task it is
         // named after, while a custom queue's name says nothing about who writes to it.
-        const answer = async (
-          metrics: unknown,
-          kind: "task" | "custom",
-          state: Record<string, unknown> | undefined
-        ) => {
+        const answer = async (metrics: unknown, kind: "task" | "custom", state: QueueLiveRead) => {
           const base = withLiveState(metrics, kind, state);
           if (base.queueType !== "custom" || !hasAuth || !projectRef || !environmentName) {
             return base;
@@ -463,7 +506,9 @@ export function buildApiTools(args: {
           // Neither kind has metrics, so the live row is the only thing that can tell them
           // apart: a paused or empty queue that exists, against a name that doesn't.
           const kind = type ?? "task";
-          const state = (await live(kind)) ?? (await live(otherKind));
+          const primary = await live(kind);
+          const state =
+            primary.kind === "row" ? primary : pickQueueLiveState(primary, await live(otherKind));
           return await answer(first.data, kind, state);
         }
         const kind = type ?? "task";

internal-packages/dashboard-agent/src/tool-api.ts

@@ -1,5 +1,13 @@
-import { describe, expect, it } from "vitest";
-import { consumerTasksForQueue, queueMetricsAreEmpty } from "./tool-api";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import {
+  buildApiTools,
+  consumerTasksForQueue,
+  pickQueueLiveState,
+  queueMetricsAreEmpty,
+  readQueueLiveState,
+  withLiveState,
+} from "./tool-api";
+import { createApiClient } from "./tool-api-client";
 
 /**
  * The metrics route answers an unknown queue with zeroes rather than a 404, so asking for
@@ -68,3 +76,116 @@ describe("consumerTasksForQueue", () => {
     expect(consumerTasksForQueue({ worker: { tasks: [{}] } }, "email-sends")).toEqual([]);
   });
 });
+
+/**
+ * A queue nobody can read is not a queue that isn't there. Only the route answering 404 is
+ * evidence of absence; a 401, a 429 or a 5xx is evidence of nothing, and reporting one as
+ * `exists: false` tells the model a queue holding thousands of runs was deleted.
+ */
+describe("the queue's live row has three answers, not two", () => {
+  const metrics = { peakQueued: 4800, startedCount: 12 };
+
+  it("reads a row, a 404 and a failed read apart", () => {
+    expect(readQueueLiveState({ ok: true, data: { paused: true } })).toEqual({
+      kind: "row",
+      row: { paused: true },
+    });
+    expect(readQueueLiveState({ ok: false, status: 404 })).toEqual({ kind: "missing" });
+    for (const status of [401, 403, 429, 500, 503]) {
+      expect(readQueueLiveState({ ok: false, status })).toEqual({ kind: "unknown", status });
+    }
+    // No current environment: nothing was asked, so nothing is known.
+    expect(readQueueLiveState(null)).toEqual({ kind: "unknown" });
+  });
+
+  it("says unknown rather than absent when the read failed", () => {
+    expect(withLiveState(metrics, "custom", { kind: "unknown", status: 503 })).toMatchObject({
+      exists: "unknown",
+      liveStateError: "Couldn't read the queue's live row (status 503).",
+    });
+    expect(withLiveState(metrics, "custom", { kind: "missing" })).toMatchObject({ exists: false });
+    expect(
+      withLiveState(metrics, "custom", { kind: "row", row: { paused: true, queued: 9 } })
+    ).toMatchObject({ exists: true, paused: true, queuedNow: 9 });
+  });
+
+  it("prefers a row, then a failed read, over a single 404", () => {
+    const row = { kind: "row", row: { paused: false } } as const;
+    const missing = { kind: "missing" } as const;
+    const unknown = { kind: "unknown", status: 500 } as const;
+
+    expect(pickQueueLiveState(missing, row)).toEqual(row);
+    expect(pickQueueLiveState(unknown, row)).toEqual(row);
+    // One kind 404s while the other read broke: that is not proof the name is free.
+    expect(pickQueueLiveState(missing, unknown)).toEqual(unknown);
+    expect(pickQueueLiveState(unknown, missing)).toEqual(unknown);
+    expect(pickQueueLiveState(missing, missing)).toEqual(missing);
+  });
+});
+
+/** The same three cases through `get_queue`, since the tool output is what the model reads. */
+describe("get_queue reports the live read it actually got", () => {
+  const ORIGIN = "https://api.example.com";
+
+  function stubFetch(liveResponse: () => Response) {
+    vi.stubGlobal(
+      "fetch",
+      vi.fn(async (input: any) => {
+        const url = typeof input === "string" ? input : input.url;
+        if (url.endsWith("/jwt")) {
+          return new Response(JSON.stringify({ token: "env-jwt" }), { status: 200 });
+        }
+        if (url.includes("/metrics")) {
+          return new Response(JSON.stringify({ peakQueued: 4800, startedCount: 12 }), {
+            status: 200,
+          });
+        }
+        return liveResponse();
+      })
+    );
+  }
+
+  function getQueue() {
+    const ctx = {
+      userActorToken: "uat",
+      apiOrigin: ORIGIN,
+      projectRef: "proj_ref",
+      environmentName: "dev",
+    };
+    const tools = buildApiTools({
+      ctx,
+      client: createApiClient(ctx),
+      renderInvestigations: (() => []) as any,
+    });
+    return (input: any) => (tools.get_queue as any).execute(input, {} as any);
+  }
+
+  afterEach(() => vi.unstubAllGlobals());
+
+  it("reports a healthy row as the queue that exists", async () => {
+    stubFetch(
+      () =>
+        new Response(JSON.stringify({ type: "custom", paused: true, queued: 31 }), { status: 200 })
+    );
+    await expect(getQueue()({ queue: "email-sends", type: "custom" })).resolves.toMatchObject({
+      exists: true,
+      paused: true,
+      queuedNow: 31,
+    });
+  });
+
+  it("reports a 404 as the queue that isn't there", async () => {
+    stubFetch(() => new Response("", { status: 404 }));
+    await expect(getQueue()({ queue: "email-sends", type: "custom" })).resolves.toMatchObject({
+      exists: false,
+    });
+  });
+
+  it("reports a failed read as unknown, never as absent", async () => {
+    stubFetch(() => new Response("", { status: 503 }));
+    const answer = await getQueue()({ queue: "email-sends", type: "custom" });
+    expect(answer).toMatchObject({ exists: "unknown" });
+    expect(answer.exists).not.toBe(false);
+    expect(answer.liveStateError).toContain("503");
+  });
+});

internal-packages/dashboard-agent/src/tool-queue.test.ts

@@ -190,7 +190,7 @@ export const getReportSchema = tool({
 
 export const getQueueSchema = tool({
   description:
-    "Get one queue's metrics over a window: wait latency (p50/p95), peak depth, how many runs started (throughput), and how often the queue was throttled by its concurrency limit. Use this for 'how deep is the X queue', 'is X backed up', or 'why are runs waiting'. The answer also carries the queue's live row: `paused`, `queuedNow`, `runningNow`, `concurrencyLimit`, and `exists: false` when no queue of that name is there at all. For a custom queue it also carries `consumerTasks`: the deployed tasks whose queue config names this queue.",
+    "Get one queue's metrics over a window: wait latency (p50/p95), peak depth, how many runs started (throughput), and how often the queue was throttled by its concurrency limit. Use this for 'how deep is the X queue', 'is X backed up', or 'why are runs waiting'. The answer also carries the queue's live row: `paused`, `queuedNow`, `runningNow`, `concurrencyLimit`, and `exists: false` when no queue of that name is there at all. When that read fails rather than answers, `exists` is `\"unknown\"` with a `liveStateError`: the queue's state is unknown, not missing. For a custom queue it also carries `consumerTasks`: the deployed tasks whose queue config names this queue.",
   inputSchema: z.object({
     queue: z
       .string()
@@ -483,7 +483,7 @@ You have read-only tools that act as the user against their own account:
 - ask_support: ask the Trigger.dev support assistant about how Trigger.dev works (docs, concepts, features, configuration, how-tos).
 - render_view: render a structured view in the panel from the block catalog. The catalog has the "diagnosis" block (a failure card for a single run), the "chart" block (a line/bar chart of run_query results), the "actions" block (a row of 1-3 buttons offering next steps — a watch intent opens the watch card pre-filled, an ask intent sends the labelled question as the user's next message), and the "investigation" block (a live card for a hypothesis-driven investigation).
 - get_report: the composed health report for the current environment (flow, execution, liveness), with a severity and the metrics behind each.
-- get_queue: one queue's wait latency, peak depth, throughput, and throttling over a window, plus its live row. Lead with paused when it is true: a paused queue explains its own emptiness, and every metric under it is a consequence, not a finding — say it is paused, and only then the numbers. queuedNow is what is waiting right now, which a window of metrics cannot show; exists:false is the only thing that means the queue isn't there, never zeroed metrics. A custom queue's name is not a task id, so no task being named after it is not evidence about it — never conclude from list_tasks or a deployment that a queue is unconsumed, undeployed, deleted, or renamed. consumerTasks is the answer to "who feeds this queue": empty means nothing deployed writes to it, and absent means you did not ask a custom queue.
+- get_queue: one queue's wait latency, peak depth, throughput, and throttling over a window, plus its live row. Lead with paused when it is true: a paused queue explains its own emptiness, so say it is paused and only then the numbers. queuedNow is what is waiting right now, which a window of metrics cannot show; exists:false is the only thing that means the queue isn't there, never zeroed metrics, and exists:"unknown" means the live read failed — unknown, never missing. A custom queue's name is not a task id, so no task being named after it is not evidence about it — never conclude from list_tasks or a deployment that it is unconsumed, deleted, or renamed. consumerTasks is the answer to "who feeds this queue": empty means nothing deployed writes to it, and absent means you did not ask a custom queue.
 - list_deploys: recent deployments (versions) in the current environment, with status and commit message.
 - get_deploy: one deployment's detail, or the current promoted one when you omit the version.
 - correlate_version: the version, commit, and pull request a specific run actually ran.