fix(logs): mark withheld trace span content so logs query stops reporting it as missing

When secret projection omits a span's content field or falls back to
structural-only spans, it now sets `contentWithheld: true` on the span. The
v2 log contract documents the marker, and `logs query` reports a withheld
note (and a `contentWithheld` count) instead of "path not found on span",
and keeps such runs under --where rather than silently filtering them out.

Author: waleedlatif1

apps/docs/openapi-v2-logs.json

@@ -1448,6 +1448,11 @@
             "type": "number",
             "description": "Offset from the root span in milliseconds."
           },
+          "contentWithheld": {
+            "type": "boolean",
+            "const": true,
+            "description": "Present when the log projection withheld some or all of this span's content (input, output, thinking, error message) because it could not be proven safe to display. Status, timing, and usage remain. A field missing from a span carrying this marker may exist on the run but was not returned."
+          },
           "toolCalls": {
             "type": "array",
             "items": {

apps/sim/lib/api/contracts/logs.ts

@@ -218,6 +218,7 @@ export type LogTraceSpan = {
   tokens?: number | { total?: number; input?: number; output?: number }
   cost?: { total?: number; input?: number; output?: number; toolCost?: number }
   relativeStartMs?: number
+  contentWithheld?: true
   toolCalls?: Array<z.output<typeof toolCallSchema>>
   children?: LogTraceSpan[]
 }
@@ -273,6 +274,12 @@ export const traceSpanSchema: z.ZodType<LogTraceSpan> = z
           .number()
           .describe('Offset from the root span in milliseconds.')
           .optional(),
+        contentWithheld: z
+          .literal(true)
+          .describe(
+            "Present when the log projection withheld some or all of this span's content (input, output, thinking, error message) because it could not be proven safe to display. Status, timing, and usage remain. A field missing from a span carrying this marker may exist on the run but was not returned."
+          )
+          .optional(),
         toolCalls: z.array(toolCallSchema).describe('Tool calls recorded by the span.').optional(),
         children: z.array(traceSpanSchema).describe('Nested child trace spans.').optional(),
       })

apps/sim/lib/logs/execution/trace-secret-projection.test.ts

@@ -502,6 +502,57 @@ describe('projectTraceSpansForSecrets', () => {
     expect(span.children?.[0]).not.toHaveProperty('output')
   })
 
+  it('marks every span of a structural-only fallback as content-withheld', async () => {
+    const source = [
+      createSpan({
+        output: { secret: 'raw' },
+        children: [createSpan({ id: 'child-1', output: { secret: 'raw' } })],
+      }),
+    ]
+
+    const incomplete = await projectTraceSpansForSecrets(source, {
+      registry: createRegistry([], false),
+      store: STORE,
+    })
+    const failedMatcher = await projectTraceSpansForSecrets(source, {
+      registry: createRegistry([
+        { plaintext: 's'.repeat(64 * 1024 + 1), replacement: '{{TOO_LARGE}}' },
+      ]),
+      store: STORE,
+    })
+    const failedInvariant = await enforceTraceSpanSecretInvariant(
+      [createSpan({ output: { apiKey: '[REDACTED]' } })],
+      {
+        registry: createRegistry([{ plaintext: 'REDACTED', replacement: '{{X}}' }]),
+        store: STORE,
+      }
+    )
+
+    for (const [span] of [incomplete, failedMatcher]) {
+      expect(span.contentWithheld).toBe(true)
+      expect(span.children?.[0].contentWithheld).toBe(true)
+    }
+    expect(failedInvariant[0].contentWithheld).toBe(true)
+  })
+
+  it('marks a span whose content field was omitted and leaves fully projected spans unmarked', async () => {
+    const [omitted, intact] = await projectTraceSpansForSecrets(
+      [
+        createSpan({ input: { safe: 'keep' }, output: { secretval: 1, '{{TOKEN}}': 2 } }),
+        createSpan({ id: 'span-2', output: { value: 'secretval' } }),
+      ],
+      {
+        registry: createRegistry([{ plaintext: 'secretval', replacement: '{{TOKEN}}' }]),
+        store: STORE,
+      }
+    )
+
+    expect(omitted.input).toEqual({ safe: 'keep' })
+    expect(omitted.contentWithheld).toBe(true)
+    expect(intact.output).toEqual({ value: '{{TOKEN}}' })
+    expect(intact).not.toHaveProperty('contentWithheld')
+  })
+
   it('fails closed when a later log transform reintroduces an active literal', async () => {
     const source = [createSpan({ output: { apiKey: '[REDACTED]' } })]
 

apps/sim/lib/logs/execution/trace-secret-projection.ts

@@ -965,7 +965,7 @@ async function sanitizeTraceSpan(
     if (value === undefined) continue
     const sanitized = await sanitizeContentField(value, context)
     if (sanitized === OMIT) {
-      projected = omit(projected, [key]) as TraceSpan
+      projected = { ...(omit(projected, [key]) as TraceSpan), contentWithheld: true }
     } else {
       projected[key] = sanitized as never
     }
@@ -1106,6 +1106,7 @@ function structuralOnlySpan(
 
   return {
     ...structural,
+    contentWithheld: true,
     ...(modelToolCalls
       ? {
           modelToolCalls: projectBoundedArray(modelToolCalls, (call) =>

apps/sim/lib/logs/types.ts

@@ -308,6 +308,14 @@ export interface TraceSpan {
    * one read, not the run.
    */
   childTraceAccess?: 'granted' | 'disabled' | 'missing' | 'truncated'
+  /**
+   * Set by secret projection when it omitted one of this span's own content fields
+   * (`input`, `output`, `thinking`, `errorMessage`) it could not prove safe, or reduced
+   * the span to structure only (which also strips tool-call and provider-segment
+   * content). Status, timing, and usage remain. Without it a withheld field is
+   * indistinguishable from one the block never produced.
+   */
+  contentWithheld?: true
   model?: string
   cost?: {
     input?: number

apps/sim/lib/mothership/agent-cli/engines.test.ts

@@ -101,6 +101,61 @@ describe('logs query', () => {
   })
 })
 
+const WITHHELD_RUNS = {
+  [RUNS_PATH]: {
+    data: [
+      { runId: 'run-1', status: 'completed' },
+      { runId: 'run-2', status: 'completed' },
+    ],
+  },
+  '/api/v2/logs/run-1': {
+    data: {
+      traceSpans: [{ name: 'Count rows', status: 'success', output: { result: { count: 6 } } }],
+    },
+  },
+  // The log projection withheld this span's content; its structure survived.
+  '/api/v2/logs/run-2': {
+    data: {
+      traceSpans: [{ name: 'Count rows', status: 'success', duration: 12, contentWithheld: true }],
+    },
+  },
+}
+
+async function logsQueryWithheld(flags: Record<string, string>) {
+  const result = await runEngine('logs query', ['wf-1'], runtimeWith(WITHHELD_RUNS), {
+    block: 'Count rows',
+    ...flags,
+  })
+  expect(result.exitCode).toBe(0)
+  return JSON.parse(result.stdout)
+}
+
+describe('logs query on a span whose content was withheld', () => {
+  it('reports withheld content instead of a missing path, and counts it', async () => {
+    const report = await logsQueryWithheld({ field: 'result.count' })
+    expect(report.rows[0]).toMatchObject({ runId: 'run-1', value: 6 })
+    expect(report.rows[1]).toMatchObject({ runId: 'run-2', hits: 1, value: null })
+    expect(report.rows[1].note).not.toBe('path not found on span')
+    expect(report.rows[1].note).toMatch(/withheld/)
+    expect(report.contentWithheld).toBe(1)
+  })
+
+  it('still answers structural paths on a withheld span', async () => {
+    const report = await logsQueryWithheld({ field: 'duration' })
+    expect(report.rows[1]).toMatchObject({ runId: 'run-2', value: 12 })
+    expect(report.rows[1].note).toBeUndefined()
+    expect(report.contentWithheld).toBeUndefined()
+  })
+
+  it('keeps a withheld run under --where instead of counting it as filtered out', async () => {
+    const report = await logsQueryWithheld({ where: 'result.count=6' })
+    expect(report.filteredOut).toBe(0)
+    expect(report.rows.map((row: { runId: string }) => row.runId)).toEqual(['run-1', 'run-2'])
+    expect(report.rows[1].note).toMatch(/withheld/)
+    expect(report.contentWithheld).toBe(1)
+  })
+})
+
 const DEPS_STATE = {
   blocks: {
     'trigger-1': { type: 'starter', name: 'Start' },

apps/sim/lib/mothership/agent-cli/engines/query.ts

@@ -23,6 +23,11 @@ const MAX_LIMIT = 50
 const TRACE_FETCH_CONCURRENCY = 5
 const VALUE_MAX_CHARS = 600
 const PATH_NOT_FOUND_NOTE = 'path not found on span'
+/**
+ * The log projection withheld this span's content rather than return what it could not
+ * prove safe to display, so an unresolved content path says nothing about the run.
+ */
+const CONTENT_WITHHELD_NOTE = 'span content withheld by the log projection; value unknown'
 
 /**
  * Keys that live on the span itself. A path whose head is anything else is read
@@ -56,6 +61,7 @@ interface QueryTraceSpan {
   duration?: number
   input?: Record<string, unknown>
   output?: Record<string, unknown>
+  contentWithheld?: true
   children?: QueryTraceSpan[]
 }
 
@@ -158,6 +164,7 @@ export const logsQueryCommand: AgentCliEngine = {
 
     let filteredOut = 0
     let missingTrace = 0
+    let contentWithheld = 0
     let fieldResolvedUnder: 'output' | undefined
     let whereResolvedUnder: 'output' | undefined
     const rows = await mapWithConcurrency(
@@ -193,23 +200,35 @@ export const logsQueryCommand: AgentCliEngine = {
           }
           return { ...base, hits: 0, value: null }
         }
+        const matchedRow = {
+          ...base,
+          hits: matches.length,
+          blockStatus: last.status ?? 'success',
+        }
         if (wherePath) {
           const matched = resolveSpanPathWithFallback(last, wherePath)
           if (matched.resolvedUnder) whereResolvedUnder = matched.resolvedUnder
+          // Filtering on a withheld value would silently drop a run that may well match.
+          if (matched.value === undefined && last.contentWithheld) {
+            contentWithheld++
+            return { ...matchedRow, value: null, note: CONTENT_WITHHELD_NOTE }
+          }
           if (String(matched.value) !== whereValue) {
             filteredOut++
             return undefined
           }
         }
         const resolved = resolveSpanPathWithFallback(last, field)
         if (resolved.resolvedUnder) fieldResolvedUnder = resolved.resolvedUnder
-        return {
-          ...base,
-          hits: matches.length,
-          blockStatus: last.status ?? 'success',
-          value: clipValue(resolved.value),
-          ...(resolved.value === undefined ? { note: PATH_NOT_FOUND_NOTE } : {}),
+        if (resolved.value === undefined) {
+          if (last.contentWithheld) contentWithheld++
+          return {
+            ...matchedRow,
+            value: null,
+            note: last.contentWithheld ? CONTENT_WITHHELD_NOTE : PATH_NOT_FOUND_NOTE,
+          }
         }
+        return { ...matchedRow, value: clipValue(resolved.value) }
       }
     )
 
@@ -230,6 +249,7 @@ export const logsQueryCommand: AgentCliEngine = {
               }
             : {}),
           ...(missingTrace ? { missingTrace } : {}),
+          ...(contentWithheld ? { contentWithheld } : {}),
           rows: kept,
         },
         null,

packages/sim-cli/src/generated/v2-api.ts

@@ -6244,6 +6244,7 @@ type GetLogResponseRef1 = {
     toolCost?: number
   }
   relativeStartMs?: number
+  contentWithheld?: true
   toolCalls?: Array<{
     id?: string
     name?: string
@@ -9037,6 +9038,7 @@ type ListLogsResponseRef2 = {
     toolCost?: number
   }
   relativeStartMs?: number
+  contentWithheld?: true
   toolCalls?: Array<{
     id?: string
     name?: string