Trace worker requests and reserve time for verification

CHANGELOG.md

@@ -7,6 +7,9 @@ project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
 
+- Bounded metadata-only request/tool traces for durable workers, with paged retrieval, repeated-read fingerprints, usage coverage and phase timings. Benchmarks retain trace artifacts.
+- Opt-in token/time reserves stop restricted OpenCode workers, confirm idle, and verify their patches before the hard limit. Failed checks use the existing bounded repair; hard budgets, errors, cancellation and ownership remain authoritative.
+
 ### Added
 - Optional scoped OpenCode workers using the supported custom-agent API: model-specific instructions, local tools, preserved repository rules, no automatic skills catalog, and a fixed 24-step limit. The full `standard` agent remains the default; `focused` adds an experimental compact prompt.
 - Separate engine uncached/cache-read/cache-write/reasoning/request metrics, accumulated across repairs. Paired repository-regression benchmark with withheld evaluator checks and a real-engine local request-size probe.

benchmarks/results/2026-09-21/checkpoint-smoke.json

@@ -0,0 +1,97 @@
+{
+  "testedAt": "2026-09-21T07:58:17.141Z",
+  "engineVersion": "1.18.31",
+  "scope": "Real installed engine, synthetic local inference; verifies stop-before-check, not coding quality or token savings.",
+  "requests": 2,
+  "checkedIdle": true,
+  "status": "completed",
+  "completion": {
+    "source": "checkpoint",
+    "checkpoint": {
+      "requested": true,
+      "confirmed": true,
+      "trigger": "time"
+    }
+  },
+  "usage": {
+    "inTokens": 100,
+    "outTokens": 5,
+    "estimated": false,
+    "available": true,
+    "uncachedInTokens": 100,
+    "cacheReadTokens": 0,
+    "cacheWriteTokens": 0,
+    "reasoningTokens": 0,
+    "modelRequests": 2
+  },
+  "elapsedMs": 5075,
+  "trace": {
+    "version": 1,
+    "requestCoverage": "observed",
+    "coverage": "Observed engine assistant messages and tool events; provider retries, child sessions and full request payloads are not captured.",
+    "summary": {
+      "requestCoverage": "observed",
+      "requests": 2,
+      "requestsWithUsage": 1,
+      "tools": 1,
+      "repeatedReads": 0,
+      "toolOutputBytes": 24,
+      "requestsWithTiming": 2,
+      "toolsWithTiming": 1,
+      "cumulativeAssistantMs": 4679,
+      "cumulativeToolMs": 6,
+      "verificationMs": 3,
+      "executionMs": 5074,
+      "discardedEvents": 0
+    },
+    "requests": [
+      {
+        "id": "r1",
+        "startedAt": 1789977492445,
+        "completedAt": 1789977493049,
+        "turn": 1,
+        "model": "fixture/coder",
+        "usage": {
+          "input": 100,
+          "output": 5,
+          "reasoning": 0,
+          "cacheRead": 0,
+          "cacheWrite": 0
+        },
+        "usageReported": true,
+        "elapsedMs": 604
+      },
+      {
+        "id": "r2",
+        "startedAt": 1789977493050,
+        "completedAt": 1789977497125,
+        "turn": 1,
+        "model": "fixture/coder",
+        "usage": {
+          "input": 0,
+          "output": 0,
+          "reasoning": 0,
+          "cacheRead": 0,
+          "cacheWrite": 0
+        },
+        "usageReported": false,
+        "failed": true,
+        "elapsedMs": 4075
+      }
+    ],
+    "tools": [
+      {
+        "id": "t1",
+        "startedAt": 1789977493037,
+        "completedAt": 1789977493043,
+        "turn": 1,
+        "name": "write",
+        "request": "r1",
+        "fingerprint": "87a2872fa824640d4bc65bf2",
+        "ok": true,
+        "outputBytes": 24,
+        "elapsedMs": 6
+      }
+    ]
+  }
+}

docs/coding-benchmarks.md

@@ -44,7 +44,7 @@ Task/profile ordering alternates between repetitions to reduce systematic order
 
 ## Read the report
 
-The JSON records the source hash, engine version, model, preflight, per-task results, patches, verification, usage and elapsed time. Companion `.patches/` files make solutions inspectable. Temporary service/worktrees are removed after complete runs. A transient read timeout is retried without replaying mutations. After an interruption or unrecoverable transport error, the service is stopped and an atomic checkpoint plus worktrees are retained. Resume with `node scripts/benchmark-coding.js --resume <report.json>`; existing job IDs are recovered, interrupted execution is not silently replayed, and an already recorded application is verified rather than applied again. Logical job failures remain scored failures. Use `--cases` with comma-separated task IDs for a smaller diagnostic run. The report has an explicit `complete` flag and unfinished IDs. The runner exits nonzero if any task is unaccepted or unfinished, but retains the report so failures remain part of the comparison.
+The JSON records the source hash, engine version, model, preflight, per-task results, patches, verification, usage and elapsed time. Companion `.patches/` files make solutions inspectable. New runs also retain `.traces/` metadata with per-message usage, tool timing/output sizes, and repeated-read fingerprints. Timings include queue, preparation, execution and collection; engine message lifetimes are not pure inference time. Temporary service/worktrees are removed after complete runs. A transient read timeout is retried without replaying mutations. After an interruption or unrecoverable transport error, the service is stopped and an atomic checkpoint plus worktrees are retained. Resume with `node scripts/benchmark-coding.js --resume <report.json>`; existing job IDs are recovered, interrupted execution is not silently replayed, and an already recorded application is verified rather than applied again. Logical job failures remain scored failures. Use `--cases` with comma-separated task IDs for a smaller diagnostic run. The report has an explicit `complete` flag and unfinished IDs. The runner exits nonzero if any task is unaccepted or unfinished, but retains the report so failures remain part of the comparison.
 
 The profile summaries include:
 

docs/worker-service.md

@@ -54,7 +54,7 @@ The CLI accepts the same JSON as `npm run jobs -- submit tasks.json`, or a singl
 | `jobs_wait` | Wait up to 25 seconds for a terminal result, or a change after `after_revision`. |
 | `jobs_get` | Compact evidence; `detail: true` includes the contract, attempts, and acceptance output. |
 | `jobs_list` | Recover recent IDs; filter by status and paginate. |
-| `jobs_read` | Read a character page from `patch` or `result`; continue at `next_offset`. |
+| `jobs_read` | Read a character page from `patch`, `result`, or `trace`; continue at `next_offset`. |
 | `jobs_cancel` | Stop queued or active work while retaining evidence and workspaces. |
 | `jobs_apply` | Validate and apply a completed isolated patch to the original source. |
 | `jobs_status` | Inspect shared capacity, provider cooldowns, and counts. |
@@ -103,6 +103,36 @@ Desktop sessions and legacy delegations keep their existing default agent. Permi
 
 Engine usage now reports `uncachedInTokens`, `cacheReadTokens`, `cacheWriteTokens`, `reasoningTokens`, and `modelRequests` alongside existing totals. `inTokens` includes uncached input plus cache reads/writes; `outTokens` includes output plus reasoning. Repeated usage events are deduplicated by message, and repair turns accumulate usage. These measure the observed session; independently running engine background/child sessions are not a complete account billing ledger. Missing detailed fields mean unavailable, not zero.
 
+## Request traces and verification reserves
+
+Durable jobs save a bounded `trace.json`, retrievable with `jobs_read(id, artifact: "trace")`. Compact status includes the trace summary and queue/preparation/execution/collection timings. Benchmark runs preserve traces in a companion `.traces/` directory before deleting temporary workspaces.
+
+The trace merges engine assistant-message updates by ID and records observed per-request usage, timestamps, tool names, tool durations, output byte counts, and repeated read/search argument fingerprints. Fingerprints use a per-execution secret; saved traces contain no prompts, source text, tool arguments, tool outputs, or raw errors. Existing result/patch artifacts still contain their normal task evidence. Each request/tool table is capped at 1,000 entries, with discarded events counted.
+
+Timing is observational: assistant-message lifetimes can include tool execution; cumulative durations overlap and are not pure inference time. Missing timestamps remain unknown. Requests initialized with zero tokens are not marked usage-reported without a completed-step receipt or nonzero usage. Interrupted requests can therefore have unknown final usage. Native workers currently provide tool traces only, with request coverage marked unavailable; engine background/child work and internal provider retries are not fully captured. Traces become durable when execution returns, not after every event. Repeated argument fingerprints identify candidates for inspection, not proof that a read was unnecessary after edits.
+
+To reserve room for verification on an isolated, pinned OpenCode worker:
+
+```json
+{
+  "model": "opencode/big-pickle",
+  "engine_profile": "scoped",
+  "max_tokens": 200000,
+  "timeout_ms": 180000,
+  "verification_reserve_tokens": 40000,
+  "verification_reserve_ms": 30000,
+  "checks": ["npm test"]
+}
+```
+
+Restart an older worker service with `npm run jobs -- stop` before using these capabilities; the client refuses to reuse a daemon from the previous capability version.
+
+Both reserve settings default to zero. They require `scoped` or `focused`, worktree isolation, and caller acceptance checks. At the first soft threshold in the initial model turn, OmniWork requests an engine abort, waits for the prompt response, and requires an acknowledged stop plus idle status. Only then does it run the checks. Passing checks can finish the job without an additional model-authored summary. Failed checks can trigger the existing bounded repair in the same agent/workspace; repairs use the remaining original budget and receive no second soft checkpoint.
+
+The hard token cap, original deadline, caller cancellation, ownership checks, and integration checks still apply. A request can overshoot the soft and hard token thresholds. Stop-confirmation failure or a provider error cannot be converted into success. Engine startup, tool shutdown, and check duration may exhaust the reserved time; these settings do not guarantee completion. Arbitrary detached processes are outside this coordination guarantee. Use strong acceptance commands; a passing smoke test cannot establish requirements it does not test.
+
+Run `node scripts/probe-engine-checkpoint.js` to exercise the installed engine against synthetic local inference. It writes a fixture, stalls a subsequent model response, and asserts stop/idle before verification. This probe spends no real inference tokens and does not measure coding-quality gains.
+
 ## Context and practical boundaries
 
 Workers receive a bounded task contract, selected file excerpts, and a cached repository orientation map of relevant paths and declarations. The map is capped at 4,000 characters; explicit file excerpts total at most 24,000 characters. Full content remains accessible through tools. Parent clients receive compact evidence and can retrieve artifacts without absorbing every worker transcript.
@@ -113,7 +143,7 @@ Worktrees isolate ordinary edits; they are not operating-system sandboxes. Worke
 
 There is no dependency graph, automatic cross-job conflict resolution, durable provider conversation resumption after a daemon crash, or automatic workspace garbage collection yet. Parallel tasks should have independent ownership. Retained worktrees and artifacts use disk space. On restart, uncertain in-flight execution is marked interrupted instead of silently replayed; inspect it before resubmission. An interrupted integration may require inspecting both source and the retained integration directory. The service exits after 15 idle minutes without queued or active jobs, and the next client restarts it. Completed results remain on disk.
 
-Artifacts live beneath `<data-dir>/jobs/tasks/<id>/`; `job.json`, `result.json`, `changes.patch`, and the worktree provide evidence. The service descriptor contains a private token for its loopback API; do not share it. `service.log` records service errors. To clean a retained Git worktree, stop relevant jobs, inspect the output, and use `git worktree remove <path>` before deleting its artifact directory.
+Artifacts live beneath `<data-dir>/jobs/tasks/<id>/`; `job.json`, `result.json`, `trace.json`, `changes.patch`, and the worktree provide evidence. The service descriptor contains a private token for its loopback API; do not share it. `service.log` records service errors. To clean a retained Git worktree, stop relevant jobs, inspect the output, and use `git worktree remove <path>` before deleting its artifact directory.
 
 ## Validation and design references
 

electron/execution-trace.js

@@ -0,0 +1,72 @@
+"use strict";
+const crypto = require("node:crypto");
+const number = v => Number.isFinite(v) && v >= 0 ? v : null;
+const elapsed = row => row.startedAt != null && row.completedAt != null ? Math.max(0, row.completedAt - row.startedAt) : null;
+const USAGE = ["input", "output", "reasoning"];
+// Canonicalize arguments only in memory. A per-execution secret prevents saved
+// fingerprints from becoming a dictionary of repository paths or shell commands.
+function canonical(value) {
+  if (Array.isArray(value)) return value.map(canonical);
+  if (value && typeof value === "object") return Object.fromEntries(Object.keys(value).sort().map(k => [k, canonical(value[k])]));
+  return value;
+}
+class ExecutionTrace {
+  constructor({ limit = 1000 } = {}) {
+    this.limit = limit; this.key = crypto.randomBytes(32); this.requests = new Map(); this.tools = new Map();
+    this.discardedEvents = 0; this.verificationMs = 0; this.startedAt = Date.now();
+  }
+  row(map, key, prefix) {
+    if (!key) return null;
+    if (map.has(key)) return map.get(key);
+    if (map.size >= this.limit) { this.discardedEvents++; return null; }
+    const row = { id: prefix + (map.size + 1), startedAt: null, completedAt: null };
+    map.set(key, row); return row;
+  }
+  event(type, p, turn) {
+    if (!p?.id) return;
+    if (type === "request") {
+      const row = this.row(this.requests, `${turn}:${p.id}`, "r"); if (!row) return;
+      row.turn = turn;
+      if (typeof p.model === "string") row.model = p.model.slice(0, 300);
+      for (const k of ["startedAt", "completedAt"]) if (number(p[k]) != null) row[k] = p[k];
+      if (p.failed === true) row.failed = true;
+      if (p.tokens) {
+        row.usage = Object.fromEntries(USAGE.map(k => [k, number(p.tokens[k])]));
+        row.usage.cacheRead = number(p.tokens.cache?.read);
+        row.usage.cacheWrite = number(p.tokens.cache?.write);
+        // OpenCode initializes in-flight messages with zero tokens. Those are
+        // not a provider usage receipt, especially when generation is aborted.
+        row.usageReported = Boolean(row.usageReported || p.usageReported || Object.values(row.usage).some(n => n > 0));
+      }
+    }
+    if (type === "tool_call") {
+      const row = this.row(this.tools, `${turn}:${p.id}`, "t"); if (!row) return;
+      row.turn = turn; row.name = String(p.name || "unknown").slice(0, 100);
+      row.startedAt ??= number(p.startedAt) ?? Date.now();
+      row.request = p.requestId ? this.row(this.requests, `${turn}:${p.requestId}`, "r")?.id || null : null;
+      row.fingerprint = crypto.createHmac("sha256", this.key).update(JSON.stringify([row.name, canonical(p.args || {})])).digest("hex").slice(0, 24);
+    }
+    if (type === "tool_result") {
+      const row = this.row(this.tools, `${turn}:${p.id}`, "t"); if (!row) return;
+      row.completedAt = number(p.completedAt) ?? Date.now(); row.ok = typeof p.ok === "boolean" ? p.ok : null;
+      row.outputBytes = number(p.outputBytes) ?? Buffer.byteLength(String(p.result || ""));
+      if (Number.isInteger(p.exitCode)) row.exitCode = p.exitCode;
+    }
+  }
+  snapshot() {
+    const requests = [...this.requests.values()].map(r => ({ ...r, elapsedMs: elapsed(r) }));
+    const tools = [...this.tools.values()].map(r => ({ ...r, elapsedMs: elapsed(r) }));
+    const reads = new Set(); let repeatedReads = 0;
+    for (const t of tools) if (/^(read|read_file|glob|grep|list|list_dir)$/.test(t.name)) {
+      if (reads.has(t.fingerprint)) repeatedReads++; else reads.add(t.fingerprint);
+    }
+    return { version: 1, requestCoverage: requests.length ? "observed" : "unavailable", coverage: "Observed engine assistant messages and tool events; provider retries, child sessions and full request payloads are not captured.",
+      summary: { requestCoverage: requests.length ? "observed" : "unavailable", requests: requests.length, requestsWithUsage: requests.filter(r => r.usageReported && r.usage?.input != null && r.usage?.output != null).length,
+        tools: tools.length, repeatedReads, toolOutputBytes: tools.reduce((n,t) => n + (t.outputBytes || 0), 0),
+        requestsWithTiming: requests.filter(r => r.elapsedMs != null).length, toolsWithTiming: tools.filter(t => t.elapsedMs != null).length,
+        cumulativeAssistantMs: requests.some(r => r.elapsedMs != null) ? requests.reduce((n,r) => n + (r.elapsedMs || 0), 0) : null,
+        cumulativeToolMs: tools.some(t => t.elapsedMs != null) ? tools.reduce((n,t) => n + (t.elapsedMs || 0), 0) : null,
+        verificationMs: this.verificationMs, executionMs: Date.now() - this.startedAt, discardedEvents: this.discardedEvents }, requests, tools };
+  }
+}
+module.exports = { ExecutionTrace };