test: add real OpenCode OTLP integration coverage

package.json

@@ -24,7 +24,8 @@
   },
   "description": "OpenTelemetry telemetry plugin for opencode CLI",
   "devDependencies": {
-    "@opencode/plugin": "^2.0.18",
+    "@opencode/plugin": "2.0.20",
+    "@opencode/sdk": "2.0.20",
     "@types/bun": "latest",
     "@typescript-eslint/parser": "^8.62.0",
     "eslint": "^9.24.0",
@@ -80,6 +81,7 @@
     "check:jsdoc-coverage": "bun scripts/check-jsdoc-coverage.mjs",
     "lint": "bun --bun eslint .",
     "prepack": "bun run build",
+    "test:integration": "bun test tests/integration",
     "typecheck": "tsc --noEmit"
   },
   "type": "module",

tests/integration/README.md

@@ -0,0 +1,29 @@
+# OpenCode integration tests
+
+```sh
+bun run test:integration
+```
+
+These tests run the pinned OpenCode SDK (`2.0.20`) with the actual plugin in a fresh Bun subprocess for each scenario. They require Git, but no model credentials, Docker, or external telemetry service. The regular `bun test` command and CI include them too.
+
+Two temporary HTTP servers bind to loopback on automatically assigned ports:
+
+- An OpenAI-compatible Chat Completions endpoint returns scripted SSE responses and captures incoming model requests. It requests the built-in `read` tool, then finishes after receiving its real result. Only model responses are mocked; tool execution and OpenCode events are not.
+- An OTLP/JSON receiver accepts `/v1/traces`, `/v1/metrics`, and `/v1/logs`. Assertions inspect the actual exported JSON rather than mocked exporter calls.
+
+Each subprocess gets its own temporary Git project, database, home, and XDG directories. It inherits only environment variables needed to launch processes, not provider credentials or telemetry configuration. Model catalog fetching and project config discovery are disabled, and the fixture provider is selected explicitly. Keep the SDK and plugin development dependency versions aligned to avoid duplicate OpenCode runtime modules.
+
+The runner waits for session completion, drains plugin processing during host cleanup, and explicitly flushes telemetry before exiting. Structured results use an awaited file write rather than stdout, avoiding truncated JSON when a subprocess exits with buffered console output on Linux. The read fixture exercises results larger than 8 KiB. Receivers stay alive until the runner exits. A 30-second process deadline prevents hangs. Runner errors include stdout, stderr, model requests, and received payloads; temporary files and servers are cleaned up afterwards.
+
+## Coverage
+
+- Successful text generation exports session and LLM spans with correct parenting, token attributes, session metrics, and logs.
+- A real local `read` returns fixture contents to the model and emits a successful tool-result log.
+- Disabling tool tracing suppresses tool spans without changing actual execution.
+- Local tool spans and duration metrics have three explicit `test.failing` regression cases for [PR #134](https://github.com/DEVtheOPS/opencode-plugin-otel/pull/134). These assertions run against real exported telemetry and currently fail as expected on `main`. Scenario setup and successful execution checks are ordinary tests, so infrastructure failures cannot satisfy an expected-failure assertion. Once the fix lands, change both `test.failing` registrations to `test`; an unexpected pass deliberately fails the suite until then.
+
+Metric assertions use the latest matching cumulative data point, not the sum of repeated exports. Durations are compared between signals rather than against fixed wall-clock values.
+
+## Extending the suite
+
+Add scripted scenarios for tool errors, malformed arguments, permission rejection, shell commits, subagents, and trace propagation. Preserve real OpenCode event production instead of synthesizing `session.tool.*` payloads. Keep live-provider checks separate from deterministic PR checks. This receiver covers HTTP/JSON; gRPC and HTTP/protobuf need separate transport coverage, for example with a temporary OpenTelemetry Collector.

tests/integration/fixtures.ts

@@ -0,0 +1,118 @@
+import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"
+import { tmpdir } from "node:os"
+import { join } from "node:path"
+
+export type Json = null | boolean | number | string | Json[] | { [key: string]: Json }
+export type Payload = { [key: string]: Json }
+
+export async function runScenario(scenario: "text" | "read", disabledTraces = "") {
+  const directory = await mkdtemp(join(tmpdir(), "opencode-otel-integration-"))
+  const requests: Payload[] = []
+  const received: Record<string, Payload[]> = { traces: [], metrics: [], logs: [] }
+  const failures: string[] = []
+  const receiver = Bun.serve({
+    hostname: "127.0.0.1",
+    port: 0,
+    async fetch(request) {
+      const signal = new URL(request.url).pathname.replace("/v1/", "")
+      if (request.method !== "POST" || !received[signal]) return new Response(null, { status: 404 })
+      try {
+        received[signal].push(await request.json() as Payload)
+        return Response.json({})
+      } catch (error) {
+        failures.push(String(error))
+        return new Response(null, { status: 400 })
+      }
+    },
+  })
+  const provider = Bun.serve({
+    hostname: "127.0.0.1",
+    port: 0,
+    async fetch(request) {
+      if (new URL(request.url).pathname !== "/v1/chat/completions") return new Response(null, { status: 404 })
+      const body = await request.json() as Payload
+      requests.push(body)
+      if (requests.length > 3) return Response.json({ error: { message: "Unexpected model request" } }, { status: 400 })
+      const messages = body["messages"] as Payload[]
+      const hasToolResult = messages.some(message => message["role"] === "tool")
+      const tools = body["tools"] as Payload[] | undefined
+      const wantsRead = scenario === "read" && !hasToolResult
+      if (wantsRead && !tools?.some(tool => (tool["function"] as Payload)["name"] === "read")) {
+        failures.push("OpenCode did not expose the read tool")
+        return Response.json({ error: { message: failures.at(-1) } }, { status: 400 })
+      }
+      const chunk = (delta: Payload, finish: string | null = null, usage?: Payload) => ({
+        id: "chatcmpl-integration",
+        object: "chat.completion.chunk",
+        created: 1,
+        model: "scripted",
+        choices: [{ index: 0, delta, finish_reason: finish }],
+        ...(usage ? { usage } : {}),
+      })
+      const delta: Payload = wantsRead
+        ? { tool_calls: [{ index: 0, id: "call_read", type: "function", function: { name: "read", arguments: JSON.stringify({ path: join(directory, "fixture.txt") }) } }] }
+        : { content: "Fixture complete." }
+      const chunks = [
+        chunk({ role: "assistant" }),
+        chunk(delta),
+        chunk({}, wantsRead ? "tool_calls" : "stop", { prompt_tokens: 20, completion_tokens: 5, total_tokens: 25 }),
+      ]
+      return new Response(chunks.map(value => `data: ${JSON.stringify(value)}\n\n`).join("") + "data: [DONE]\n\n", {
+        headers: { "content-type": "text/event-stream" },
+      })
+    },
+  })
+  let child: Bun.Subprocess<"ignore", "pipe", "pipe"> | undefined
+  let timer: ReturnType<typeof setTimeout> | undefined
+  let timedOut = false
+  try {
+    await mkdir(join(directory, "config"))
+    await writeFile(join(directory, "fixture.txt"), "integration-fixture-contents\n".repeat(256))
+    const env: Record<string, string> = {}
+    for (const key of ["PATH", "SystemRoot", "TMPDIR", "TEMP", "TMP"]) {
+      if (process.env[key]) env[key] = process.env[key]!
+    }
+    Object.assign(env, {
+      HOME: directory,
+      XDG_CONFIG_HOME: join(directory, "config"),
+      XDG_DATA_HOME: join(directory, "data"),
+      XDG_CACHE_HOME: join(directory, "cache"),
+      XDG_STATE_HOME: join(directory, "state"),
+      INTEGRATION_PROJECT: directory,
+      INTEGRATION_SCENARIO: scenario,
+      INTEGRATION_MODEL_URL: `${provider.url}v1`,
+      OPENCODE_ENABLE_TELEMETRY: "1",
+      OPENCODE_OTLP_ENDPOINT: String(receiver.url),
+      OPENCODE_OTLP_PROTOCOL: "http/json",
+      OPENCODE_DISABLE_TRACES: disabledTraces,
+    })
+    const git = Bun.spawnSync(["git", "init", "--quiet", directory], { env })
+    if (git.exitCode !== 0) throw new Error(git.stderr.toString())
+    child = Bun.spawn([process.execPath, join(import.meta.dir, "runner.ts")], {
+      cwd: directory,
+      env,
+      stdin: "ignore",
+      stdout: "pipe",
+      stderr: "pipe",
+    })
+    const stdout = new Response(child.stdout).text()
+    const stderr = new Response(child.stderr).text()
+    timer = setTimeout(() => { timedOut = true; child?.kill() }, 30_000)
+    const [code, output, errors] = await Promise.all([child.exited, stdout, stderr])
+    if (timedOut || code !== 0 || failures.length) {
+      throw new Error(JSON.stringify({ timedOut, code, output, errors, failures, requests, received }, null, 2))
+    }
+    try {
+      const result = JSON.parse(await readFile(join(directory, "result.json"), "utf8")) as Payload
+      return { received, requests, result, output, errors }
+    } catch (error) {
+      throw new Error(JSON.stringify({ error: String(error), output, errors, requests, received }, null, 2))
+    }
+  } finally {
+    if (timer) clearTimeout(timer)
+    if (child) { child.kill(); await child.exited }
+    provider.stop(true)
+    receiver.stop(true)
+    await rm(directory, { recursive: true, force: true })
+  }
+}

tests/integration/opencode.test.ts

@@ -0,0 +1,82 @@
+import { beforeAll, describe, expect, test } from "bun:test"
+import { runScenario } from "./fixtures.ts"
+import { attributes, metricPoints, objects, signalRecords } from "./otlp.ts"
+
+test("exports real OpenCode session telemetry over OTLP/JSON", async () => {
+  const { received, requests, result } = await runScenario("text")
+  expect(requests).toHaveLength(1)
+  expect(result["sessionID"]).toBeString()
+  expect(received["traces"]!.length).toBeGreaterThan(0)
+  expect(received["metrics"]!.length).toBeGreaterThan(0)
+  expect(received["logs"]!.length).toBeGreaterThan(0)
+
+  const spans = signalRecords(received["traces"]!, "Spans")
+  const run = spans.find(span => span["name"] === "opencode.session")!
+  const llm = spans.find(span => span["name"] === "opencode.llm")!
+  expect(attributes(run)["session.id"]).toBe(result["sessionID"])
+  expect(llm["traceId"]).toBe(run["traceId"])
+  expect(llm["parentSpanId"]).toBe(run["spanId"])
+  expect(attributes(llm)["llm.token_count.prompt"]).toBe(20)
+  expect(attributes(llm)["llm.token_count.completion"]).toBe(5)
+
+  const metrics = signalRecords(received["metrics"]!, "Metrics")
+  const sessions = metricPoints(metrics, "opencode.session.count", "sum").at(-1)!
+  expect(Number(sessions["asDouble"] ?? sessions["asInt"])).toBe(1)
+  expect(attributes(sessions)["session.id"]).toBe(result["sessionID"])
+  const logs = signalRecords(received["logs"]!, "Logs")
+  expect(logs.some(log => attributes(log)["event.name"] === "api_request")).toBe(true)
+  expect(logs.some(log => attributes(log)["event.name"] === "session.idle")).toBe(true)
+}, 40_000)
+
+for (const disabled of [false, true]) {
+  describe(`local read with tool tracing ${disabled ? "disabled" : "enabled"}`, () => {
+    let scenario: Awaited<ReturnType<typeof runScenario>>
+    beforeAll(async () => { scenario = await runScenario("read", disabled ? "tool" : "") }, 40_000)
+
+    test("executes the real read tool and reports its contents to the provider", () => {
+      expect(scenario.requests).toHaveLength(2)
+      const messages = objects(scenario.requests[1]!["messages"])
+      const result = messages.find(message => message["role"] === "tool")!
+      expect(result["tool_call_id"]).toBe("call_read")
+      expect(result["content"]).toContain("integration-fixture-contents")
+      const tool = objects(scenario.result["messages"])
+        .flatMap(message => objects(message["content"])).find(part => part["type"] === "tool")!
+      expect(tool["executed"]).toBe(false)
+      expect(JSON.stringify(scenario.result).length).toBeGreaterThan(8192)
+      expect(result["content"]).toContain("256: integration-fixture-contents")
+      const logs = signalRecords(scenario.received["logs"]!, "Logs")
+      const log = logs.find(record => attributes(record)["event.name"] === "tool_result")!
+      expect(attributes(log)["tool_name"]).toBe("read")
+      expect(attributes(log)["success"]).toBe(true)
+    })
+
+    if (disabled) {
+      test("does not export tool spans", () => {
+        expect(signalRecords(scenario.received["traces"]!, "Spans")
+          .some(span => String(span["name"]).startsWith("opencode.tool."))).toBe(false)
+      })
+    } else {
+      test.failing("#134: exports a locally executed tool span under its LLM span", () => {
+        const spans = signalRecords(scenario.received["traces"]!, "Spans")
+        const tool = spans.find(span => span["name"] === "opencode.tool.read")
+        expect(tool).toBeDefined()
+        const parent = spans.find(span => span["spanId"] === tool!["parentSpanId"])!
+        expect(parent["name"]).toBe("opencode.llm")
+        expect(tool!["traceId"]).toBe(parent["traceId"])
+        expect(attributes(tool!)["output.value"]).toContain("integration-fixture-contents")
+      })
+    }
+
+    test.failing("#134: exports a duration sample for the locally executed tool", () => {
+      const metrics = signalRecords(scenario.received["metrics"]!, "Metrics")
+      const points = metricPoints(metrics, "opencode.tool.duration", "histogram")
+        .filter(point => attributes(point)["tool_name"] === "read")
+      expect(points.length).toBeGreaterThan(0)
+      expect(Number(points.at(-1)!["count"])).toBe(1)
+      expect(attributes(points.at(-1)!)["success"]).toBe(true)
+      const logs = signalRecords(scenario.received["logs"]!, "Logs")
+      const log = logs.find(record => attributes(record)["event.name"] === "tool_result")!
+      expect(Number(points.at(-1)!["sum"])).toBe(Number(attributes(log)["duration_ms"]))
+    })
+  })
+}

tests/integration/otlp.ts

@@ -0,0 +1,24 @@
+import type { Json, Payload } from "./fixtures.ts"
+
+export function objects(value: Json | undefined): Payload[] {
+  if (!Array.isArray(value)) return []
+  return value.filter((item): item is Payload => typeof item === "object" && item !== null && !Array.isArray(item))
+}
+
+export function attributes(record: Payload): Record<string, Json> {
+  return Object.fromEntries(objects(record["attributes"]).map(attribute => {
+    const value = attribute["value"] as Payload
+    return [String(attribute["key"]), value["stringValue"] ?? value["boolValue"] ?? value["intValue"] ?? value["doubleValue"] ?? null]
+  }))
+}
+
+export function signalRecords(payloads: Payload[], signal: "Spans" | "Metrics" | "Logs"): Payload[] {
+  const records = signal === "Spans" ? "spans" : signal === "Metrics" ? "metrics" : "logRecords"
+  return payloads.flatMap(payload => objects(payload[`resource${signal}`])
+    .flatMap(resource => objects(resource[`scope${signal}`]).flatMap(scope => objects(scope[records]))))
+}
+
+export function metricPoints(metrics: Payload[], name: string, kind: "sum" | "histogram"): Payload[] {
+  return metrics.filter(metric => metric["name"] === name)
+    .flatMap(metric => objects((metric[kind] as Payload | undefined)?.["dataPoints"]))
+}

tests/integration/runner.ts

@@ -0,0 +1,55 @@
+import { OpenCode, Model, Provider } from "@opencode/sdk"
+import plugin from "../../src/index.ts"
+import { acquireTracingState, flushSharedOtel } from "../../src/state.ts"
+
+const directory = process.env["INTEGRATION_PROJECT"]!
+const providerID = Provider.ID.make("integration")
+const modelID = Model.ID.make("scripted")
+const model = Model.Info.default(providerID, modelID)
+const toolMode = process.env["INTEGRATION_SCENARIO"] === "read"
+
+await using host = await OpenCode.create({
+  database: { path: `${directory}/opencode.db` },
+  models: { fetch: false, snapshot: false },
+  config: { project: false, directory: `${directory}/config`, content: "{}" },
+  fs: { filewatcher: false, fff: false },
+  plugins: [
+    {
+      id: "integration.fixtures",
+      async setup(ctx) {
+        await ctx.provider.transform(editor => editor.add({
+          info: {
+            ...Provider.Info.empty(providerID),
+            activation: "enabled",
+            package: "@opencode/ai/providers/openai-compatible",
+            settings: { baseURL: process.env["INTEGRATION_MODEL_URL"]! },
+          },
+          models: [model],
+        }))
+        await ctx.session.hook("title", event => { event.result = "Integration fixture" })
+        await ctx.session.hook("context", event => {
+          for (const name of Object.keys(event.tools)) {
+            if (!toolMode || name !== "read") delete event.tools[name]
+          }
+        })
+      },
+    },
+    plugin,
+  ],
+})
+
+const session = await host.sessions.create({
+  location: { directory },
+  model: { providerID, id: modelID },
+  title: "Integration fixture",
+})
+await host.sessions.prompt({ sessionID: session.id, text: "Read fixture.txt and report its contents." })
+await host.sessions.wait({ sessionID: session.id })
+await acquireTracingState().eventQueue
+const messages = await host.sessions.context({ sessionID: session.id })
+if (!messages.some(message => message.type === "idle" && message.outcome === "succeeded")) {
+  throw new Error(`Session did not succeed: ${JSON.stringify(messages)}`)
+}
+await host.close()
+await flushSharedOtel()
+await Bun.write(`${directory}/result.json`, JSON.stringify({ sessionID: session.id, messages }))