diff --git a/.github/workflows/_release.yml b/.github/workflows/_release.yml index 7405386..41cac4a 100644 --- a/.github/workflows/_release.yml +++ b/.github/workflows/_release.yml @@ -2,9 +2,9 @@ # release.yml (production repos) and test-release.yml (the shared test repo), # which supply `dist_repo` and `record`. # -# Always: build and `make publish` to . Claude and Codex releases -# also bump each plugin manifest version. Antigravity's native manifest has no -# version field, so its repository tag is the release version. +# Always: build and `make publish` to . Claude, Codex, and Grok +# releases also bump each plugin manifest version. Antigravity's native +# manifest has no version field, so its repository tag is the release version. # When record=true (production): commit the bump to main when needed, tag # v-, and create a GitHub Release on the monorepo. After the # distribution is deployed, its repo gets an unsuffixed v tag and diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9d74900..483ff9d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,6 +2,7 @@ # antigravity -> braintrustdata/braintrust-antigravity-plugin # claude -> braintrustdata/braintrust-claude-plugin # codex -> braintrustdata/braintrust-codex-plugin +# grok -> braintrustdata/braintrust-grok-plugin # # Full flow: stamp manifest versions where the agent format supports them -> # commit to main when needed -> monorepo tag/release using @@ -22,7 +23,7 @@ on: description: "Plugin bundle to release" required: true type: choice - options: [antigravity, claude, codex] + options: [antigravity, claude, codex, grok] permissions: contents: write @@ -33,6 +34,6 @@ jobs: with: version: ${{ inputs.version }} plugin: ${{ inputs.plugin }} - dist_repo: ${{ inputs.plugin == 'antigravity' && 'braintrustdata/braintrust-antigravity-plugin' || inputs.plugin == 'codex' && 'braintrustdata/braintrust-codex-plugin' || 'braintrustdata/braintrust-claude-plugin' }} + dist_repo: ${{ inputs.plugin == 'antigravity' && 'braintrustdata/braintrust-antigravity-plugin' || inputs.plugin == 'codex' && 'braintrustdata/braintrust-codex-plugin' || inputs.plugin == 'grok' && 'braintrustdata/braintrust-grok-plugin' || 'braintrustdata/braintrust-claude-plugin' }} record: true secrets: inherit diff --git a/.github/workflows/test-release.yml b/.github/workflows/test-release.yml index 4acd7da..8067b06 100644 --- a/.github/workflows/test-release.yml +++ b/.github/workflows/test-release.yml @@ -19,7 +19,7 @@ on: description: "Plugin bundle to release. Overwrites braintrustdata/test-coding-agent-dist" required: true type: choice - options: [antigravity, claude, codex] + options: [antigravity, claude, codex, grok] permissions: contents: write diff --git a/AGENTS.md b/AGENTS.md index 94a0c9a..e3dcb89 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -41,7 +41,7 @@ cargo test --manifest-path bt-daemon/Cargo.toml --all-features ## Versioning and distribution -Versioning is per distribution. Claude and Codex plugins carry their version in +Versioning is per distribution. Claude, Codex, and Grok plugins carry their version in their plugin manifests, and `scripts/set-plugin-version.py` updates those manifests for a release. Antigravity's schema does not expose a version field, so its distribution is versioned by repository tags and GitHub Releases. @@ -54,6 +54,7 @@ is unversioned. | antigravity | `braintrustdata/braintrust-antigravity-plugin` | | claude | `braintrustdata/braintrust-claude-plugin` | | codex | `braintrustdata/braintrust-codex-plugin` | +| grok | `braintrustdata/braintrust-grok-plugin` | A distribution repository is a generated artifact. Each deploy clones it, replaces the tracked tree with a fresh build, and pushes the result. diff --git a/README.md b/README.md index d6a70f9..3ec77fc 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ For further instructions, see the instructions for your desired coding agent | Google Antigravity | [braintrustdata/braintrust-antigravity-plugin](https://github.com/braintrustdata/braintrust-antigravity-plugin) | | Claude Code | [braintrustdata/braintrust-claude-plugin](https://github.com/braintrustdata/braintrust-claude-plugin) | | Codex | [braintrustdata/braintrust-codex-plugin](https://github.com/braintrustdata/braintrust-codex-plugin) | +| Grok | [braintrustdata/braintrust-grok-plugin](https://github.com/braintrustdata/braintrust-grok-plugin) | | OpenCode | npm: [`@braintrust/trace-opencode`](https://www.npmjs.com/package/@braintrust/trace-opencode) | | Pi | npm: [`@braintrust/pi-extension`](https://www.npmjs.com/package/@braintrust/pi-extension) | diff --git a/bt-daemon/README.md b/bt-daemon/README.md index 7dce617..0041c29 100644 --- a/bt-daemon/README.md +++ b/bt-daemon/README.md @@ -49,6 +49,7 @@ Each coding agent reads an independent non-credential `braintrust.json` file: - OpenCode: `$XDG_CONFIG_HOME/opencode/braintrust.json`, falling back to `~/.config/opencode/braintrust.json` - Pi: `~/.pi/agent/braintrust.json` +- Grok: `~/.grok/braintrust.json` `BT_DAEMON_CONFIG` can override the path for isolated tests and managed hosts. @@ -140,16 +141,17 @@ different profiles, organizations, projects, experiments, or parent spans. ## Status Phases 0–5 are implemented: protocol, daemon lifecycle, Braintrust sink, -Codex and Claude translators, `bt daemon` integration, and thin hook shims for -both shipped plugins. Every coding-agent capture request returns after the raw +Codex, Claude, and Grok translators, `bt daemon` integration, and thin hook shims for +all shipped plugins. Every coding-agent capture request returns after the raw event is flushed to its journal; authentication, correlation, translation, and reporting run on daemon-owned workers. Restart recovery replays the redacted journal with deterministic span ids, so resubmitted rows merge into the same spans instead of creating duplicates. Claude and Codex lifecycle entries -reference a daemon-owned transcript mirror, so recovery does not depend on -mutable external paths without re-recording the transcript on every turn. -Explicit turn/session-end flushes are bounded, and sessions can target project -logs or an experiment. +reference a daemon-owned transcript mirror; Grok records independent bounded +updates and events mirrors. Recovery therefore does not depend on mutable +external paths or copy a full transcript into every event. Explicit +turn/session-end flushes are bounded, and sessions can target project logs or +an experiment. Memory is bounded end to end, while on-disk records stay complete: the daemon never holds a transcript or a whole journal in memory, mirroring and replay diff --git a/bt-daemon/src/delivery_ledger.rs b/bt-daemon/src/delivery_ledger.rs index f865993..64070b3 100644 --- a/bt-daemon/src/delivery_ledger.rs +++ b/bt-daemon/src/delivery_ledger.rs @@ -18,12 +18,16 @@ use std::path::{Path, PathBuf}; struct LedgerFile { #[serde(default, alias = "terminal_span_ids")] completed_span_ids: HashSet, + #[serde(default)] + late_merge_span_ids: HashSet, } struct DeliveryLedger { path: PathBuf, known: HashSet, pending: HashSet, + known_late_merges: HashSet, + pending_late_merges: HashSet, } impl DeliveryLedger { @@ -46,15 +50,17 @@ impl DeliveryLedger { crate::ids::session_storage_id(source, session_id), &fingerprint_id[..32] )); - let known = match tokio::fs::read(&path).await { - Ok(bytes) => serde_json::from_slice::(&bytes)?.completed_span_ids, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => HashSet::new(), + let persisted = match tokio::fs::read(&path).await { + Ok(bytes) => serde_json::from_slice::(&bytes)?, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => LedgerFile::default(), Err(error) => return Err(error.into()), }; Ok(Self { path, - known, + known: persisted.completed_span_ids, pending: HashSet::new(), + known_late_merges: persisted.late_merge_span_ids, + pending_late_merges: HashSet::new(), }) } @@ -64,7 +70,12 @@ impl DeliveryLedger { let row = match op { SpanOp::Insert(row) | SpanOp::Merge(row) => row, }; - !self.known.contains(&row.span_id) && !self.pending.contains(&row.span_id) + if row.allow_late_merge { + !self.known_late_merges.contains(&row.span_id) + && !self.pending_late_merges.contains(&row.span_id) + } else { + !self.known.contains(&row.span_id) && !self.pending.contains(&row.span_id) + } }) .cloned() .collect() @@ -75,17 +86,21 @@ impl DeliveryLedger { let row = match op { SpanOp::Insert(row) | SpanOp::Merge(row) => row, }; - if row.end_ms.is_some() { + if row.allow_late_merge { + self.pending_late_merges.insert(row.span_id.clone()); + } else if row.end_ms.is_some() { self.pending.insert(row.span_id.clone()); } } } async fn commit(&mut self) -> anyhow::Result<()> { - if self.pending.is_empty() { + if self.pending.is_empty() && self.pending_late_merges.is_empty() { return Ok(()); } self.known.extend(self.pending.drain()); + self.known_late_merges + .extend(self.pending_late_merges.drain()); let parent = self.path.parent().expect("ledger path has a parent"); tokio::fs::create_dir_all(parent).await?; let temp = self @@ -95,6 +110,7 @@ impl DeliveryLedger { &temp, serde_json::to_vec(&LedgerFile { completed_span_ids: self.known.clone(), + late_merge_span_ids: self.known_late_merges.clone(), })?, ) .await?; @@ -237,6 +253,15 @@ mod tests { }) } + fn late_merge(span_id: &str) -> SpanOp { + SpanOp::Merge(SpanRow { + span_id: span_id.into(), + root_span_id: "root".into(), + allow_late_merge: true, + ..Default::default() + }) + } + #[tokio::test] async fn a_destination_receives_a_terminal_span_only_once_across_sink_instances() { let temp = tempfile::tempdir().unwrap(); @@ -276,6 +301,45 @@ mod tests { assert!(repeated_output.lock().unwrap().is_empty()); } + #[tokio::test] + async fn a_completed_span_receives_one_late_merge_across_sink_instances() { + let temp = tempfile::tempdir().unwrap(); + let first = RecordingSink::default(); + let mut first = LedgerSink::new( + Box::new(first), + temp.path(), + "grok", + "session-1", + Some(&config("project-a")), + ) + .await; + assert_eq!(first.emit(&[terminal("span-1")]).await.unwrap(), 1); + first.flush().await.unwrap(); + + let second = RecordingSink::default(); + let mut second = LedgerSink::new( + Box::new(second), + temp.path(), + "grok", + "session-1", + Some(&config("project-a")), + ) + .await; + assert_eq!(second.emit(&[late_merge("span-1")]).await.unwrap(), 1); + second.flush().await.unwrap(); + + let third = RecordingSink::default(); + let mut third = LedgerSink::new( + Box::new(third), + temp.path(), + "grok", + "session-1", + Some(&config("project-a")), + ) + .await; + assert_eq!(third.emit(&[late_merge("span-1")]).await.unwrap(), 0); + } + #[tokio::test] async fn a_different_destination_replays_the_same_terminal_span() { let temp = tempfile::tempdir().unwrap(); diff --git a/bt-daemon/src/dispatch.rs b/bt-daemon/src/dispatch.rs index 6b4e1da..b3c87c3 100644 --- a/bt-daemon/src/dispatch.rs +++ b/bt-daemon/src/dispatch.rs @@ -214,12 +214,15 @@ impl Session { } /// Agent transcript files are external mutable state. Mirror them into -/// daemon-owned storage at lifecycle boundaries and journal only a reference, -/// so recovery/replay does not depend on a path that Claude may later rewrite -/// or delete — and so the transcript is stored once rather than re-copied into -/// every event. Fail open: without a reference the translator reads the live -/// path exactly as before. +/// daemon-owned storage at lifecycle boundaries and journal only bounded +/// references, so recovery sees exactly the bytes that live translation +/// observed instead of depending on mutable external paths or copying a full +/// transcript into every event. Capture failures remain fail-open. pub(crate) async fn hydrate_transcript_reference(data_dir: &std::path::Path, env: &mut Envelope) { + if env.source == "grok" { + hydrate_grok_transcript_references(data_dir, env).await; + return; + } let should_capture = match env.source.as_str() { "codex" => true, "claude-code" => matches!( @@ -265,6 +268,57 @@ pub(crate) async fn hydrate_transcript_reference(data_dir: &std::path::Path, env } } +async fn hydrate_grok_transcript_references(data_dir: &std::path::Path, env: &mut Envelope) { + let Some(transcript) = env + .payload + .get("transcriptPath") + .or_else(|| env.payload.get("transcript_path")) + .and_then(serde_json::Value::as_str) + .map(std::path::PathBuf::from) + else { + return; + }; + let Some(session_dir) = transcript.parent() else { + return; + }; + + let mut mirrors = serde_json::Map::new(); + for (name, key) in [ + ("updates.jsonl", "updates"), + ("events.jsonl", "events"), + ("system_prompt.txt", "system_prompt"), + ] { + let source = session_dir.join(name); + let Some(source_str) = source.to_str() else { + continue; + }; + let mirror_session = crate::ids::session_namespace(&env.source, &env.session_id); + match crate::transcript_mirror::capture(data_dir, &mirror_session, source_str).await { + Ok((mirror, through)) => { + mirrors.insert( + key.to_string(), + serde_json::json!({ + "path": source, + "mirror": mirror, + "through": through, + }), + ); + } + Err(error) => { + tracing::debug!(session_id = %env.session_id, %error, file = name, "Grok transcript mirror skipped"); + } + } + } + if !mirrors.is_empty() { + if let Some(payload) = env.payload.as_object_mut() { + payload.insert( + "_bt_grok_transcript_mirrors".to_string(), + serde_json::Value::Object(mirrors), + ); + } + } +} + struct SessionActor { session_id: String, translator_session_id: String, @@ -753,3 +807,51 @@ pub(crate) fn is_tool_lifecycle_event(event: &str) -> bool { | "tool.execute.after" ) } + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn grok_hydration_mirrors_transcripts_and_system_prompt_at_one_boundary() { + let tmp = tempfile::tempdir().unwrap(); + let session = tmp.path().join("native-session"); + tokio::fs::create_dir(&session).await.unwrap(); + tokio::fs::write(session.join("chat_history.jsonl"), b"") + .await + .unwrap(); + tokio::fs::write(session.join("updates.jsonl"), b"{\"update\":1}\n") + .await + .unwrap(); + tokio::fs::write(session.join("events.jsonl"), b"{\"event\":1}\n") + .await + .unwrap(); + tokio::fs::write(session.join("system_prompt.txt"), b"You are Grok.") + .await + .unwrap(); + let mut env = Envelope { + source: "grok".into(), + source_version: None, + plugin_version: None, + session_id: "session-1".into(), + event: "stop".into(), + ts_ms: 1, + managed_run_id: None, + capture: None, + payload: serde_json::json!({ + "transcriptPath": session.join("chat_history.jsonl") + }), + route: None, + config: None, + }; + + hydrate_transcript_reference(tmp.path(), &mut env).await; + + let mirrors = &env.payload["_bt_grok_transcript_mirrors"]; + for name in ["updates", "events", "system_prompt"] { + let path = mirrors[name]["mirror"].as_str().unwrap(); + assert!(std::path::Path::new(path).is_file()); + assert!(mirrors[name]["through"].as_u64().unwrap() > 0); + } + } +} diff --git a/bt-daemon/src/lib.rs b/bt-daemon/src/lib.rs index f08d1f0..4831773 100644 --- a/bt-daemon/src/lib.rs +++ b/bt-daemon/src/lib.rs @@ -95,6 +95,10 @@ pub struct HookArgs { /// Optional agent version, forwarded for payload-drift handling. #[arg(long)] pub source_version: Option, + /// Optional instrumentation package version, forwarded independently from + /// the coding agent version. + #[arg(long)] + pub plugin_version: Option, /// Socket path override. #[arg(long)] pub socket: Option, @@ -292,6 +296,46 @@ pub async fn run_serve(args: ServeArgs, opts: ServeOptions) -> anyhow::Result<() server::run(args, opts).await } +fn build_hook_envelope( + args: &HookArgs, + route: SessionRoute, + payload: serde_json::Value, + session_id: String, + event: String, +) -> Envelope { + Envelope { + source: args.source.clone(), + source_version: args.source_version.clone(), + plugin_version: args.plugin_version.clone(), + session_id, + event, + ts_ms: now_ms(), + managed_run_id: std::env::var(MANAGED_RUN_ID_ENV) + .ok() + .filter(|value| !value.is_empty()), + capture: None, + payload, + route: Some(route), + config: None, + } +} + +pub(crate) fn should_flush_hook_event(event: &str, flush_on_turn_end: bool) -> bool { + matches!(event, "SessionEnd" | "session_end") + || (flush_on_turn_end + && matches!( + event, + "Stop" + | "stop" + | "StopFailure" + | "stop_failure" + | "StopCancelled" + | "stop_cancelled" + | "SubagentStop" + | "subagent_stop" + )) +} + /// Capture one hook event from `stdin` and forward it to the daemon. /// /// `route` contains only non-secret profile and destination selection. @@ -334,24 +378,11 @@ pub async fn run_hook( route.flush_mode = wire::FlushMode::FlushOnTurnEnd; } apply_additional_metadata(&mut route, args.additional_metadata.as_deref())?; - let env = Envelope { - source: args.source.clone(), - source_version: args.source_version.clone(), - plugin_version: None, - session_id, - event, - ts_ms: now_ms(), - managed_run_id: std::env::var(MANAGED_RUN_ID_ENV) - .ok() - .filter(|value| !value.is_empty()), - capture: None, - payload, - route: Some(route), - config: None, - }; + let env = build_hook_envelope(&args, route, payload, session_id, event); let socket = paths::socket_path(args.socket.as_deref()); forward_envelope(&env, &socket, &host, args.no_spawn).await?; + Ok(()) } @@ -376,6 +407,17 @@ pub(crate) fn apply_additional_metadata( Ok(()) } +fn initialize_params(env: &Envelope) -> serde_json::Value { + serde_json::json!({ + "protocol_version": PROTOCOL_VERSION, + "client": { + "source": env.source, + "plugin_version": env.plugin_version, + "pid": std::process::id() + } + }) +} + /// Ensure a daemon is up and forward one already-built [`Envelope`] to it /// (`initialize` handshake + `event.log`). Also the seam in-process clients and /// tests use to send events without going through stdin. @@ -388,17 +430,7 @@ pub async fn forward_envelope( let stream = client::ensure_daemon(socket, host, no_spawn).await?; let mut conn = client::Conn::new(stream); let initialized = conn - .request( - method::INITIALIZE, - serde_json::json!({ - "protocol_version": PROTOCOL_VERSION, - "client": { - "source": env.source, - "plugin_version": env.source_version, - "pid": std::process::id() - } - }), - ) + .request(method::INITIALIZE, initialize_params(env)) .await?; let initialized: wire::InitializeResult = serde_json::from_value(initialized)?; if initialized.daemon_version != host.version { @@ -420,18 +452,8 @@ pub async fn forward_envelope( } let stream = client::ensure_daemon(socket, host, false).await?; conn = client::Conn::new(stream); - conn.request( - method::INITIALIZE, - serde_json::json!({ - "protocol_version": PROTOCOL_VERSION, - "client": { - "source": env.source, - "plugin_version": env.source_version, - "pid": std::process::id() - } - }), - ) - .await?; + conn.request(method::INITIALIZE, initialize_params(env)) + .await?; } conn.request(method::EVENT_LOG, env).await?; Ok(()) @@ -1332,12 +1354,75 @@ mod tests { args: ServeArgs, } + #[derive(Debug, Parser)] + struct HookCli { + #[command(flatten)] + args: HookArgs, + } + #[test] fn serve_defaults_to_short_journal_backed_session_retirement() { let args = ServeCli::try_parse_from(["test"]).unwrap().args; assert_eq!(args.session_idle_timeout_secs, 30); } + #[test] + fn hook_plugin_version_reaches_envelope_and_every_initialize_attempt() { + let args = HookCli::try_parse_from([ + "test", + "--source", + "grok", + "--source-version", + "1.0.13", + "--plugin-version", + "0.1.0", + ]) + .unwrap() + .args; + assert_eq!(args.source_version.as_deref(), Some("1.0.13")); + assert_eq!(args.plugin_version.as_deref(), Some("0.1.0")); + + let payload = serde_json::json!({"native": "unchanged"}); + let env = build_hook_envelope( + &args, + SessionRoute::default(), + payload.clone(), + "session-1".into(), + "SessionEnd".into(), + ); + assert_eq!(env.source_version.as_deref(), Some("1.0.13")); + assert_eq!(env.plugin_version.as_deref(), Some("0.1.0")); + assert_eq!(env.payload, payload); + + // Both the initial connection and the post-restart retry use this + // shared parameter builder. + let initialize = initialize_params(&env); + assert_eq!(initialize["client"]["source"], "grok"); + assert_eq!(initialize["client"]["plugin_version"], "0.1.0"); + assert_ne!(initialize["client"]["plugin_version"], "1.0.13"); + } + + #[test] + fn hook_flush_recognizes_native_and_documented_terminal_events() { + for event in ["session_end", "SessionEnd"] { + assert!(should_flush_hook_event(event, false)); + } + for event in [ + "stop", + "Stop", + "stop_failure", + "StopFailure", + "stop_cancelled", + "StopCancelled", + "subagent_stop", + "SubagentStop", + ] { + assert!(!should_flush_hook_event(event, false)); + assert!(should_flush_hook_event(event, true)); + } + assert!(!should_flush_hook_event("turn_completed", true)); + } + #[test] fn additional_metadata_overrides_a_route_only_with_a_json_object() { let mut route = SessionRoute { diff --git a/bt-daemon/src/paths.rs b/bt-daemon/src/paths.rs index 98b2bed..1e3048b 100644 --- a/bt-daemon/src/paths.rs +++ b/bt-daemon/src/paths.rs @@ -103,6 +103,7 @@ pub fn agent_settings_path(source: &str, explicit: Option<&Path>) -> PathBuf { .join("opencode") .join("braintrust.json"), "pi" => home().join(".pi").join("agent").join("braintrust.json"), + "grok" => home().join(".grok").join("braintrust.json"), "antigravity" => home() .join(".gemini") .join("config") diff --git a/bt-daemon/src/server.rs b/bt-daemon/src/server.rs index c957bfc..5d9f43c 100644 --- a/bt-daemon/src/server.rs +++ b/bt-daemon/src/server.rs @@ -845,11 +845,13 @@ async fn dispatch_ingress_event(daemon: &Arc, event: PendingEvent) { return; } - let schedule_flush = event.env.event == "SessionEnd" - || (matches!( + let schedule_flush = crate::should_flush_hook_event( + &event.env.event, + matches!( event.env.route.as_ref().map(|route| route.flush_mode), Some(crate::wire::FlushMode::FlushOnTurnEnd) - ) && matches!(event.env.event.as_str(), "Stop" | "SubagentStop")); + ), + ); let flush_source = event.env.source.clone(); let flush_session_id = event.env.session_id.clone(); let journal_through = event.journal_through; diff --git a/bt-daemon/src/setup.rs b/bt-daemon/src/setup.rs index 90de93c..7909051 100644 --- a/bt-daemon/src/setup.rs +++ b/bt-daemon/src/setup.rs @@ -16,6 +16,8 @@ const CODEX_PLUGIN: &str = "trace-codex@braintrust-codex-plugins"; const CLAUDE_MARKETPLACE: &str = "braintrust-claude-plugin"; const CLAUDE_MARKETPLACE_SOURCE: &str = "braintrustdata/braintrust-claude-plugin"; const CLAUDE_PLUGIN: &str = "trace-claude-code@braintrust-claude-plugin"; +const GROK_PLUGIN: &str = "trace-grok"; +const GROK_PLUGIN_SOURCE: &str = "braintrustdata/braintrust-grok-plugin"; const OPENCODE_PLUGIN: &str = "@braintrust/trace-opencode@^1"; const PI_PLUGIN: &str = "npm:@braintrust/pi-extension@^1"; const ANTIGRAVITY_PLUGIN: &str = "braintrust-antigravity-tracing"; @@ -261,6 +263,50 @@ fn disable_claude(runner: &mut impl CommandRunner) -> anyhow::Result<()> { Ok(()) } +fn grok_plugin(value: &Value) -> Option<&Value> { + value + .as_array()? + .iter() + .find(|item| item.get("name").and_then(Value::as_str) == Some(GROK_PLUGIN)) +} + +fn grok_plugin_is_published(item: &Value) -> bool { + item.get("source") + .and_then(Value::as_str) + .is_some_and(|source| github_repo_matches(source, GROK_PLUGIN_SOURCE)) +} + +fn setup_grok(runner: &mut impl CommandRunner) -> anyhow::Result<()> { + let plugins = runner.json("grok", &["plugin", "list", "--json"])?; + // `bt trace enable grok` is the user's trust boundary. Grok's `--trust` + // applies to this plugin installation and does not change folder trust. + match grok_plugin(&plugins) { + Some(plugin) if grok_plugin_is_published(plugin) => { + runner.run("grok", &["plugin", "update", GROK_PLUGIN])?; + } + Some(_) => { + runner.run("grok", &["plugin", "uninstall", GROK_PLUGIN, "--confirm"])?; + runner.run( + "grok", + &["plugin", "install", GROK_PLUGIN_SOURCE, "--trust"], + )?; + } + None => runner.run( + "grok", + &["plugin", "install", GROK_PLUGIN_SOURCE, "--trust"], + )?, + } + runner.run("grok", &["plugin", "enable", GROK_PLUGIN]) +} + +fn disable_grok(runner: &mut impl CommandRunner) -> anyhow::Result<()> { + let plugins = runner.json("grok", &["plugin", "list", "--json"])?; + if grok_plugin(&plugins).is_some_and(grok_plugin_is_published) { + runner.run("grok", &["plugin", "uninstall", GROK_PLUGIN, "--confirm"])?; + } + Ok(()) +} + fn load_object(path: &Path) -> anyhow::Result> { match std::fs::read(path) { Ok(raw) => { @@ -534,6 +580,7 @@ pub fn run_disable(agent: SetupAgent) -> anyhow::Result { SetupAgent::Claude => disable_claude(&mut runner)?, SetupAgent::OpenCode => disable_opencode()?, SetupAgent::Pi => disable_pi(&mut runner)?, + SetupAgent::Grok => disable_grok(&mut runner)?, SetupAgent::Antigravity => disable_antigravity(&mut runner)?, } let settings_path = paths::agent_settings_path(source, None); @@ -551,6 +598,7 @@ fn agent_details(agent: SetupAgent) -> (&'static str, &'static str) { SetupAgent::Claude => ("claude", "Claude Code"), SetupAgent::OpenCode => ("opencode", "OpenCode"), SetupAgent::Pi => ("pi", "Pi"), + SetupAgent::Grok => ("grok", "Grok"), SetupAgent::Antigravity => ("antigravity", "Google Antigravity"), } } @@ -577,6 +625,10 @@ pub fn run_enable(args: EnableArgs, route: SessionRoute) -> anyhow::Result { + setup_grok(&mut runner)?; + ("grok", "Grok") + } SetupAgent::Antigravity => { setup_antigravity(&mut runner)?; ("antigravity", "Google Antigravity") @@ -807,6 +859,101 @@ mod tests { assert_eq!(std::fs::read_to_string(path).unwrap(), contents); } + #[test] + fn grok_installs_published_plugin_then_enables_it_when_missing() { + let mut runner = FakeRunner::new([serde_json::json!([ + {"name": "other-plugin", "source": "somebody/other-plugin"} + ])]); + + setup_grok(&mut runner).unwrap(); + + assert_eq!( + runner.calls, + [ + "grok plugin list --json", + "grok plugin install braintrustdata/braintrust-grok-plugin --trust", + "grok plugin enable trace-grok", + ] + ); + } + + #[test] + fn grok_repeated_enable_updates_and_enables_the_published_plugin() { + let mut runner = FakeRunner::new([serde_json::json!([ + { + "name": GROK_PLUGIN, + "source": "https://github.com/braintrustdata/braintrust-grok-plugin.git", + "status": "installed" + }, + {"name": "other-plugin", "source": "somebody/other-plugin"} + ])]); + + setup_grok(&mut runner).unwrap(); + + assert_eq!( + runner.calls, + [ + "grok plugin list --json", + "grok plugin update trace-grok", + "grok plugin enable trace-grok", + ] + ); + } + + #[test] + fn grok_enable_reconciles_only_a_conflicting_same_name_plugin() { + let mut runner = FakeRunner::new([serde_json::json!([ + { + "name": GROK_PLUGIN, + "source": "/tmp/local-trace-grok", + "status": "installed" + }, + {"name": "other-plugin", "source": "somebody/other-plugin"} + ])]); + + setup_grok(&mut runner).unwrap(); + + assert_eq!( + runner.calls, + [ + "grok plugin list --json", + "grok plugin uninstall trace-grok --confirm", + "grok plugin install braintrustdata/braintrust-grok-plugin --trust", + "grok plugin enable trace-grok", + ] + ); + } + + #[test] + fn grok_disable_removes_only_the_published_braintrust_plugin() { + let mut runner = FakeRunner::new([serde_json::json!([ + { + "name": GROK_PLUGIN, + "source": GROK_PLUGIN_SOURCE, + "status": "installed" + }, + {"name": "other-plugin", "source": "somebody/other-plugin"} + ])]); + + disable_grok(&mut runner).unwrap(); + + assert_eq!( + runner.calls, + [ + "grok plugin list --json", + "grok plugin uninstall trace-grok --confirm", + ] + ); + + let mut local = FakeRunner::new([serde_json::json!([{ + "name": GROK_PLUGIN, + "source": "/tmp/local-trace-grok", + "status": "installed" + }])]); + disable_grok(&mut local).unwrap(); + assert_eq!(local.calls, ["grok plugin list --json"]); + } + #[test] fn opencode_reconciles_the_published_plugin_and_preserves_config() { let temp = tempfile::tempdir().unwrap(); diff --git a/bt-daemon/src/trace_command.rs b/bt-daemon/src/trace_command.rs index 234aa03..07f1f32 100644 --- a/bt-daemon/src/trace_command.rs +++ b/bt-daemon/src/trace_command.rs @@ -67,6 +67,7 @@ pub enum DoctorAgent { OpenCode, Pi, Antigravity, + Grok, } impl DoctorAgent { @@ -77,6 +78,7 @@ impl DoctorAgent { Self::OpenCode => "opencode", Self::Pi => "pi", Self::Antigravity => "antigravity", + Self::Grok => "grok", } } @@ -87,6 +89,7 @@ impl DoctorAgent { Self::OpenCode => "OpenCode", Self::Pi => "Pi", Self::Antigravity => "Google Antigravity", + Self::Grok => "Grok", } } } @@ -120,6 +123,8 @@ pub enum SetupAgent { OpenCode, /// Install the published Pi tracing extension. Pi, + /// Install the published Grok tracing plugin. + Grok, /// Install the Google Antigravity tracing hooks. Antigravity, } @@ -226,6 +231,7 @@ mod tests { "antigravity", "Google Antigravity", ), + ("grok", DoctorAgent::Grok, "grok", "Grok"), ] { let cli = Cli::try_parse_from(["bt", "doctor", agent]).unwrap(); assert!(matches!( @@ -260,4 +266,37 @@ mod tests { assert!(Cli::try_parse_from(["bt", "setup", "antigravity", "--disable"]).is_err()); } + + #[test] + fn grok_uses_shared_enable_disable_and_doctor_commands() { + for command in ["enable", "setup"] { + let parsed = Cli::try_parse_from(["bt", command, "grok"]).unwrap(); + assert!(matches!( + parsed.trace.command, + TraceCommand::Setup(SetupArgs { + agent: SetupAgent::Grok, + .. + }) + )); + } + + let parsed = Cli::try_parse_from(["bt", "disable", "grok"]).unwrap(); + assert!(matches!( + parsed.trace.command, + TraceCommand::Disable(DisableArgs { + agent: SetupAgent::Grok + }) + )); + + let parsed = Cli::try_parse_from(["bt", "doctor", "grok"]).unwrap(); + assert!(matches!( + parsed.trace.command, + TraceCommand::Doctor(DoctorArgs { + agent: DoctorAgent::Grok + }) + )); + + assert_eq!(DoctorAgent::Grok.source(), "grok"); + assert_eq!(DoctorAgent::Grok.display_name(), "Grok"); + } } diff --git a/bt-daemon/src/trace_runtime.rs b/bt-daemon/src/trace_runtime.rs index 20ff59d..91a5a3b 100644 --- a/bt-daemon/src/trace_runtime.rs +++ b/bt-daemon/src/trace_runtime.rs @@ -5,7 +5,7 @@ //! behavior, setup, managed runs, imports, and output contracts stay here with //! the coding-agent integrations. -use crate::trace_command::{DoctorArgs, TraceCommand}; +use crate::trace_command::{DoctorAgent, DoctorArgs, TraceCommand}; use crate::wire::{AuthSelection, AuthSource, SessionConfig, SessionRoute}; use crate::{ apply_additional_metadata, braintrust_serve_options, paths, run_disable, run_enable, run_hook, @@ -227,6 +227,12 @@ fn print_output(output: TraceCommandOutput, format: OutputFormat) -> anyhow::Res Ok(()) } +fn plugin_activation_warning(agent: DoctorAgent, enabled: bool) -> Option<&'static str> { + (agent == DoctorAgent::Grok && enabled).then_some( + "Grok 1.0.13 requires `/reload-plugins` in each active session after plugin installation or update before its hooks become active", + ) +} + async fn doctor_output(host: &TraceHostContext, args: DoctorArgs) -> DoctorCommandOutput { let source = args.agent.source(); let settings_path = paths::agent_settings_path(source, None); @@ -244,6 +250,9 @@ async fn doctor_output(host: &TraceHostContext, args: DoctorArgs) -> DoctorComma "tracing is disabled; run `bt trace enable {source}`" )); } + if let Some(warning) = plugin_activation_warning(args.agent, enabled) { + warnings.push(warning.into()); + } let (route, route_source) = match settings.route { Some(route) => (Some(route), "settings_file".to_string()), @@ -428,12 +437,22 @@ mod tests { command: mounted, services: Arc::new(PanicHost), }; + assert_eq!( host_info(&context).serve_argv, ["/path with spaces/bt", "trace", "daemon"] ); } + #[test] + fn doctor_warns_about_grok_1_0_13_plugin_activation_after_enable() { + let warning = plugin_activation_warning(DoctorAgent::Grok, true).unwrap(); + assert!(warning.contains("Grok 1.0.13")); + assert!(warning.contains("/reload-plugins")); + assert_eq!(plugin_activation_warning(DoctorAgent::Grok, false), None); + assert_eq!(plugin_activation_warning(DoctorAgent::Codex, true), None); + } + struct PanicHost; #[async_trait] @@ -681,6 +700,7 @@ mod tests { let args = crate::HookArgs { source: "codex".into(), source_version: None, + plugin_version: None, socket: None, session_id_field: "session_id".into(), event_field: "hook_event_name".into(), diff --git a/bt-daemon/src/translate/debug.rs b/bt-daemon/src/translate/debug.rs index 199d428..4c5948e 100644 --- a/bt-daemon/src/translate/debug.rs +++ b/bt-daemon/src/translate/debug.rs @@ -48,6 +48,7 @@ impl AgentTranslator for DebugTranslator { metadata: Some(serde_json::json!({ "session_id": ctx.session_id })), metrics: None, error: None, + allow_late_merge: false, tags: None, })); } @@ -68,6 +69,7 @@ impl AgentTranslator for DebugTranslator { metadata: Some(serde_json::json!({ "seq": seq, "source": event.source })), metrics: None, error: None, + allow_late_merge: false, tags: None, })); diff --git a/bt-daemon/src/translate/grok.rs b/bt-daemon/src/translate/grok.rs new file mode 100644 index 0000000..74cb1bd --- /dev/null +++ b/bt-daemon/src/translate/grok.rs @@ -0,0 +1,1244 @@ +//! Grok transcript-authoritative translator. +//! +//! Hooks are wake-up and flush signals. `updates.jsonl` is the authoritative +//! trace data source; `events.jsonl` is independently tailed, best-effort tool +//! enrichment. Both are mirrored into daemon-owned storage at hook boundaries. + +use super::recent::{RecentMap, RecentSet}; +use super::{AgentTranslator, SessionCtx, SpanOp, SpanRow, SpanType, TranslatorFactory}; +use crate::ids; +use crate::wire::Envelope; +use serde_json::{json, Map, Value}; +use std::collections::BTreeMap; +use std::io::{Read, Seek, SeekFrom}; + +const CATCH_UP_BYTE_BUDGET: u64 = 64 * 1024; +const CATCH_UP_RECORD_BUDGET: usize = 256; +const MAX_RECORD_BYTES: usize = 1024 * 1024; +const CURSOR_TAIL_BYTES: usize = 128; +const MAX_OPEN_TOOLS: usize = 256; +const MAX_OUTPUT_CHUNKS: usize = 2_048; +const MAX_OUTPUT_BYTES: usize = 2 * 1024 * 1024; +const MAX_SYSTEM_PROMPT_BYTES: u64 = 2 * 1024 * 1024; + +pub struct GrokTranslatorFactory; + +impl TranslatorFactory for GrokTranslatorFactory { + fn source(&self) -> &str { + "grok" + } + + fn create(&self, session_id: &str) -> Box { + Box::new(GrokTranslator::new(session_id)) + } +} + +#[derive(Default)] +struct BoundedOutput { + values: Vec, + bytes: usize, + omitted_chunks: u64, + omitted_bytes: u64, +} + +impl BoundedOutput { + fn push(&mut self, value: Value) { + let bytes = serde_json::to_vec(&value) + .map(|encoded| encoded.len()) + .unwrap_or(MAX_OUTPUT_BYTES.saturating_add(1)); + if self.values.len() >= MAX_OUTPUT_CHUNKS + || self.bytes.saturating_add(bytes) > MAX_OUTPUT_BYTES + { + self.omitted_chunks = self.omitted_chunks.saturating_add(1); + self.omitted_bytes = self.omitted_bytes.saturating_add(bytes as u64); + return; + } + self.bytes += bytes; + self.values.push(value); + } + + fn is_empty(&self) -> bool { + self.values.is_empty() + } + + fn add_truncation_metadata(&self, metadata: &mut Map, prefix: &str) { + if self.omitted_chunks == 0 { + return; + } + metadata.insert(format!("{prefix}_truncated"), json!(true)); + metadata.insert( + format!("{prefix}_omitted_chunks"), + json!(self.omitted_chunks), + ); + metadata.insert(format!("{prefix}_omitted_bytes"), json!(self.omitted_bytes)); + } + + fn into_value(self) -> Value { + Value::Array(self.values) + } + + fn into_message_content(self) -> Value { + if self.values.iter().all(Value::is_string) { + let mut text = String::with_capacity(self.bytes); + for value in self.values { + if let Value::String(chunk) = value { + text.push_str(&chunk); + } + } + Value::String(text) + } else if self.values.len() == 1 { + self.values.into_iter().next().unwrap_or(Value::Null) + } else { + Value::Array(self.values) + } + } +} + +struct OpenTurn { + span_id: String, + key: String, + model: Option, + prompt_id: Option, + llm_span_ids: Vec, + assistant_output: BoundedOutput, +} + +struct OpenLlm { + span_id: String, + prompt_id: Option, + stream_start_ms: Option, + output: BoundedOutput, + reasoning: BoundedOutput, + start_ms: i64, + last_ms: i64, +} + +struct OpenTool { + span_id: String, + start_ms: i64, +} + +struct CompletedTool { + span_id: String, + end_ms: i64, +} + +#[derive(Clone, Default)] +struct TranscriptCursor { + path: Option, + offset: u64, + partial: Vec, + discarding_oversize: bool, + tail: Vec, +} + +impl TranscriptCursor { + fn reset(&mut self, path: &str) { + self.path = Some(path.to_string()); + self.offset = 0; + self.partial.clear(); + self.discarding_oversize = false; + self.tail.clear(); + } + + fn remember(&mut self, bytes: &[u8]) { + self.tail.extend_from_slice(bytes); + if self.tail.len() > CURSOR_TAIL_BYTES { + self.tail.drain(..self.tail.len() - CURSOR_TAIL_BYTES); + } + } +} + +struct TranscriptBatch { + cursor: TranscriptCursor, + records: Vec, + complete: bool, +} + +#[derive(Clone)] +struct PendingWork { + event: Envelope, +} + +struct GrokTranslator { + session_id: String, + session_span_id: String, + root_span_id: String, + root_open: bool, + root_closed: bool, + turn_seq: u32, + current_turn: Option, + open_llm: Option, + open_tools: BTreeMap, + completed_tools: RecentMap, + emitted_turns: RecentSet, + emitted_chunks: RecentSet, + updates: TranscriptCursor, + events: TranscriptCursor, + system_prompt: Option, + first_llm_span_id: Option, + first_llm_user_input: Option, + pending: Option, + last_ts_ms: i64, +} + +impl GrokTranslator { + fn new(session_id: &str) -> Self { + let root = ids::span_id(session_id, "session"); + Self { + session_id: session_id.to_string(), + session_span_id: root.clone(), + root_span_id: root, + root_open: false, + root_closed: false, + turn_seq: 0, + current_turn: None, + open_llm: None, + open_tools: BTreeMap::new(), + completed_tools: RecentMap::default(), + emitted_turns: RecentSet::default(), + emitted_chunks: RecentSet::default(), + updates: TranscriptCursor::default(), + events: TranscriptCursor::default(), + system_prompt: None, + first_llm_span_id: None, + first_llm_user_input: None, + pending: None, + last_ts_ms: 0, + } + } + + fn ensure_root( + &mut self, + ts_ms: i64, + event: &Envelope, + ctx: &SessionCtx, + ops: &mut Vec, + ) { + if self.root_open || self.root_closed { + return; + } + let attached = ctx + .config + .as_ref() + .map(|config| config.attached_span_ids()) + .unwrap_or_default(); + self.root_span_id = attached.1.unwrap_or_else(|| self.session_span_id.clone()); + self.root_open = true; + + let mut metadata = ctx + .config + .as_ref() + .and_then(|config| config.additional_metadata.as_ref()) + .and_then(Value::as_object) + .cloned() + .unwrap_or_default(); + metadata.retain(|key, _| !key.starts_with("_bt_")); + metadata.insert("source".into(), json!("grok")); + metadata.insert("session_id".into(), json!(self.session_id)); + metadata.insert("trace_source".into(), json!("session_transcript")); + for field in ["cwd", "workspaceRoot", "permissionMode", "transcriptPath"] { + if let Some(value) = event.payload.get(field) { + metadata.insert(field.into(), value.clone()); + } + } + if let Some(version) = &event.source_version { + metadata.insert("grok_version".into(), json!(version)); + } + if let Some(version) = &event.plugin_version { + metadata.insert("plugin_version".into(), json!(version)); + } + ops.push(SpanOp::Insert(SpanRow { + span_id: self.session_span_id.clone(), + root_span_id: self.root_span_id.clone(), + parent_span_ids: attached.0.into_iter().collect(), + name: "Grok".into(), + span_type: SpanType::Task, + start_ms: Some(ts_ms), + metadata: Some(Value::Object(metadata)), + ..Default::default() + })); + } + + fn process_update( + &mut self, + record: &Value, + event: &Envelope, + ctx: &SessionCtx, + ops: &mut Vec, + ) { + let Some(params) = record.get("params") else { + return; + }; + let Some(update) = params.get("update") else { + return; + }; + let Some(kind) = update.get("sessionUpdate").and_then(Value::as_str) else { + return; + }; + let ts_ms = record_ts_ms(record).unwrap_or(event.ts_ms); + + if matches!(kind, "agent_thought_chunk" | "agent_message_chunk") { + if self.current_turn.is_none() { + return; + } + let serialized = serde_json::to_string(record).unwrap_or_default(); + let chunk_key = ids::span_id(&self.session_id, &format!("chunk:{serialized}")); + if !self.emitted_chunks.insert(chunk_key) { + return; + } + } + self.last_ts_ms = self.last_ts_ms.max(ts_ms); + + match kind { + "user_message_chunk" => { + let meta = update + .get("_meta") + .or_else(|| params.get("_meta")) + .and_then(Value::as_object); + let next_number = self.turn_seq.saturating_add(1); + let turn_key = turn_key(meta, next_number); + if self.emitted_turns.contains(&turn_key) { + return; + } + self.emitted_turns.insert(turn_key.clone()); + self.close_incomplete_turn(ts_ms, "new_turn", None, false, ops); + self.ensure_root(ts_ms, event, ctx, ops); + self.turn_seq = next_number; + let span_id = ids::span_id(&self.session_id, &format!("turn:{turn_key}")); + let model = meta + .and_then(|m| m.get("modelId")) + .and_then(Value::as_str) + .map(str::to_owned); + let prompt_id = meta + .and_then(|m| m.get("promptId")) + .and_then(Value::as_str) + .map(str::to_owned); + let user_input = update.get("content").map(content_value); + if self.first_llm_span_id.is_none() { + self.first_llm_user_input = user_input.clone(); + } + ops.push(SpanOp::Insert(SpanRow { + span_id: span_id.clone(), + root_span_id: self.root_span_id.clone(), + parent_span_ids: vec![self.session_span_id.clone()], + name: format!("Turn {}", self.turn_seq), + span_type: SpanType::Task, + start_ms: Some(ts_ms), + input: user_input, + metadata: Some(json!({"transcript_turn_key": turn_key})), + ..Default::default() + })); + self.current_turn = Some(OpenTurn { + span_id, + key: turn_key, + model, + prompt_id, + llm_span_ids: Vec::new(), + assistant_output: BoundedOutput::default(), + }); + } + "agent_thought_chunk" | "agent_message_chunk" => { + if self.current_turn.is_none() { + return; + } + let meta = params + .get("_meta") + .or_else(|| update.get("_meta")) + .and_then(Value::as_object); + let prompt_id = meta + .and_then(|m| m.get("promptId")) + .and_then(Value::as_str) + .map(str::to_owned); + let stream_start_ms = meta + .and_then(|m| m.get("streamStartMs")) + .and_then(Value::as_i64); + let needs_new = self.open_llm.as_ref().is_none_or(|llm| { + identities_differ( + llm.prompt_id.as_deref(), + llm.stream_start_ms, + prompt_id.as_deref(), + stream_start_ms, + ) + }); + if needs_new { + let boundary_ms = stream_start_ms.unwrap_or(ts_ms); + self.close_llm(boundary_ms, "new_stream", ops); + let turn = self.current_turn.as_ref().expect("turn checked above"); + let sequence = turn.llm_span_ids.len() + 1; + let turn_span_id = turn.span_id.clone(); + let turn_key = turn.key.clone(); + let model = turn.model.clone().unwrap_or_else(|| "Grok".into()); + let identity_prompt = prompt_id.as_deref().unwrap_or("unknown"); + let identity_stream = stream_start_ms + .map(|value| value.to_string()) + .unwrap_or_else(|| "unknown".into()); + let span_id = ids::span_id( + &self.session_id, + &format!( + "turn:{turn_key}:llm:{identity_prompt}:{identity_stream}:{sequence}" + ), + ); + let is_first_llm = self.first_llm_span_id.is_none(); + let input = if is_first_llm { + self.first_llm_span_id = Some(span_id.clone()); + first_llm_input( + self.system_prompt.as_deref(), + self.first_llm_user_input.as_ref(), + ) + } else { + None + }; + let mut metadata = Map::new(); + metadata.insert("model".into(), json!(model.clone())); + metadata.insert("trace_source".into(), json!("session_transcript")); + metadata.insert("input_unavailable".into(), json!(true)); + metadata.insert("llm_sequence".into(), json!(sequence)); + metadata.insert("transcript_turn_key".into(), json!(turn_key)); + if is_first_llm { + add_first_llm_input_metadata( + &mut metadata, + self.system_prompt.is_some(), + self.first_llm_user_input.is_some(), + ); + } + if let Some(prompt_id) = &prompt_id { + metadata.insert("prompt_id".into(), json!(prompt_id)); + } + if let Some(stream_start_ms) = stream_start_ms { + metadata.insert("stream_start_ms".into(), json!(stream_start_ms)); + metadata.insert("boundary_source".into(), json!("streamStartMs")); + } + ops.push(SpanOp::Insert(SpanRow { + span_id: span_id.clone(), + root_span_id: self.root_span_id.clone(), + parent_span_ids: vec![turn_span_id], + name: format!("{model} call {sequence}"), + span_type: SpanType::Llm, + start_ms: Some(boundary_ms), + input, + metadata: Some(Value::Object(metadata)), + ..Default::default() + })); + self.current_turn + .as_mut() + .expect("turn checked above") + .llm_span_ids + .push(span_id.clone()); + self.open_llm = Some(OpenLlm { + span_id, + prompt_id: prompt_id.clone(), + stream_start_ms, + output: BoundedOutput::default(), + reasoning: BoundedOutput::default(), + start_ms: boundary_ms, + last_ms: ts_ms.max(boundary_ms), + }); + } else if let Some(llm) = self.open_llm.as_mut() { + if llm.prompt_id.is_none() { + llm.prompt_id = prompt_id.clone(); + } + if llm.stream_start_ms.is_none() { + llm.stream_start_ms = stream_start_ms; + } + } + if let (Some(prompt_id), Some(turn)) = (prompt_id, self.current_turn.as_mut()) { + turn.prompt_id.get_or_insert(prompt_id); + } + let is_reasoning = kind == "agent_thought_chunk"; + if let Some(content) = update.get("content") { + if let Some(llm) = self.open_llm.as_mut() { + llm.last_ms = llm.last_ms.max(ts_ms); + if is_reasoning { + llm.reasoning.push(content_value(content)); + } else { + llm.output.push(content_value(content)); + } + } + if !is_reasoning { + self.current_turn + .as_mut() + .expect("turn checked above") + .assistant_output + .push(content.clone()); + } + } + } + "tool_call" => { + self.close_llm(ts_ms, "tool_call", ops); + let Some(turn_span_id) = + self.current_turn.as_ref().map(|turn| turn.span_id.clone()) + else { + return; + }; + let Some(call_id) = update.get("toolCallId").and_then(Value::as_str) else { + return; + }; + if self.open_tools.contains_key(call_id) { + return; + } + let call_key = call_id.to_string(); + if let Some(completed) = self.completed_tools.remove(&call_key) { + self.completed_tools.insert(call_key, completed); + return; + } + self.evict_open_tool_if_needed(ts_ms, ops); + let span_id = ids::span_id(&self.session_id, &format!("tool:{call_id}")); + let name = update + .get("title") + .and_then(Value::as_str) + .unwrap_or("Tool") + .to_string(); + ops.push(SpanOp::Insert(SpanRow { + span_id: span_id.clone(), + root_span_id: self.root_span_id.clone(), + parent_span_ids: vec![turn_span_id], + name, + span_type: SpanType::Tool, + start_ms: Some(ts_ms), + input: update.get("rawInput").cloned(), + metadata: Some(json!({"tool_call_id": call_id})), + ..Default::default() + })); + self.open_tools.insert( + call_key, + OpenTool { + span_id, + start_ms: ts_ms, + }, + ); + } + "tool_call_update" => { + self.close_llm(ts_ms, "tool_update", ops); + let Some(call_id) = update.get("toolCallId").and_then(Value::as_str) else { + return; + }; + let status = update.get("status").and_then(Value::as_str); + if !matches!(status, Some("completed" | "failed" | "error" | "cancelled")) { + return; + } + let tool = if let Some(tool) = self.open_tools.remove(call_id) { + tool + } else { + let Some(turn) = self.current_turn.as_ref() else { + return; + }; + let span_id = ids::span_id(&self.session_id, &format!("tool:{call_id}")); + ops.push(SpanOp::Insert(SpanRow { + span_id: span_id.clone(), + root_span_id: self.root_span_id.clone(), + parent_span_ids: vec![turn.span_id.clone()], + name: update + .get("title") + .and_then(Value::as_str) + .unwrap_or("Tool") + .to_string(), + span_type: SpanType::Tool, + start_ms: Some(ts_ms), + input: update.get("rawInput").cloned(), + metadata: Some(json!({ + "tool_call_id": call_id, + "missing_start": true + })), + ..Default::default() + })); + OpenTool { + span_id, + start_ms: ts_ms, + } + }; + let end_ms = ts_ms.max(tool.start_ms); + self.completed_tools.insert( + call_id.to_string(), + CompletedTool { + span_id: tool.span_id.clone(), + end_ms, + }, + ); + let cancelled = status == Some("cancelled"); + let failed = matches!(status, Some("failed" | "error")); + let mut metadata = Map::new(); + if let Some(kind) = update.get("kind") { + metadata.insert("kind".into(), kind.clone()); + } + if let Some(status) = status { + metadata.insert("status".into(), json!(status)); + } + if cancelled { + metadata.insert("cancelled".into(), json!(true)); + } + ops.push(SpanOp::Merge(SpanRow { + span_id: tool.span_id, + root_span_id: self.root_span_id.clone(), + end_ms: Some(end_ms), + output: update + .get("rawOutput") + .or_else(|| update.get("content")) + .cloned(), + error: failed.then(|| format!("Grok tool {}", status.unwrap_or("failed"))), + metadata: (!metadata.is_empty()).then_some(Value::Object(metadata)), + ..Default::default() + })); + } + "turn_completed" => { + let Some(turn) = self.current_turn.as_ref() else { + return; + }; + let usage = update.get("usage"); + let metrics = usage.map(usage_metrics); + let native_model_calls = usage + .and_then(|value| value.get("modelCalls")) + .and_then(Value::as_u64); + let reconstructed_model_calls = turn.llm_span_ids.len() as u64; + if let (Some(llm_span_id), Some(metrics)) = (turn.llm_span_ids.last(), &metrics) { + ops.push(SpanOp::Merge(SpanRow { + span_id: llm_span_id.clone(), + root_span_id: self.root_span_id.clone(), + metrics: Some(metrics.clone()), + metadata: Some(json!({ + "usage_scope": "turn", + "usage_attribution": "last_llm" + })), + ..Default::default() + })); + } + let mut metadata = Map::new(); + if let Some(prompt_id) = update.get("prompt_id") { + metadata.insert("prompt_id".into(), prompt_id.clone()); + } + if let Some(stop_reason) = update.get("stop_reason") { + metadata.insert("stop_reason".into(), stop_reason.clone()); + } + if let Some(model_usage) = usage.and_then(|value| value.get("modelUsage")) { + metadata.insert("model_usage".into(), model_usage.clone()); + } + if native_model_calls.is_some_and(|native| native != reconstructed_model_calls) { + metadata.insert( + "model_call_count_mismatch".into(), + json!({ + "native": native_model_calls, + "reconstructed": reconstructed_model_calls + }), + ); + } + if let (Some(expected), Some(actual)) = ( + turn.prompt_id.as_deref(), + update.get("prompt_id").and_then(Value::as_str), + ) { + if expected != actual { + metadata.insert( + "prompt_id_mismatch".into(), + json!({"observed": expected, "completed": actual}), + ); + } + } + self.close_turn(ts_ms, "turn_completed", metadata, metrics, None, ops); + } + _ => {} + } + } + + fn process_event(&mut self, record: &Value, ops: &mut Vec) { + if record.get("type").and_then(Value::as_str) != Some("tool_completed") { + return; + } + let Some(call_id) = record.get("tool_call_id").and_then(Value::as_str) else { + return; + }; + let key = call_id.to_string(); + let Some(completed) = self.completed_tools.remove(&key) else { + return; + }; + let native_end_ms = record_ts_ms(record).unwrap_or(completed.end_ms); + let end_ms = native_end_ms.max(completed.end_ms); + let span_id = completed.span_id.clone(); + self.completed_tools.insert( + key, + CompletedTool { + span_id: completed.span_id, + end_ms, + }, + ); + let outcome = record.get("outcome").and_then(Value::as_str); + let cancelled = matches!(outcome, Some("cancelled" | "canceled")); + let failed = !cancelled && !matches!(outcome, Some("success") | None); + let mut metadata = Map::new(); + if let Some(duration) = record.get("duration_ms") { + metadata.insert("duration_ms".into(), duration.clone()); + } + if let Some(outcome) = record.get("outcome") { + metadata.insert("outcome".into(), outcome.clone()); + } + if cancelled { + metadata.insert("cancelled".into(), json!(true)); + } + ops.push(SpanOp::Merge(SpanRow { + span_id, + root_span_id: self.root_span_id.clone(), + end_ms: Some(end_ms), + metadata: Some(Value::Object(metadata)), + error: failed.then(|| format!("Grok tool outcome: {}", outcome.unwrap_or("error"))), + allow_late_merge: true, + ..Default::default() + })); + } + + fn close_llm(&mut self, end_ms: i64, reason: &str, ops: &mut Vec) { + let Some(llm) = self.open_llm.take() else { + return; + }; + let mut metadata = Map::new(); + metadata.insert("close_reason".into(), json!(reason)); + llm.output.add_truncation_metadata(&mut metadata, "output"); + llm.reasoning + .add_truncation_metadata(&mut metadata, "reasoning"); + let has_reasoning = !llm.reasoning.is_empty(); + let mut output = json!({ + "role": "assistant", + "content": llm.output.into_message_content(), + }); + if has_reasoning { + output["reasoning"] = json!([{ + "id": "reasoning", + "content": llm.reasoning.into_message_content(), + }]); + } + ops.push(SpanOp::Merge(SpanRow { + span_id: llm.span_id, + root_span_id: self.root_span_id.clone(), + end_ms: Some(end_ms.max(llm.last_ms).max(llm.start_ms)), + output: Some(Value::Array(vec![output])), + metadata: Some(Value::Object(metadata)), + ..Default::default() + })); + } + + fn evict_open_tool_if_needed(&mut self, end_ms: i64, ops: &mut Vec) { + if self.open_tools.len() < MAX_OPEN_TOOLS { + return; + } + let Some(call_id) = self.open_tools.keys().next().cloned() else { + return; + }; + let Some(tool) = self.open_tools.remove(&call_id) else { + return; + }; + let end_ms = end_ms.max(tool.start_ms); + self.completed_tools.insert( + call_id, + CompletedTool { + span_id: tool.span_id.clone(), + end_ms, + }, + ); + ops.push(SpanOp::Merge(SpanRow { + span_id: tool.span_id, + root_span_id: self.root_span_id.clone(), + end_ms: Some(end_ms), + metadata: Some(json!({ + "incomplete": true, + "close_reason": "open_tool_limit" + })), + ..Default::default() + })); + } + + fn close_tools(&mut self, end_ms: i64, reason: &str, ops: &mut Vec) { + for (call_id, tool) in std::mem::take(&mut self.open_tools) { + let end_ms = end_ms.max(tool.start_ms); + self.completed_tools.insert( + call_id, + CompletedTool { + span_id: tool.span_id.clone(), + end_ms, + }, + ); + ops.push(SpanOp::Merge(SpanRow { + span_id: tool.span_id, + root_span_id: self.root_span_id.clone(), + end_ms: Some(end_ms), + metadata: Some(json!({"incomplete": true, "close_reason": reason})), + ..Default::default() + })); + } + } + + fn close_turn( + &mut self, + end_ms: i64, + reason: &str, + mut metadata: Map, + metrics: Option, + error: Option, + ops: &mut Vec, + ) { + let end_ms = end_ms.max(self.last_ts_ms); + self.close_llm(end_ms, reason, ops); + self.close_tools(end_ms, reason, ops); + let Some(turn) = self.current_turn.take() else { + return; + }; + metadata.insert("close_reason".into(), json!(reason)); + turn.assistant_output + .add_truncation_metadata(&mut metadata, "assistant_output"); + ops.push(SpanOp::Merge(SpanRow { + span_id: turn.span_id, + root_span_id: self.root_span_id.clone(), + end_ms: Some(end_ms), + output: Some(turn.assistant_output.into_value()), + metadata: Some(Value::Object(metadata)), + metrics, + error, + ..Default::default() + })); + } + + fn close_incomplete_turn( + &mut self, + end_ms: i64, + reason: &str, + error: Option, + cancelled: bool, + ops: &mut Vec, + ) { + if self.current_turn.is_none() { + return; + } + let mut metadata = Map::new(); + metadata.insert("incomplete".into(), json!(true)); + if cancelled { + metadata.insert("cancelled".into(), json!(true)); + } + self.close_turn(end_ms, reason, metadata, None, error, ops); + } + + fn process_system_prompt(&mut self, reference: Option<&Value>, ops: &mut Vec) { + if self.system_prompt.is_some() { + return; + } + let prompt = match read_text_snapshot(reference) { + Ok(prompt) => prompt, + Err(error) => { + tracing::debug!( + session_id = %self.session_id, + error = %error, + "Grok system prompt snapshot unavailable" + ); + return; + } + }; + let Some(prompt) = prompt else { + return; + }; + let input = first_llm_input(Some(&prompt), self.first_llm_user_input.as_ref()) + .expect("system prompt produces LLM input"); + self.system_prompt = Some(prompt); + if let Some(span_id) = self.first_llm_span_id.clone() { + let mut metadata = Map::new(); + add_first_llm_input_metadata(&mut metadata, true, self.first_llm_user_input.is_some()); + ops.push(SpanOp::Merge(SpanRow { + span_id, + root_span_id: self.root_span_id.clone(), + input: Some(input), + metadata: Some(Value::Object(metadata)), + ..Default::default() + })); + } + } + + fn process_event_enrichment(&mut self, reference: Option<&Value>, ops: &mut Vec) { + let Some(reference) = reference else { + return; + }; + let events = match read_new_records(Some(reference), &self.events) { + Ok(events) => events, + Err(error) => { + tracing::debug!( + session_id = %self.session_id, + error = %error, + "Grok events transcript enrichment unavailable" + ); + return; + } + }; + self.events = events.cursor; + for record in events.records { + self.process_event(&record, ops); + } + } + + fn process_transcripts( + &mut self, + event: &Envelope, + ctx: &SessionCtx, + ) -> anyhow::Result<(Vec, bool)> { + let mut ops = Vec::new(); + let Some(mirrors) = event.payload.get("_bt_grok_transcript_mirrors") else { + return Ok((ops, true)); + }; + + // Updates are authoritative: their read and cursor must succeed before + // any translator state advances. Events can enrich completed tools but + // cannot delay update catch-up, terminal handling, or flush. + let updates = read_new_records(mirrors.get("updates"), &self.updates)?; + let complete = updates.complete; + self.updates = updates.cursor; + self.process_system_prompt(mirrors.get("system_prompt"), &mut ops); + for record in updates.records { + self.process_update(&record, event, ctx, &mut ops); + } + self.process_event_enrichment(mirrors.get("events"), &mut ops); + Ok((ops, complete)) + } + + fn finish_hook(&mut self, event: &Envelope, ops: &mut Vec) { + let terminal_ms = self.last_ts_ms.max(event.ts_ms); + match event.event.as_str() { + // Transcript `turn_completed` is authoritative. A passive stop hook + // can race the final transcript write, so closing here would make + // later catch-up records lose their turn parent. + "Stop" | "stop" => {} + "StopFailure" | "stop_failure" => { + self.last_ts_ms = terminal_ms; + self.close_incomplete_turn( + terminal_ms, + event.event.as_str(), + Some("Grok turn failed".into()), + false, + ops, + ); + } + "StopCancelled" | "stop_cancelled" => { + self.last_ts_ms = terminal_ms; + self.close_incomplete_turn(terminal_ms, event.event.as_str(), None, true, ops); + } + "SessionEnd" | "session_end" => { + self.last_ts_ms = terminal_ms; + self.close_incomplete_turn(terminal_ms, event.event.as_str(), None, false, ops); + if self.root_open { + self.root_open = false; + self.root_closed = true; + ops.push(SpanOp::Merge(SpanRow { + span_id: self.session_span_id.clone(), + root_span_id: self.root_span_id.clone(), + end_ms: Some(terminal_ms), + ..Default::default() + })); + } + } + _ => {} + } + } +} + +impl AgentTranslator for GrokTranslator { + fn handle(&mut self, event: &Envelope, ctx: &SessionCtx) -> anyhow::Result> { + anyhow::ensure!( + self.pending.is_none(), + "Grok translator has pending catch-up work; drain it before handling another event" + ); + let (mut ops, complete) = self.process_transcripts(event, ctx)?; + if complete { + self.finish_hook(event, &mut ops); + } else { + self.pending = Some(PendingWork { + event: event.clone(), + }); + } + Ok(ops) + } + + fn drain_pending(&mut self, ctx: &SessionCtx) -> anyhow::Result>> { + let Some(pending) = self.pending.clone() else { + return Ok(None); + }; + let (mut ops, complete) = self.process_transcripts(&pending.event, ctx)?; + if complete { + self.pending = None; + self.finish_hook(&pending.event, &mut ops); + } + Ok(Some(ops)) + } + + fn flush(&mut self, _ctx: &SessionCtx) -> anyhow::Result> { + anyhow::ensure!( + self.pending.is_none(), + "Grok translator has pending catch-up work; drain it before flushing" + ); + let mut ops = Vec::new(); + self.close_incomplete_turn(self.last_ts_ms, "flush", None, false, &mut ops); + if self.root_open { + self.root_open = false; + self.root_closed = true; + ops.push(SpanOp::Merge(SpanRow { + span_id: self.session_span_id.clone(), + root_span_id: self.root_span_id.clone(), + end_ms: Some(self.last_ts_ms), + ..Default::default() + })); + } + Ok(ops) + } +} + +fn first_llm_input(system_prompt: Option<&str>, user_message: Option<&Value>) -> Option { + let mut messages = Vec::with_capacity(2); + if let Some(system_prompt) = system_prompt { + messages.push(json!({"role": "system", "content": system_prompt})); + } + if let Some(user_message) = user_message { + messages.push(json!({"role": "user", "content": user_message})); + } + (!messages.is_empty()).then_some(Value::Array(messages)) +} + +fn add_first_llm_input_metadata( + metadata: &mut Map, + has_system_prompt: bool, + has_user_message: bool, +) { + if has_system_prompt { + metadata.insert("system_prompt_included".into(), json!(true)); + } + if has_user_message { + metadata.insert("user_message_included".into(), json!(true)); + } + let scope = match (has_system_prompt, has_user_message) { + (true, true) => Some("system_and_user"), + (true, false) => Some("system_prompt_only"), + (false, true) => Some("user_message_only"), + (false, false) => None, + }; + if let Some(scope) = scope { + metadata.insert("input_scope".into(), json!(scope)); + } +} + +fn read_text_snapshot(reference: Option<&Value>) -> anyhow::Result> { + let Some(reference) = reference else { + return Ok(None); + }; + let Some(path) = reference + .get("mirror") + .and_then(Value::as_str) + .or_else(|| reference.get("path").and_then(Value::as_str)) + else { + return Ok(None); + }; + let mut file = std::fs::File::open(path)?; + let file_len = file.metadata()?.len(); + let through = reference + .get("through") + .and_then(Value::as_u64) + .unwrap_or(file_len); + anyhow::ensure!( + file_len >= through, + "Grok system prompt mirror {path} is shorter than captured boundary {through}" + ); + anyhow::ensure!( + through <= MAX_SYSTEM_PROMPT_BYTES, + "Grok system prompt exceeds {MAX_SYSTEM_PROMPT_BYTES} bytes" + ); + let mut bytes = vec![0; through as usize]; + file.read_exact(&mut bytes)?; + let prompt = String::from_utf8(bytes)?; + Ok((!prompt.is_empty()).then_some(prompt)) +} + +fn read_new_records( + reference: Option<&Value>, + cursor: &TranscriptCursor, +) -> anyhow::Result { + let Some(reference) = reference else { + return Ok(TranscriptBatch { + cursor: cursor.clone(), + records: Vec::new(), + complete: true, + }); + }; + let Some(path) = reference + .get("mirror") + .and_then(Value::as_str) + .or_else(|| reference.get("path").and_then(Value::as_str)) + else { + return Ok(TranscriptBatch { + cursor: cursor.clone(), + records: Vec::new(), + complete: true, + }); + }; + + let captured_through = reference.get("through").and_then(Value::as_u64); + let mut file = match std::fs::File::open(path) { + Ok(file) => file, + Err(error) + if error.kind() == std::io::ErrorKind::NotFound && captured_through == Some(0) => + { + let mut next = cursor.clone(); + if next.path.as_deref() != Some(path) || next.offset != 0 { + next.reset(path); + } + return Ok(TranscriptBatch { + cursor: next, + records: Vec::new(), + complete: true, + }); + } + Err(error) => return Err(error.into()), + }; + let file_len = file.metadata()?.len(); + let through = captured_through.unwrap_or(file_len); + anyhow::ensure!( + file_len >= through, + "Grok transcript mirror {path} is shorter than captured boundary {through}" + ); + + let mut next = cursor.clone(); + if next.path.as_deref() != Some(path) + || next.offset > through + || !cursor_tail_matches(&mut file, &next)? + { + next.reset(path); + } + if next.offset == through { + return Ok(TranscriptBatch { + cursor: next, + records: Vec::new(), + complete: true, + }); + } + + file.seek(SeekFrom::Start(next.offset))?; + let read_len = (through - next.offset).min(CATCH_UP_BYTE_BUDGET) as usize; + let mut bytes = vec![0; read_len]; + file.read_exact(&mut bytes)?; + + let mut records = Vec::new(); + let mut completed_lines = 0usize; + let mut consumed = 0usize; + for byte in bytes.iter().copied() { + consumed += 1; + if next.discarding_oversize { + if byte == b'\n' { + next.discarding_oversize = false; + completed_lines += 1; + } + } else if byte == b'\n' { + if next.partial.last() == Some(&b'\r') { + next.partial.pop(); + } + if let Ok(record) = serde_json::from_slice(&next.partial) { + records.push(record); + } + next.partial.clear(); + completed_lines += 1; + } else if next.partial.len() < MAX_RECORD_BYTES { + next.partial.push(byte); + } else { + next.partial.clear(); + next.discarding_oversize = true; + } + + if completed_lines >= CATCH_UP_RECORD_BUDGET { + break; + } + } + next.offset = next.offset.saturating_add(consumed as u64); + next.remember(&bytes[..consumed]); + + Ok(TranscriptBatch { + complete: next.offset == through, + cursor: next, + records, + }) +} + +fn cursor_tail_matches( + file: &mut std::fs::File, + cursor: &TranscriptCursor, +) -> anyhow::Result { + if cursor.offset == 0 || cursor.tail.is_empty() { + return Ok(true); + } + if cursor.offset < cursor.tail.len() as u64 { + return Ok(false); + } + file.seek(SeekFrom::Start(cursor.offset - cursor.tail.len() as u64))?; + let mut observed = vec![0; cursor.tail.len()]; + if file.read_exact(&mut observed).is_err() { + return Ok(false); + } + Ok(observed == cursor.tail) +} + +fn turn_key(meta: Option<&Map>, sequence: u32) -> String { + if let Some(prompt_id) = meta + .and_then(|value| value.get("promptId")) + .and_then(Value::as_str) + { + return format!("prompt:{prompt_id}"); + } + if let Some(prompt_index) = meta.and_then(|value| value.get("promptIndex")) { + return format!("prompt-index:{prompt_index}"); + } + format!("sequence:{sequence}") +} + +fn identities_differ( + open_prompt_id: Option<&str>, + open_stream_start_ms: Option, + prompt_id: Option<&str>, + stream_start_ms: Option, +) -> bool { + matches!((open_prompt_id, prompt_id), (Some(open), Some(next)) if open != next) + || matches!( + (open_stream_start_ms, stream_start_ms), + (Some(open), Some(next)) if open != next + ) +} + +fn record_ts_ms(record: &Value) -> Option { + record + .pointer("/params/_meta/agentTimestampMs") + .and_then(Value::as_i64) + .or_else(|| { + record + .get("ts") + .and_then(Value::as_str) + .and_then(|ts| chrono::DateTime::parse_from_rfc3339(ts).ok()) + .map(|ts| ts.timestamp_millis()) + }) + .or_else(|| { + record + .get("timestamp") + .and_then(Value::as_i64) + .map(|seconds| seconds.saturating_mul(1000)) + }) +} + +fn content_value(content: &Value) -> Value { + content + .get("text") + .cloned() + .unwrap_or_else(|| content.clone()) +} + +fn usage_metrics(usage: &Value) -> Value { + let mut metrics = Map::new(); + for (native, common) in [ + ("inputTokens", "prompt_tokens"), + ("outputTokens", "completion_tokens"), + ("totalTokens", "tokens"), + ("cachedReadTokens", "prompt_cached_tokens"), + ("cacheCreationTokens", "prompt_cache_creation_tokens"), + ("reasoningTokens", "completion_reasoning_tokens"), + ("apiDurationMs", "api_duration_ms"), + ("costUsdTicks", "cost_usd_ticks"), + ("modelCalls", "model_calls"), + ] { + if let Some(value) = usage.get(native) { + metrics.insert(common.into(), value.clone()); + } + } + Value::Object(metrics) +} diff --git a/bt-daemon/src/translate/mod.rs b/bt-daemon/src/translate/mod.rs index a193502..43c3037 100644 --- a/bt-daemon/src/translate/mod.rs +++ b/bt-daemon/src/translate/mod.rs @@ -12,6 +12,7 @@ mod claude; mod codex; mod debug; mod git; +mod grok; mod opencode; mod pi; mod recent; @@ -21,6 +22,7 @@ pub use antigravity::AntigravityTranslatorFactory; pub use claude::ClaudeTranslatorFactory; pub use codex::CodexTranslatorFactory; pub use debug::DebugTranslatorFactory; +pub use grok::GrokTranslatorFactory; pub use opencode::OpenCodeTranslatorFactory; pub use pi::PiTranslatorFactory; @@ -63,6 +65,11 @@ pub struct SpanRow { pub metrics: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub error: Option, + /// Sink-only directive for a deterministic enrichment that may arrive + /// after this span's terminal row was durably delivered. + #[serde(skip)] + #[doc(hidden)] + pub allow_late_merge: bool, /// Labels for filtering in Braintrust (e.g. `compaction`, `permission-request`). #[serde(default, skip_serializing_if = "Option::is_none")] pub tags: Option>, @@ -145,6 +152,7 @@ impl Registry { r.register(Box::new(AntigravityTranslatorFactory::new(git.clone()))); r.register(Box::new(ClaudeTranslatorFactory::new(git.clone()))); r.register(Box::new(CodexTranslatorFactory::new(git.clone()))); + r.register(Box::new(GrokTranslatorFactory)); r.register(Box::new(OpenCodeTranslatorFactory::new(git.clone()))); r.register(Box::new(PiTranslatorFactory::new(git))); r @@ -168,6 +176,7 @@ impl Registry { "open-code" | "opencode" => "opencode", "antigravity" => "antigravity", "codex" => "codex", + "grok" => "grok", "pi" => "pi", "debug" => "debug", _ => return None, diff --git a/bt-daemon/tests/braintrust_sink.rs b/bt-daemon/tests/braintrust_sink.rs index b1d054d..ef0e9cc 100644 --- a/bt-daemon/tests/braintrust_sink.rs +++ b/bt-daemon/tests/braintrust_sink.rs @@ -51,6 +51,7 @@ fn row( metadata: None, metrics: None, error: None, + allow_late_merge: false, tags: None, } } diff --git a/bt-daemon/tests/fixtures/grok/transcript/edge-events.jsonl b/bt-daemon/tests/fixtures/grok/transcript/edge-events.jsonl new file mode 100644 index 0000000..680680f --- /dev/null +++ b/bt-daemon/tests/fixtures/grok/transcript/edge-events.jsonl @@ -0,0 +1 @@ +{"ts":"1970-01-01T00:00:02.210Z","type":"tool_completed","tool_call_id":"cancelled-call","duration_ms":10,"outcome":"cancelled"} diff --git a/bt-daemon/tests/fixtures/grok/transcript/edge-updates.jsonl b/bt-daemon/tests/fixtures/grok/transcript/edge-updates.jsonl new file mode 100644 index 0000000..2e442a5 --- /dev/null +++ b/bt-daemon/tests/fixtures/grok/transcript/edge-updates.jsonl @@ -0,0 +1,12 @@ +{"params":{"update":{"sessionUpdate":"user_message_chunk","content":{"text":"first"}},"_meta":{"modelId":"grok-edge","promptIndex":0,"agentTimestampMs":1000}}} +{"params":{"update":{"sessionUpdate":"user_message_chunk","content":{"text":"first"}},"_meta":{"modelId":"grok-edge","promptIndex":0,"agentTimestampMs":1000}}} +{"params":{"update":{"sessionUpdate":"agent_thought_chunk","content":{"text":"thinking"}},"_meta":{"promptId":"prompt-a","agentTimestampMs":1010,"chunkId":1}}} +{"params":{"update":{"sessionUpdate":"agent_message_chunk","content":{"text":"answer-a"}},"_meta":{"promptId":"prompt-a","agentTimestampMs":1020,"chunkId":2}}} +{malformed complete record} +{"params":{"update":{"sessionUpdate":"agent_message_chunk","content":{"text":"answer-b"}},"_meta":{"promptId":"prompt-b","agentTimestampMs":1040,"chunkId":3}}} +{"params":{"update":{"sessionUpdate":"turn_completed","prompt_id":"prompt-b","stop_reason":"end_turn","usage":{"inputTokens":30,"outputTokens":6,"totalTokens":36,"modelCalls":3}},"_meta":{"agentTimestampMs":1050}}} +{"params":{"update":{"sessionUpdate":"user_message_chunk","content":{"text":"second"}},"_meta":{"modelId":"grok-edge","promptIndex":1,"agentTimestampMs":2000}}} +{"params":{"update":{"sessionUpdate":"agent_message_chunk","content":{"text":"second answer"}},"_meta":{"promptId":"prompt-c","streamStartMs":2100,"agentTimestampMs":2110,"chunkId":4}}} +{"params":{"update":{"sessionUpdate":"tool_call","toolCallId":"cancelled-call","title":"shell","rawInput":{"command":"sleep 1"}},"_meta":{"agentTimestampMs":2200}}} +{"params":{"update":{"sessionUpdate":"tool_call_update","toolCallId":"cancelled-call","status":"cancelled","rawOutput":"interrupted"},"_meta":{"agentTimestampMs":2210}}} +{"params":{"update":{"sessionUpdate":"turn_completed","prompt_id":"prompt-c","stop_reason":"cancelled","usage":{"inputTokens":10,"outputTokens":2,"totalTokens":12,"modelCalls":1}},"_meta":{"agentTimestampMs":2300}}} diff --git a/bt-daemon/tests/fixtures/grok/transcript/events.jsonl b/bt-daemon/tests/fixtures/grok/transcript/events.jsonl new file mode 100644 index 0000000..2779c2a --- /dev/null +++ b/bt-daemon/tests/fixtures/grok/transcript/events.jsonl @@ -0,0 +1,3 @@ +{"ts":"1970-01-01T00:00:01.200Z","type":"tool_started","tool_name":"read_file"} +{"ts":"1970-01-01T00:00:01.300Z","type":"tool_completed","tool_call_id":"call-1","tool_name":"read_file","duration_ms":100,"outcome":"success"} +{"ts":"1970-01-01T00:00:01.500Z","type":"turn_ended","outcome":"completed"} diff --git a/bt-daemon/tests/fixtures/grok/transcript/partial-updates.jsonl b/bt-daemon/tests/fixtures/grok/transcript/partial-updates.jsonl new file mode 100644 index 0000000..1909e60 --- /dev/null +++ b/bt-daemon/tests/fixtures/grok/transcript/partial-updates.jsonl @@ -0,0 +1,2 @@ +{"params":{"update":{"sessionUpdate":"user_message_chunk","content":{"text":"partial"}},"_meta":{"modelId":"grok-edge","promptIndex":0,"agentTimestampMs":1000}}} +{"params":{"update":{"sessionUpdate":"agent_message_chunk","content":{"text":"wait"}},"_meta":{"promptId":"partial-prompt" \ No newline at end of file diff --git a/bt-daemon/tests/fixtures/grok/transcript/system_prompt.txt b/bt-daemon/tests/fixtures/grok/transcript/system_prompt.txt new file mode 100644 index 0000000..6926e48 --- /dev/null +++ b/bt-daemon/tests/fixtures/grok/transcript/system_prompt.txt @@ -0,0 +1 @@ +You are Grok, a test coding assistant. diff --git a/bt-daemon/tests/fixtures/grok/transcript/updates.jsonl b/bt-daemon/tests/fixtures/grok/transcript/updates.jsonl new file mode 100644 index 0000000..b602046 --- /dev/null +++ b/bt-daemon/tests/fixtures/grok/transcript/updates.jsonl @@ -0,0 +1,8 @@ +{"timestamp":1,"params":{"sessionId":"grok-session","update":{"sessionUpdate":"user_message_chunk","content":{"type":"text","text":"inspect the fixture"},"_meta":{"modelId":"grok-4.6-build","promptIndex":0}},"_meta":{"agentTimestampMs":1000}}} +{"timestamp":1,"params":{"sessionId":"grok-session","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":"I will read it."}},"_meta":{"promptId":"prompt-1","streamStartMs":1100,"agentTimestampMs":1110,"chunkId":1}}} +{"timestamp":1,"params":{"sessionId":"grok-session","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"Reading the fixture."}},"_meta":{"promptId":"prompt-1","streamStartMs":1100,"agentTimestampMs":1120,"chunkId":2}}} +{"timestamp":1,"params":{"sessionId":"grok-session","update":{"sessionUpdate":"tool_call","toolCallId":"call-1","title":"read_file","rawInput":{"path":"fixture.txt"}},"_meta":{"agentTimestampMs":1200}}} +{"timestamp":1,"params":{"sessionId":"grok-session","update":{"sessionUpdate":"tool_call_update","toolCallId":"call-1","kind":"read","status":"completed","rawOutput":"fixture contents"},"_meta":{"promptId":"prompt-1","agentTimestampMs":1300}}} +{"timestamp":1,"params":{"sessionId":"grok-session","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":"The read succeeded."}},"_meta":{"promptId":"prompt-1","streamStartMs":1400,"agentTimestampMs":1410,"chunkId":3}}} +{"timestamp":1,"params":{"sessionId":"grok-session","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"Done."}},"_meta":{"promptId":"prompt-1","streamStartMs":1400,"agentTimestampMs":1420,"chunkId":4}}} +{"timestamp":1,"params":{"sessionId":"grok-session","update":{"sessionUpdate":"turn_completed","prompt_id":"prompt-1","stop_reason":"end_turn","usage":{"inputTokens":100,"outputTokens":20,"totalTokens":120,"cachedReadTokens":40,"cacheCreationTokens":5,"reasoningTokens":7,"modelCalls":2,"apiDurationMs":400,"costUsdTicks":12,"modelUsage":{"grok-4.6-build":{"inputTokens":100,"outputTokens":20,"modelCalls":2}}}},"_meta":{"agentTimestampMs":1500}}} diff --git a/bt-daemon/tests/grok_translator.rs b/bt-daemon/tests/grok_translator.rs new file mode 100644 index 0000000..8fca6c6 --- /dev/null +++ b/bt-daemon/tests/grok_translator.rs @@ -0,0 +1,631 @@ +use bt_daemon::wire::Envelope; +use bt_daemon::{AgentTranslator, Registry, SessionCtx, SpanOp, SpanType}; +use serde_json::json; +use std::path::{Path, PathBuf}; + +fn fixture(name: &str) -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/grok/transcript") + .join(name) +} + +fn envelope(updates_through: u64, events_through: u64) -> Envelope { + Envelope { + source: "grok".into(), + source_version: Some("1.0.13".into()), + plugin_version: Some("0.1.0".into()), + session_id: "grok-session".into(), + event: "Wake".into(), + ts_ms: 1_600, + managed_run_id: None, + capture: None, + payload: json!({ + "cwd": "/repo", + "workspaceRoot": "/repo", + "transcriptPath": "/native/session/chat_history.jsonl", + "_bt_grok_transcript_mirrors": { + "updates": { + "mirror": fixture("updates.jsonl"), + "through": updates_through + }, + "events": { + "mirror": fixture("events.jsonl"), + "through": events_through + }, + "system_prompt": { + "mirror": fixture("system_prompt.txt"), + "through": std::fs::metadata(fixture("system_prompt.txt")).unwrap().len() + } + } + }), + route: None, + config: None, + } +} + +fn ctx() -> SessionCtx { + SessionCtx { + session_id: "grok-session".into(), + config: None, + } +} + +fn point_at(event: &mut Envelope, updates: &Path, events: &Path) { + event.payload["_bt_grok_transcript_mirrors"]["updates"]["mirror"] = json!(updates); + event.payload["_bt_grok_transcript_mirrors"]["updates"]["through"] = + json!(std::fs::metadata(updates).unwrap().len()); + event.payload["_bt_grok_transcript_mirrors"]["events"]["mirror"] = json!(events); + event.payload["_bt_grok_transcript_mirrors"]["events"]["through"] = + json!(std::fs::metadata(events).unwrap().len()); +} + +fn drain(translator: &mut dyn AgentTranslator) -> Vec { + let mut ops = Vec::new(); + while let Some(batch) = translator.drain_pending(&ctx()).unwrap() { + ops.extend(batch); + } + ops +} + +#[test] +fn grok_transcript_builds_turn_llm_and_tool_spans_with_aggregate_usage() { + let updates = std::fs::metadata(fixture("updates.jsonl")).unwrap().len(); + let events = std::fs::metadata(fixture("events.jsonl")).unwrap().len(); + let registry = Registry::default_agents(); + assert!(registry.sources().contains(&"grok".to_string())); + let mut translator = registry.create("grok", "grok-session"); + let ops = translator + .handle(&envelope(updates, events), &ctx()) + .unwrap(); + + let inserts: Vec<_> = ops + .iter() + .filter_map(|op| match op { + SpanOp::Insert(row) => Some(row), + _ => None, + }) + .collect(); + assert_eq!(inserts.len(), 5); + let root = inserts.iter().find(|row| row.name == "Grok").unwrap(); + assert_eq!(root.span_type, SpanType::Task); + assert_eq!( + root.metadata.as_ref().unwrap()["trace_source"], + "session_transcript" + ); + + let turn = inserts.iter().find(|row| row.name == "Turn 1").unwrap(); + assert_eq!(turn.input, Some(json!("inspect the fixture"))); + assert_eq!(turn.parent_span_ids, vec![root.span_id.clone()]); + + let llms: Vec<_> = inserts + .iter() + .filter(|row| row.span_type == SpanType::Llm) + .collect(); + assert_eq!(llms.len(), 2); + assert_eq!(llms[0].name, "grok-4.6-build call 1"); + assert_eq!(llms[1].name, "grok-4.6-build call 2"); + assert!(llms + .iter() + .all(|row| row.parent_span_ids == vec![turn.span_id.clone()])); + let system_prompt = std::fs::read_to_string(fixture("system_prompt.txt")).unwrap(); + assert_eq!( + llms[0].input, + Some(json!([ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": "inspect the fixture"} + ])) + ); + assert!(llms[1].input.is_none()); + assert!(llms.iter().all(|row| { + row.metadata.as_ref().is_some_and(|metadata| { + metadata["trace_source"] == "session_transcript" + && metadata["input_unavailable"] == true + && metadata["boundary_source"] == "streamStartMs" + }) + })); + assert_eq!( + llms[0].metadata.as_ref().unwrap()["system_prompt_included"], + true + ); + assert_eq!( + llms[0].metadata.as_ref().unwrap()["user_message_included"], + true + ); + assert_eq!( + llms[0].metadata.as_ref().unwrap()["input_scope"], + "system_and_user" + ); + assert!(llms[1] + .metadata + .as_ref() + .unwrap() + .get("system_prompt_included") + .is_none()); + let first_llm_close = ops + .iter() + .find_map(|op| match op { + SpanOp::Merge(row) if row.span_id == llms[0].span_id && row.output.is_some() => { + Some(row) + } + _ => None, + }) + .unwrap(); + assert_eq!( + first_llm_close.output, + Some(json!([{ + "role": "assistant", + "content": "Reading the fixture.", + "reasoning": [{ + "id": "reasoning", + "content": "I will read it." + }] + }])) + ); + + let tool = inserts + .iter() + .find(|row| row.span_type == SpanType::Tool) + .unwrap(); + assert_eq!(tool.name, "read_file"); + assert_eq!(tool.input, Some(json!({"path": "fixture.txt"}))); + assert_eq!(tool.parent_span_ids, vec![turn.span_id.clone()]); + + let tool_merges: Vec<_> = ops + .iter() + .filter_map(|op| match op { + SpanOp::Merge(row) if row.span_id == tool.span_id => Some(row), + _ => None, + }) + .collect(); + assert_eq!(tool_merges.len(), 2); + assert_eq!(tool_merges[0].output, Some(json!("fixture contents"))); + assert_eq!( + tool_merges[1].metadata.as_ref().unwrap()["duration_ms"], + 100 + ); + + let turn_merge = ops + .iter() + .find_map(|op| match op { + SpanOp::Merge(row) if row.span_id == turn.span_id => Some(row), + _ => None, + }) + .unwrap(); + assert_eq!( + turn_merge.output, + Some(json!([ + {"type": "text", "text": "Reading the fixture."}, + {"type": "text", "text": "Done."} + ])) + ); + let metrics = turn_merge.metrics.as_ref().unwrap(); + assert_eq!(metrics["prompt_tokens"], 100); + assert_eq!(metrics["completion_tokens"], 20); + assert_eq!(metrics["prompt_cached_tokens"], 40); + assert_eq!(metrics["model_calls"], 2); + assert_eq!(metrics["cost_usd_ticks"], 12); + assert!(metrics.get("estimated_cost").is_none()); + assert!(!ops.iter().any( + |op| matches!(op, SpanOp::Merge(row) if row.span_id == llms[0].span_id && row.metrics.is_some()), + )); + let last_llm_usage = ops + .iter() + .find_map(|op| match op { + SpanOp::Merge(row) if row.span_id == llms[1].span_id && row.metrics.is_some() => { + Some(row) + } + _ => None, + }) + .expect("turn aggregate usage must be attributed to the final LLM"); + assert_eq!(last_llm_usage.metrics.as_ref(), Some(metrics)); + assert_eq!( + last_llm_usage.metadata.as_ref().unwrap(), + &json!({ + "usage_scope": "turn", + "usage_attribution": "last_llm" + }) + ); +} + +#[test] +fn grok_transcript_reads_incrementally_and_replay_is_deterministic() { + let updates_path = fixture("updates.jsonl"); + let bytes = std::fs::read(&updates_path).unwrap(); + let first_through = bytes + .iter() + .enumerate() + .filter(|(_, byte)| **byte == b'\n') + .nth(4) + .map(|(index, _)| index as u64 + 1) + .unwrap(); + let full_through = bytes.len() as u64; + let events = std::fs::metadata(fixture("events.jsonl")).unwrap().len(); + + let registry = Registry::default_agents(); + let mut incremental = registry.create("grok", "grok-session"); + let first = incremental + .handle(&envelope(first_through, 0), &ctx()) + .unwrap(); + let second = incremental + .handle(&envelope(full_through, events), &ctx()) + .unwrap(); + assert!(incremental + .handle(&envelope(full_through, events), &ctx()) + .unwrap() + .is_empty()); + + let mut replay = registry.create("grok", "grok-session"); + let all = replay + .handle(&envelope(full_through, events), &ctx()) + .unwrap(); + let incremental_json = + serde_json::to_value(first.into_iter().chain(second).collect::>()).unwrap(); + assert_eq!(incremental_json, serde_json::to_value(all).unwrap()); +} + +#[test] +fn grok_single_call_usage_and_missing_tool_start_are_recovered() { + let temp = tempfile::tempdir().unwrap(); + let updates_path = temp.path().join("updates.jsonl"); + let events_path = temp.path().join("events.jsonl"); + std::fs::write( + &updates_path, + concat!( + "{\"params\":{\"update\":{\"sessionUpdate\":\"user_message_chunk\",\"content\":{\"text\":\"hello\"},\"_meta\":{\"modelId\":\"grok-4.6\",\"promptIndex\":0}},\"_meta\":{\"agentTimestampMs\":1000}}}\n", + "{\"params\":{\"update\":{\"sessionUpdate\":\"agent_message_chunk\",\"content\":{\"text\":\"working\"}},\"_meta\":{\"promptId\":\"p1\",\"streamStartMs\":1100,\"agentTimestampMs\":1110}}}\n", + "{\"params\":{\"update\":{\"sessionUpdate\":\"tool_call_update\",\"toolCallId\":\"missing\",\"title\":\"read_file\",\"rawInput\":{\"path\":\"x\"},\"rawOutput\":\"y\",\"status\":\"completed\"},\"_meta\":{\"agentTimestampMs\":1200}}}\n", + "{\"params\":{\"update\":{\"sessionUpdate\":\"turn_completed\",\"prompt_id\":\"p1\",\"stop_reason\":\"end_turn\",\"usage\":{\"inputTokens\":10,\"outputTokens\":2,\"totalTokens\":12,\"modelCalls\":1,\"costUsdTicks\":25000000,\"modelUsage\":{\"grok-4.6-build\":{\"inputTokens\":10,\"outputTokens\":2,\"totalTokens\":12,\"modelCalls\":1,\"costUsdTicks\":25000000}}}},\"_meta\":{\"agentTimestampMs\":1300}}}\n" + ), + ) + .unwrap(); + std::fs::write(&events_path, "").unwrap(); + let mut event = envelope(0, 0); + point_at(&mut event, &updates_path, &events_path); + + let registry = Registry::default_agents(); + let mut translator = registry.create("grok", "grok-session"); + let ops = translator.handle(&event, &ctx()).unwrap(); + let llm = ops + .iter() + .find_map(|op| match op { + SpanOp::Insert(row) if row.span_type == SpanType::Llm => Some(row), + _ => None, + }) + .unwrap(); + assert_eq!(llm.name, "grok-4.6 call 1"); + assert!(ops.iter().any(|op| { + matches!(op, SpanOp::Merge(row) if row.span_id == llm.span_id + && row.metrics.as_ref().is_some_and(|metrics| metrics["tokens"] == 12 + && metrics["cost_usd_ticks"] == 25000000 + && metrics.get("estimated_cost").is_none())) + })); + assert!(ops.iter().any(|op| { + matches!(op, SpanOp::Insert(row) if row.span_type == SpanType::Tool && row.metadata.as_ref().is_some_and(|metadata| metadata["missing_start"] == true)) + })); +} + +#[test] +fn grok_edge_fixture_preserves_turns_skips_malformed_records_and_marks_mismatches() { + let updates = fixture("edge-updates.jsonl"); + let events = fixture("edge-events.jsonl"); + let mut event = envelope(0, 0); + point_at(&mut event, &updates, &events); + let registry = Registry::default_agents(); + let mut translator = registry.create("grok", "grok-session"); + let ops = translator.handle(&event, &ctx()).unwrap(); + + let turns: Vec<_> = ops + .iter() + .filter_map(|op| match op { + SpanOp::Insert(row) if row.name.starts_with("Turn ") => Some(row), + _ => None, + }) + .collect(); + assert_eq!( + turns.len(), + 2, + "duplicate user chunk must not mutate turn state" + ); + let llms: Vec<_> = ops + .iter() + .filter_map(|op| match op { + SpanOp::Insert(row) if row.span_type == SpanType::Llm => Some(row), + _ => None, + }) + .collect(); + assert_eq!(llms.len(), 3); + assert!(llms[0] + .metadata + .as_ref() + .is_some_and(|metadata| metadata.get("boundary_source").is_none())); + + let first_turn_merge = ops + .iter() + .find_map(|op| match op { + SpanOp::Merge(row) if row.span_id == turns[0].span_id => Some(row), + _ => None, + }) + .unwrap(); + assert_eq!( + first_turn_merge.metadata.as_ref().unwrap()["model_call_count_mismatch"], + json!({"native": 3, "reconstructed": 2}) + ); + let first_turn_llms: Vec<_> = llms + .iter() + .filter(|llm| llm.parent_span_ids == vec![turns[0].span_id.clone()]) + .collect(); + assert_eq!(first_turn_llms.len(), 2); + assert!(!ops.iter().any( + |op| matches!(op, SpanOp::Merge(row) if row.span_id == first_turn_llms[0].span_id && row.metrics.is_some()), + )); + assert!(ops.iter().any(|op| { + matches!(op, SpanOp::Merge(row) if row.span_id == first_turn_llms[1].span_id + && row.metrics.as_ref().is_some_and(|metrics| metrics["model_calls"] == 3) + && row.metadata.as_ref().is_some_and(|metadata| { + metadata["usage_scope"] == "turn" + && metadata["usage_attribution"] == "last_llm" + })) + })); + let second_llm = llms + .iter() + .find(|llm| llm.parent_span_ids == vec![turns[1].span_id.clone()]) + .unwrap(); + assert!(ops.iter().any(|op| { + matches!(op, SpanOp::Merge(row) if row.span_id == second_llm.span_id + && row.metrics.as_ref().is_some_and(|metrics| metrics["model_calls"] == 1)) + })); + let cancelled_merges: Vec<_> = ops + .iter() + .filter_map(|op| match op { + SpanOp::Merge(row) + if row + .metadata + .as_ref() + .is_some_and(|metadata| metadata["cancelled"] == true) => + { + Some(row) + } + _ => None, + }) + .collect(); + assert!(!cancelled_merges.is_empty()); + assert!(cancelled_merges.iter().all(|row| row.error.is_none())); +} + +#[test] +fn grok_partial_records_are_retried_and_complete_malformed_lines_do_not_stall() { + let temp = tempfile::tempdir().unwrap(); + let updates_path = temp.path().join("updates.jsonl"); + let events_path = temp.path().join("events.jsonl"); + let initial = std::fs::read(fixture("partial-updates.jsonl")).unwrap(); + std::fs::write(&updates_path, &initial).unwrap(); + std::fs::write(&events_path, "").unwrap(); + let mut first_event = envelope(0, 0); + point_at(&mut first_event, &updates_path, &events_path); + + let registry = Registry::default_agents(); + let mut translator = registry.create("grok", "grok-session"); + let first = translator.handle(&first_event, &ctx()).unwrap(); + assert!(!first + .iter() + .any(|op| matches!(op, SpanOp::Insert(row) if row.span_type == SpanType::Llm))); + + let mut completed = initial; + completed.extend_from_slice( + b",\"streamStartMs\":1100,\"agentTimestampMs\":1110,\"chunkId\":1}}}\nnot-json\n{\"params\":{\"update\":{\"sessionUpdate\":\"turn_completed\",\"prompt_id\":\"partial-prompt\",\"usage\":{\"modelCalls\":1}},\"_meta\":{\"agentTimestampMs\":1200}}}\n", + ); + std::fs::write(&updates_path, completed).unwrap(); + let mut second_event = envelope(0, 0); + point_at(&mut second_event, &updates_path, &events_path); + let second = translator.handle(&second_event, &ctx()).unwrap(); + assert!(second + .iter() + .any(|op| matches!(op, SpanOp::Insert(row) if row.span_type == SpanType::Llm))); + assert!(second.iter().any(|op| matches!(op, SpanOp::Merge(row) if row.metrics.as_ref().is_some_and(|metrics| metrics["model_calls"] == 1)))); +} + +#[test] +fn grok_events_failure_does_not_block_updates_and_recovers_enrichment() { + let temp = tempfile::tempdir().unwrap(); + let updates_path = temp.path().join("updates.jsonl"); + let events_path = temp.path().join("events.jsonl"); + let update_body = concat!( + "{\"params\":{\"update\":{\"sessionUpdate\":\"user_message_chunk\",\"content\":{\"text\":\"retry\"}},\"_meta\":{\"promptIndex\":0,\"agentTimestampMs\":1000}}}\n", + "{\"params\":{\"update\":{\"sessionUpdate\":\"tool_call\",\"toolCallId\":\"retry-tool\",\"title\":\"read\"},\"_meta\":{\"agentTimestampMs\":1100}}}\n", + "{\"params\":{\"update\":{\"sessionUpdate\":\"tool_call_update\",\"toolCallId\":\"retry-tool\",\"status\":\"completed\"},\"_meta\":{\"agentTimestampMs\":1200}}}\n" + ); + let event_body = "{\"ts\":\"1970-01-01T00:00:01.200Z\",\"type\":\"tool_completed\",\"tool_call_id\":\"retry-tool\",\"duration_ms\":100,\"outcome\":\"success\"}\n"; + std::fs::write(&updates_path, update_body).unwrap(); + + let mut event = envelope(update_body.len() as u64, event_body.len() as u64); + event.payload["_bt_grok_transcript_mirrors"]["updates"]["mirror"] = json!(updates_path); + event.payload["_bt_grok_transcript_mirrors"]["events"]["mirror"] = json!(events_path); + let registry = Registry::default_agents(); + let mut translator = registry.create("grok", "grok-session"); + + let primary = translator.handle(&event, &ctx()).unwrap(); + assert_eq!( + primary + .iter() + .filter(|op| matches!(op, SpanOp::Insert(row) if row.name == "Turn 1")) + .count(), + 1 + ); + assert!(primary + .iter() + .any(|op| matches!(op, SpanOp::Insert(row) if row.span_type == SpanType::Tool))); + assert!(!primary.iter().any(|op| { + matches!(op, SpanOp::Merge(row) if row.metadata.as_ref().is_some_and(|metadata| metadata.get("outcome").is_some())) + })); + + std::fs::write(&events_path, event_body).unwrap(); + let enrichment = translator.handle(&event, &ctx()).unwrap(); + assert!(!enrichment.iter().any(|op| matches!(op, SpanOp::Insert(_)))); + let enriched_tools: Vec<_> = enrichment + .iter() + .filter_map(|op| match op { + SpanOp::Merge(row) + if row.metadata.as_ref().is_some_and(|metadata| { + metadata["duration_ms"] == 100 && metadata["outcome"] == "success" + }) => + { + Some(row) + } + _ => None, + }) + .collect(); + assert_eq!(enriched_tools.len(), 1); + assert!(translator.handle(&event, &ctx()).unwrap().is_empty()); +} + +#[test] +fn grok_updates_failure_keeps_primary_state_retryable() { + let temp = tempfile::tempdir().unwrap(); + let updates_path = temp.path().join("updates.jsonl"); + let events_path = temp.path().join("events.jsonl"); + let update_body = concat!( + "{\"params\":{\"update\":{\"sessionUpdate\":\"user_message_chunk\",\"content\":{\"text\":\"retry\"}},\"_meta\":{\"promptIndex\":0,\"agentTimestampMs\":1000}}}\n", + "{\"params\":{\"update\":{\"sessionUpdate\":\"tool_call\",\"toolCallId\":\"retry-tool\",\"title\":\"read\"},\"_meta\":{\"agentTimestampMs\":1100}}}\n", + "{\"params\":{\"update\":{\"sessionUpdate\":\"tool_call_update\",\"toolCallId\":\"retry-tool\",\"status\":\"completed\"},\"_meta\":{\"agentTimestampMs\":1200}}}\n" + ); + let event_body = "{\"ts\":\"1970-01-01T00:00:01.200Z\",\"type\":\"tool_completed\",\"tool_call_id\":\"retry-tool\",\"duration_ms\":100,\"outcome\":\"success\"}\n"; + std::fs::write(&events_path, event_body).unwrap(); + + let mut event = envelope(update_body.len() as u64, event_body.len() as u64); + event.payload["_bt_grok_transcript_mirrors"]["updates"]["mirror"] = json!(updates_path); + event.payload["_bt_grok_transcript_mirrors"]["events"]["mirror"] = json!(events_path); + let registry = Registry::default_agents(); + let mut translator = registry.create("grok", "grok-session"); + assert!(translator.handle(&event, &ctx()).is_err()); + + std::fs::write(&updates_path, update_body).unwrap(); + let ops = translator.handle(&event, &ctx()).unwrap(); + assert_eq!( + ops.iter() + .filter(|op| matches!(op, SpanOp::Insert(row) if row.name == "Turn 1")) + .count(), + 1 + ); + assert!(ops.iter().any(|op| { + matches!(op, SpanOp::Merge(row) if row.metadata.as_ref().is_some_and(|metadata| { + metadata["duration_ms"] == 100 && metadata["outcome"] == "success" + })) + })); +} + +#[test] +fn grok_detects_transcript_replacement_and_replays_new_turns() { + let temp = tempfile::tempdir().unwrap(); + let updates_path = temp.path().join("updates.jsonl"); + let events_path = temp.path().join("events.jsonl"); + std::fs::write(&events_path, "").unwrap(); + let first = "{\"params\":{\"update\":{\"sessionUpdate\":\"user_message_chunk\",\"content\":{\"text\":\"old\"}},\"_meta\":{\"promptIndex\":0,\"agentTimestampMs\":1000}}}\n"; + std::fs::write(&updates_path, first).unwrap(); + let mut first_event = envelope(0, 0); + point_at(&mut first_event, &updates_path, &events_path); + let registry = Registry::default_agents(); + let mut translator = registry.create("grok", "grok-session"); + translator.handle(&first_event, &ctx()).unwrap(); + + let replacement = "{\"params\":{\"update\":{\"sessionUpdate\":\"user_message_chunk\",\"content\":{\"text\":\"new replacement transcript\"}},\"_meta\":{\"promptIndex\":1,\"agentTimestampMs\":2000}}}\n"; + std::fs::write(&updates_path, replacement).unwrap(); + let mut replacement_event = envelope(0, 0); + point_at(&mut replacement_event, &updates_path, &events_path); + let ops = translator.handle(&replacement_event, &ctx()).unwrap(); + assert!(ops.iter().any(|op| matches!(op, SpanOp::Insert(row) if row.name == "Turn 2" && row.input == Some(json!("new replacement transcript"))))); +} + +#[test] +fn grok_supports_native_and_documented_terminal_events() { + let temp = tempfile::tempdir().unwrap(); + let updates_path = temp.path().join("updates.jsonl"); + let events_path = temp.path().join("events.jsonl"); + std::fs::write(&events_path, "").unwrap(); + std::fs::write( + &updates_path, + "{\"params\":{\"update\":{\"sessionUpdate\":\"user_message_chunk\",\"content\":{\"text\":\"terminal\"}},\"_meta\":{\"promptIndex\":0,\"agentTimestampMs\":2000}}}\n", + ) + .unwrap(); + let mut base = envelope(0, 0); + point_at(&mut base, &updates_path, &events_path); + base.ts_ms = 1_500; + let registry = Registry::default_agents(); + + for event_name in ["session_end", "SessionEnd"] { + let mut translator = registry.create("grok", "grok-session"); + let mut event = base.clone(); + event.event = event_name.into(); + let ops = translator.handle(&event, &ctx()).unwrap(); + let root_id = ops + .iter() + .find_map(|op| match op { + SpanOp::Insert(row) if row.name == "Grok" => Some(row.span_id.clone()), + _ => None, + }) + .unwrap(); + assert!(ops.iter().any( + |op| matches!(op, SpanOp::Merge(row) if row.span_id == root_id && row.end_ms == Some(2000)) + )); + } + + for (event_name, should_error, cancelled) in [ + ("stop_failure", true, false), + ("StopFailure", true, false), + ("stop_cancelled", false, true), + ("StopCancelled", false, true), + ] { + let mut translator = registry.create("grok", "grok-session"); + let mut event = base.clone(); + event.event = event_name.into(); + let ops = translator.handle(&event, &ctx()).unwrap(); + let turn_id = ops + .iter() + .find_map(|op| match op { + SpanOp::Insert(row) if row.name == "Turn 1" => Some(row.span_id.clone()), + _ => None, + }) + .unwrap(); + let close = ops + .iter() + .find_map(|op| match op { + SpanOp::Merge(row) if row.span_id == turn_id => Some(row), + _ => None, + }) + .unwrap(); + assert_eq!(close.error.is_some(), should_error); + assert_eq!( + close + .metadata + .as_ref() + .is_some_and(|metadata| metadata["cancelled"] == true), + cancelled + ); + } +} + +#[test] +fn grok_catch_up_and_open_state_bounds_drain_without_losing_later_records() { + let temp = tempfile::tempdir().unwrap(); + let updates_path = temp.path().join("updates.jsonl"); + let events_path = temp.path().join("events.jsonl"); + let mut body = String::from("{\"params\":{\"update\":{\"sessionUpdate\":\"user_message_chunk\",\"content\":{\"text\":\"bounded\"}},\"_meta\":{\"promptIndex\":0,\"agentTimestampMs\":1000}}}\n"); + for index in 0..270 { + body.push_str(&format!("{{\"params\":{{\"update\":{{\"sessionUpdate\":\"tool_call\",\"toolCallId\":\"tool-{index}\",\"title\":\"tool\"}},\"_meta\":{{\"agentTimestampMs\":{}}}}}}}\n", 1100 + index)); + } + body.push_str("{\"params\":{\"update\":{\"sessionUpdate\":\"turn_completed\",\"usage\":{\"modelCalls\":0}},\"_meta\":{\"agentTimestampMs\":2000}}}\n"); + std::fs::write(&updates_path, body).unwrap(); + std::fs::write(&events_path, "").unwrap(); + let mut event = envelope(0, 0); + point_at(&mut event, &updates_path, &events_path); + let registry = Registry::default_agents(); + let mut translator = registry.create("grok", "grok-session"); + let mut ops = translator.handle(&event, &ctx()).unwrap(); + ops.extend(drain(translator.as_mut())); + assert_eq!( + ops.iter() + .filter(|op| matches!(op, SpanOp::Insert(row) if row.span_type == SpanType::Tool)) + .count(), + 270 + ); + assert!(ops.iter().any(|op| matches!(op, SpanOp::Merge(row) if row.metadata.as_ref().is_some_and(|metadata| metadata["close_reason"] == "open_tool_limit")))); +} diff --git a/bt-daemon/tests/pipeline.rs b/bt-daemon/tests/pipeline.rs index 3b03eeb..42d2468 100644 --- a/bt-daemon/tests/pipeline.rs +++ b/bt-daemon/tests/pipeline.rs @@ -6,10 +6,12 @@ use async_trait::async_trait; use bt_daemon::wire::{ AuthSelection, AuthSource, BackendAuth, Envelope, SessionConfig, SessionRoute, }; +#[cfg(all(feature = "cli", unix))] +use bt_daemon::DebugSinkFactory; use bt_daemon::{ debug_serve_options, flush_managed_run, flush_session, forward_envelope, run_serve, run_status, shutdown_daemon, source_journal_path, AuthLease, AuthProvider, AuthResolveReason, HostInfo, - Registry, ServeArgs, ServeOptions, Sink, SinkFactory, SpanOp, StatusArgs, + Registry, ServeArgs, ServeOptions, Sink, SinkFactory, SpanOp, SpanType, StatusArgs, }; #[cfg(all(feature = "cli", unix))] use bt_daemon::{run_traced, RunArgs, RunHookCommand, RunSource}; @@ -1991,3 +1993,758 @@ esac shutdown(&socket).await; handle.await.unwrap(); } + +#[cfg(all(feature = "cli", unix))] +type CreatedDebugSinks = Arc)>>>; + +#[cfg(all(feature = "cli", unix))] +struct FlushTrackingDebugSinkFactory { + inner: DebugSinkFactory, + flushes: Arc>>, + created: CreatedDebugSinks, +} + +#[cfg(all(feature = "cli", unix))] +impl SinkFactory for FlushTrackingDebugSinkFactory { + fn create( + &self, + session_id: &str, + source: &str, + plugin_version: Option<&str>, + ) -> anyhow::Result> { + self.created.lock().unwrap().push(( + session_id.to_string(), + source.to_string(), + plugin_version.map(str::to_string), + )); + Ok(Box::new(FlushTrackingDebugSink { + inner: self.inner.create(session_id, source, plugin_version)?, + session_id: session_id.to_string(), + flushes: self.flushes.clone(), + })) + } +} + +#[cfg(all(feature = "cli", unix))] +struct FlushTrackingDebugSink { + inner: Box, + session_id: String, + flushes: Arc>>, +} + +#[cfg(all(feature = "cli", unix))] +#[async_trait] +impl Sink for FlushTrackingDebugSink { + fn configure(&mut self, config: &SessionConfig) { + self.inner.configure(config); + } + + async fn emit(&mut self, ops: &[SpanOp]) -> anyhow::Result { + self.inner.emit(ops).await + } + + async fn flush(&mut self) -> anyhow::Result<()> { + self.inner.flush().await?; + *self + .flushes + .lock() + .unwrap() + .entry(self.session_id.clone()) + .or_default() += 1; + Ok(()) + } +} + +#[cfg(all(feature = "cli", unix))] +async fn start_grok_debug_daemon_at( + data_dir: PathBuf, + socket: PathBuf, + flushes: Arc>>, + created: CreatedDebugSinks, +) -> tokio::task::JoinHandle<()> { + let args = ServeArgs { + socket: Some(socket.clone()), + data_dir: Some(data_dir.clone()), + idle_timeout_secs: 0, + session_idle_timeout_secs: 0, + }; + let opts = ServeOptions { + version: env!("CARGO_PKG_VERSION").into(), + translators: Arc::new(Registry::default_agents()), + sink_factory: Arc::new(FlushTrackingDebugSinkFactory { + inner: DebugSinkFactory { + dir: data_dir.join("spans"), + }, + flushes, + created, + }), + auth_provider: Some(Arc::new(TestAuthProvider { + calls: Mutex::new(Vec::new()), + fail: false, + first_lease_expired: false, + })), + }; + let handle = tokio::spawn(async move { + let _ = run_serve(args, opts).await; + }); + wait_for(&socket).await; + handle +} + +#[cfg(all(feature = "cli", unix))] +fn grok_package_path(relative: &str) -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../src/plugins/grok/content") + .join(relative) +} + +#[cfg(all(feature = "cli", unix))] +fn grok_fixture(name: &str) -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/grok/transcript") + .join(name) +} + +#[cfg(all(feature = "cli", unix))] +fn prefix_through_lines(bytes: &[u8], line_count: usize) -> usize { + bytes + .iter() + .enumerate() + .filter(|(_, byte)| **byte == b'\n') + .nth(line_count - 1) + .map(|(index, _)| index + 1) + .expect("fixture must contain the requested line boundary") +} + +#[cfg(all(feature = "cli", unix))] +async fn assert_packaged_grok_hook_mapping( + temp: &Path, + plugin_version: &str, + payload: &serde_json::Value, +) { + use std::os::unix::fs::PermissionsExt; + use tokio::io::AsyncWriteExt; + + let fake_bt = temp.join("record-bt.sh"); + let args_file = temp.join("packaged-args.txt"); + let stdin_file = temp.join("packaged-stdin.json"); + std::fs::write( + &fake_bt, + "#!/bin/sh\nprintf '%s\\n' \"$@\" > \"$BT_ARGS_FILE\"\ncat > \"$BT_STDIN_FILE\"\n", + ) + .unwrap(); + std::fs::set_permissions(&fake_bt, std::fs::Permissions::from_mode(0o755)).unwrap(); + + let mut child = tokio::process::Command::new("bash") + .arg(grok_package_path("hooks/forward.sh")) + .env("BT_BIN", &fake_bt) + .env("BT_ARGS_FILE", &args_file) + .env("BT_STDIN_FILE", &stdin_file) + .env("GROK_VERSION", "1.0.13") + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .kill_on_drop(true) + .spawn() + .unwrap(); + let encoded = serde_json::to_vec(payload).unwrap(); + child + .stdin + .take() + .unwrap() + .write_all(&encoded) + .await + .unwrap(); + let output = child.wait_with_output().await.unwrap(); + assert!( + output.status.success(), + "packaged Grok hook failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + + let args: Vec<_> = std::fs::read_to_string(args_file) + .unwrap() + .lines() + .map(str::to_string) + .collect(); + assert_eq!( + args, + [ + "trace", + "hook", + "--source", + "grok", + "--plugin-version", + plugin_version, + "--session-id-field", + "sessionId", + "--event-field", + "hookEventName", + "--transcript-path-field", + "transcriptPath", + "--source-version", + "1.0.13", + ] + ); + let forwarded: serde_json::Value = + serde_json::from_slice(&std::fs::read(stdin_file).unwrap()).unwrap(); + assert_eq!( + &forwarded, payload, + "the adapter must forward stdin unchanged" + ); +} + +#[cfg(all(feature = "cli", unix))] +async fn invoke_grok_hook( + socket: &Path, + plugin_version: &str, + route_marker: &str, + project: &str, + payload: &serde_json::Value, +) { + use tokio::io::AsyncWriteExt; + + let invocation_settings = serde_json::json!({ + "trace_to_braintrust": true, + "route": { + "auth": {"source": "environment"}, + "destination": {"type": "project_logs", "project_name": project}, + "additional_metadata": {"route_marker": route_marker} + } + }); + let mut child = tokio::process::Command::new(env!("CARGO_BIN_EXE_bt-daemon")) + .args([ + "hook", + "--source", + "grok", + "--source-version", + "1.0.13", + "--plugin-version", + plugin_version, + "--session-id-field", + "sessionId", + "--event-field", + "hookEventName", + "--transcript-path-field", + "transcriptPath", + "--no-spawn", + ]) + .env("BT_DAEMON_SOCKET", socket) + .env( + "BT_TRACE_INVOCATION_SETTINGS", + invocation_settings.to_string(), + ) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .kill_on_drop(true) + .spawn() + .unwrap(); + child + .stdin + .take() + .unwrap() + .write_all(&serde_json::to_vec(payload).unwrap()) + .await + .unwrap(); + let output = child.wait_with_output().await.unwrap(); + assert!( + output.status.success(), + "Grok hook failed: {}", + String::from_utf8_lossy(&output.stderr) + ); +} + +#[cfg(all(feature = "cli", unix))] +async fn wait_for_grok_translation(socket: &Path, session_id: &str) { + let mut last_sessions = Vec::new(); + for _ in 0..200 { + let status = run_status(StatusArgs { + socket: Some(socket.to_path_buf()), + session_id: Some(session_id.to_string()), + }) + .await + .unwrap() + .unwrap(); + if status + .sessions + .iter() + .any(|session| session.queued == 0 && session.spans_emitted > 0) + { + return; + } + last_sessions = status.sessions; + tokio::time::sleep(Duration::from_millis(10)).await; + } + panic!("Grok session {session_id} was not translated: {last_sessions:?}"); +} + +#[cfg(all(feature = "cli", unix))] +async fn wait_for_grok_flushes( + flushes: &Arc>>, + session_id: &str, + expected: usize, +) -> usize { + for _ in 0..200 { + let observed = match flushes.lock() { + Ok(flushes) => flushes.get(session_id).copied().unwrap_or_default(), + Err(error) => panic!("Grok flush counter lock poisoned: {error}"), + }; + if observed >= expected { + return observed; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + panic!("Grok session {session_id} did not reach {expected} flushes"); +} + +#[cfg(all(feature = "cli", unix))] +async fn wait_for_grok_enrichment(path: &Path) { + for _ in 0..200 { + if std::fs::read_to_string(path).is_ok_and(|contents| { + contents.contains("\"duration_ms\":100") && contents.contains("\"outcome\":\"success\"") + }) { + return; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + panic!("Grok tool enrichment was not written to {}", path.display()); +} + +#[cfg(all(feature = "cli", unix))] +fn read_span_ops(path: &Path) -> Vec { + std::fs::read_to_string(path) + .unwrap() + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect() +} + +#[cfg(all(feature = "cli", unix))] +#[tokio::test] +async fn packaged_grok_hook_replays_bounded_transcripts_to_isolated_debug_routes() { + let temp = tempfile::tempdir().unwrap(); + let data_dir = temp.path().join("data"); + let socket = test_endpoint(temp.path()); + let manifest: serde_json::Value = serde_json::from_str( + &std::fs::read_to_string(grok_package_path(".grok-plugin/plugin.json")).unwrap(), + ) + .unwrap(); + let plugin_version = manifest["version"].as_str().unwrap(); + + let updates = std::fs::read(grok_fixture("updates.jsonl")).unwrap(); + let events = std::fs::read(grok_fixture("events.jsonl")).unwrap(); + let first_updates_through = prefix_through_lines(&updates, 5); + let system_prompt = "You are Grok, a test coding assistant.\n"; + + let primary_dir = temp.path().join("native-primary"); + std::fs::create_dir_all(&primary_dir).unwrap(); + let primary_chat = primary_dir.join("chat_history.jsonl"); + let primary_updates = primary_dir.join("updates.jsonl"); + let primary_events = primary_dir.join("events.jsonl"); + let primary_system_prompt = primary_dir.join("system_prompt.txt"); + std::fs::write(&primary_chat, "").unwrap(); + std::fs::write(&primary_updates, &updates[..first_updates_through]).unwrap(); + std::fs::write(&primary_system_prompt, system_prompt).unwrap(); + + let primary_stop = serde_json::json!({ + "sessionId": "grok-primary", + "hookEventName": "stop", + "transcriptPath": primary_chat, + "cwd": "/repo/primary", + "workspaceRoot": "/repo" + }); + assert_packaged_grok_hook_mapping(temp.path(), plugin_version, &primary_stop).await; + + let flushes = Arc::new(Mutex::new(HashMap::new())); + let created = Arc::new(Mutex::new(Vec::new())); + let first = start_grok_debug_daemon_at( + data_dir.clone(), + socket.clone(), + flushes.clone(), + created.clone(), + ) + .await; + invoke_grok_hook( + &socket, + plugin_version, + "primary-route", + "primary-project", + &primary_stop, + ) + .await; + wait_for_grok_translation(&socket, "grok-primary").await; + assert_eq!( + flushes + .lock() + .unwrap() + .get("grok-primary") + .copied() + .unwrap_or_default(), + 0, + "stop alone must not masquerade as the terminal flush" + ); + + let primary_spans_path = data_dir.join("spans/grok-primary.ndjson"); + let first_ops = read_span_ops(&primary_spans_path); + let first_llm = first_ops + .iter() + .find_map(|op| match op { + SpanOp::Insert(row) if row.span_type == SpanType::Llm => Some(row), + _ => None, + }) + .expect("partial boundary must emit the first LLM"); + assert_eq!( + first_llm.input, + Some(serde_json::json!([ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": "inspect the fixture"} + ])) + ); + assert_eq!( + first_llm.metadata.as_ref().unwrap()["input_scope"], + "system_and_user" + ); + assert_eq!( + first_llm.metadata.as_ref().unwrap()["user_message_included"], + true + ); + let first_insert_ids: Vec<_> = first_ops + .iter() + .filter_map(|op| match op { + SpanOp::Insert(row) => Some(row.span_id.clone()), + SpanOp::Merge(_) => None, + }) + .collect(); + assert_eq!( + first_insert_ids.len(), + 4, + "the partial boundary should contain root, turn, first LLM, and tool" + ); + + let decoy_dir = temp.path().join("native-decoy"); + std::fs::create_dir_all(&decoy_dir).unwrap(); + let decoy_chat = decoy_dir.join("chat_history.jsonl"); + std::fs::write(&decoy_chat, "").unwrap(); + std::fs::write( + decoy_dir.join("updates.jsonl"), + &updates[..prefix_through_lines(&updates, 1)], + ) + .unwrap(); + std::fs::write(decoy_dir.join("events.jsonl"), "").unwrap(); + let decoy_end = serde_json::json!({ + "sessionId": "grok-decoy", + "hookEventName": "session_end", + "transcriptPath": decoy_chat, + "cwd": "/repo/decoy" + }); + invoke_grok_hook( + &socket, + plugin_version, + "decoy-route", + "decoy-project", + &decoy_end, + ) + .await; + wait_for_grok_translation(&socket, "grok-decoy").await; + assert_eq!( + wait_for_grok_flushes(&flushes, "grok-decoy", 1).await, + 1, + "the native session_end event must trigger one daemon-owned flush" + ); + let decoy_journal_text = + std::fs::read_to_string(source_journal_path(&data_dir, "grok", "grok-decoy")).unwrap(); + let decoy_spans_path = data_dir.join("spans/grok-decoy.ndjson"); + let decoy_before_restart = std::fs::read(&decoy_spans_path).unwrap(); + let decoy_text = String::from_utf8(decoy_before_restart.clone()).unwrap(); + assert!( + decoy_text.contains("decoy-route"), + "decoy span output did not contain route metadata; journal={decoy_journal_text}; spans={decoy_text}" + ); + assert!(!decoy_text.contains("primary-route")); + + shutdown(&socket).await; + first.await.unwrap(); + + std::fs::write(&primary_updates, &updates).unwrap(); + std::fs::write(&primary_events, &events).unwrap(); + let second = start_grok_debug_daemon_at( + data_dir.clone(), + socket.clone(), + flushes.clone(), + created.clone(), + ) + .await; + let primary_end = serde_json::json!({ + "sessionId": "grok-primary", + "hookEventName": "session_end", + "transcriptPath": primary_chat, + "cwd": "/repo/primary", + "workspaceRoot": "/repo" + }); + invoke_grok_hook( + &socket, + plugin_version, + "primary-route", + "primary-project", + &primary_end, + ) + .await; + let flushed = flush_session("grok-primary", &socket, 5_000).await.unwrap(); + assert!( + flushed.flushed, + "explicit delivery barrier did not flush the terminal Grok event: {flushed:?}" + ); + assert_eq!( + std::fs::read(&decoy_spans_path).unwrap(), + decoy_before_restart, + "replaying the primary route must not emit into the decoy debug sink" + ); + + let journal_text = + std::fs::read_to_string(source_journal_path(&data_dir, "grok", "grok-primary")).unwrap(); + assert!(!journal_text.contains("secret-environment")); + assert!(!journal_text.contains("token_sha256_prefix")); + assert!(!journal_text.contains("decoy-route")); + let journal: Vec = journal_text + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .filter(|record: &serde_json::Value| record["_bt_record_type"] != "delivery_checkpoint") + .collect(); + assert_eq!(journal.len(), 2); + assert_eq!(journal[0]["source"], "grok"); + assert_eq!(journal[1]["source"], "grok"); + assert_eq!(journal[0]["source_version"], "1.0.13"); + assert_eq!(journal[1]["source_version"], "1.0.13"); + assert_eq!(journal[0]["plugin_version"], plugin_version); + assert_eq!(journal[1]["plugin_version"], plugin_version); + assert_eq!(journal[0]["event"], "stop"); + assert_eq!(journal[1]["event"], "session_end"); + assert_eq!( + journal[0]["route"]["additional_metadata"]["route_marker"], + "primary-route" + ); + + let first_mirrors = &journal[0]["payload"]["_bt_grok_transcript_mirrors"]; + let second_mirrors = &journal[1]["payload"]["_bt_grok_transcript_mirrors"]; + assert_eq!( + first_mirrors.as_object().unwrap().len(), + 2, + "the first hook must journal the available updates and system prompt mirrors" + ); + assert!(first_mirrors.get("events").is_none()); + assert_eq!( + first_mirrors["updates"]["through"], + first_updates_through as u64 + ); + assert_eq!(second_mirrors["updates"]["through"], updates.len() as u64); + assert!( + first_updates_through < updates.len(), + "updates must advance between lifecycle boundaries" + ); + assert_eq!( + first_mirrors["updates"]["path"], + primary_updates.to_str().unwrap() + ); + assert_eq!( + first_mirrors["updates"]["mirror"], second_mirrors["updates"]["mirror"], + "updates must append to one stable daemon-owned mirror" + ); + let updates_mirror = PathBuf::from(second_mirrors["updates"]["mirror"].as_str().unwrap()); + assert!(updates_mirror.starts_with(data_dir.join("transcripts"))); + assert_eq!( + std::fs::metadata(updates_mirror).unwrap().len(), + updates.len() as u64 + ); + assert_eq!( + first_mirrors["system_prompt"]["through"], + system_prompt.len() as u64 + ); + assert_eq!( + second_mirrors["system_prompt"]["through"], + system_prompt.len() as u64 + ); + assert_eq!( + first_mirrors["system_prompt"]["path"], + primary_system_prompt.to_str().unwrap() + ); + assert_eq!( + first_mirrors["system_prompt"]["mirror"], second_mirrors["system_prompt"]["mirror"], + "the system prompt must retain one stable daemon-owned snapshot" + ); + let system_prompt_mirror = + PathBuf::from(second_mirrors["system_prompt"]["mirror"].as_str().unwrap()); + assert!(system_prompt_mirror.starts_with(data_dir.join("transcripts"))); + assert_eq!( + std::fs::read_to_string(system_prompt_mirror).unwrap(), + system_prompt + ); + + assert_eq!(second_mirrors["events"]["through"], events.len() as u64); + assert_eq!( + second_mirrors["events"]["path"], + primary_events.to_str().unwrap() + ); + let events_mirror = PathBuf::from(second_mirrors["events"]["mirror"].as_str().unwrap()); + assert!(events_mirror.starts_with(data_dir.join("transcripts"))); + assert_eq!( + std::fs::metadata(events_mirror).unwrap().len(), + events.len() as u64 + ); + + wait_for_grok_enrichment(&primary_spans_path).await; + let all_ops = read_span_ops(&primary_spans_path); + let mut inserts_by_id: HashMap> = HashMap::new(); + for op in &all_ops { + if let SpanOp::Insert(row) = op { + inserts_by_id + .entry(row.span_id.clone()) + .or_default() + .push(serde_json::to_value(row).unwrap()); + } + } + assert_eq!( + inserts_by_id.len(), + 5, + "restart recovery must retain four prior ids and add only the second LLM" + ); + for id in &first_insert_ids { + let copies = inserts_by_id.get(id).unwrap(); + assert_eq!( + copies.len(), + 1, + "checkpointed historical insert {id} must not be replayed" + ); + } + let new_insert_ids: Vec<_> = inserts_by_id + .iter() + .filter(|(id, _)| !first_insert_ids.contains(id)) + .collect(); + assert_eq!(new_insert_ids.len(), 1); + assert_eq!( + new_insert_ids[0].1.len(), + 1, + "the newly observed second LLM must not be duplicated after recovery" + ); + + let unique_rows: Vec<_> = inserts_by_id.values().map(|copies| &copies[0]).collect(); + let root = unique_rows + .iter() + .find(|row| row["name"] == "Grok") + .unwrap(); + assert_eq!(root["span_type"], "task"); + assert_eq!(root["metadata"]["source"], "grok"); + assert_eq!(root["metadata"]["grok_version"], "1.0.13"); + assert_eq!(root["metadata"]["plugin_version"], plugin_version); + assert_eq!(root["metadata"]["route_marker"], "primary-route"); + assert_eq!(root["metadata"]["trace_source"], "session_transcript"); + assert_eq!(root["root_span_id"], root["span_id"]); + assert!( + root.get("parent_span_ids").is_none(), + "an unattached Grok root must not acquire a parent" + ); + let root_id = root["span_id"].as_str().unwrap(); + + let turn = unique_rows + .iter() + .find(|row| row["name"] == "Turn 1") + .unwrap(); + assert_eq!(turn["parent_span_ids"], serde_json::json!([root_id])); + let turn_id = turn["span_id"].as_str().unwrap(); + let turn_close = all_ops + .iter() + .find_map(|op| match op { + SpanOp::Merge(row) if row.span_id == turn_id && row.metrics.is_some() => Some(row), + _ => None, + }) + .expect("native session_end must deliver the final turn metrics"); + let metrics = turn_close.metrics.as_ref().unwrap(); + assert_eq!(metrics["prompt_tokens"], 100); + assert_eq!(metrics["completion_tokens"], 20); + assert_eq!(metrics["tokens"], 120); + assert_eq!(metrics["prompt_cached_tokens"], 40); + assert_eq!(metrics["model_calls"], 2); + assert_eq!(metrics["cost_usd_ticks"], 12); + assert!(metrics.get("estimated_cost").is_none()); + let children: Vec<_> = unique_rows + .iter() + .filter(|row| row["span_type"] == "llm" || row["span_type"] == "tool") + .collect(); + assert_eq!(children.len(), 3); + assert!(children + .iter() + .all(|row| row["parent_span_ids"] == serde_json::json!([turn_id]))); + let llm_names: HashSet<_> = children + .iter() + .filter(|row| row["span_type"] == "llm") + .map(|row| row["name"].as_str().unwrap()) + .collect(); + assert_eq!( + llm_names, + HashSet::from(["grok-4.6-build call 1", "grok-4.6-build call 2"]) + ); + let first_llm_id = children + .iter() + .find(|row| row["name"] == "grok-4.6-build call 1") + .unwrap()["span_id"] + .as_str() + .unwrap(); + let first_llm_close = all_ops + .iter() + .find_map(|op| match op { + SpanOp::Merge(row) if row.span_id == first_llm_id && row.output.is_some() => Some(row), + _ => None, + }) + .expect("the first LLM must close with a renderable assistant response"); + assert_eq!( + first_llm_close.output, + Some(serde_json::json!([{ + "role": "assistant", + "content": "Reading the fixture.", + "reasoning": [{ + "id": "reasoning", + "content": "I will read it." + }] + }])) + ); + let tool_id = children + .iter() + .find(|row| row["span_type"] == "tool") + .unwrap()["span_id"] + .as_str() + .unwrap(); + let tool_enrichment = all_ops + .iter() + .find_map(|op| match op { + SpanOp::Merge(row) + if row.span_id == tool_id + && row.metadata.as_ref().is_some_and(|metadata| { + metadata["duration_ms"] == 100 && metadata["outcome"] == "success" + }) => + { + Some(row) + } + _ => None, + }) + .expect("late events mirror must enrich the existing deterministic tool span"); + assert_eq!(tool_enrichment.span_id, tool_id); + assert!(all_ops.iter().any(|op| { + matches!( + op, + SpanOp::Merge(row) + if row.span_id == root_id && row.end_ms.is_some() + ) + })); + let primary_text = std::fs::read_to_string(&primary_spans_path).unwrap(); + assert!(primary_text.contains("primary-route")); + assert!(!primary_text.contains("decoy-route")); + + let created = created.lock().unwrap().clone(); + assert_eq!(created.len(), 3); + assert!(created.iter().all(|(_, source, version)| { + source == "grok" && version.as_deref() == Some(plugin_version) + })); + + shutdown(&socket).await; + second.await.unwrap(); +} diff --git a/docs/coding-agent-tracing-plugin-spec.md b/docs/coding-agent-tracing-plugin-spec.md new file mode 100644 index 0000000..dec2716 --- /dev/null +++ b/docs/coding-agent-tracing-plugin-spec.md @@ -0,0 +1,549 @@ +# Coding-agent tracing plugin specification + +Status: **Draft** + +This specification defines how a coding-agent integration captures native +activity and turns it into Braintrust traces in this repository. It is both the +implementation contract for a new integration and the checklist used to review +one. + +The words **MUST**, **MUST NOT**, **SHOULD**, **SHOULD NOT**, and **MAY** are to +be interpreted as normative requirements. + +## 1. Objective + +A conforming integration produces a faithful, recoverable Braintrust trace of a +coding-agent session without changing the agent's behavior or placing +Braintrust credentials in the plugin. + +The minimum useful trace is: + +```text +Coding agent session (task) +└── Turn (task) +``` + +When native evidence exists, the trace expands to: + +```text +Coding agent session (task) +└── Turn (task) + ├── LLM call (llm) + ├── Tool call (tool) + ├── Permission decision (task) + └── Subagent (task) + └── ...nested activity +``` + +An integration MUST represent only facts present in native events or a vetted +native transcript. It MUST NOT invent model calls, token counts, timing, +permission decisions, or nesting. + +## 2. Architecture and ownership + +There is no declarative plugin schema that automatically creates all tracing +features. Each agent has a native event model, so each integration supplies a +small capture adapter and an agent-specific Rust translator. + +The data path is: + +```text +agent hook/plugin + → raw native payload + → bt trace hook + → daemon Envelope + journal + → agent-specific Translator + → sink-neutral SpanOp rows + → Braintrust sink / Rust SDK +``` + +Responsibilities are intentionally separated: + +| Component | Owns | MUST NOT own | +|-------------------------------------|---------------------------------------------------------------------|-------------------------------------------------------| +| Agent hook/plugin | Native event capture and synchronous forwarding | Credentials, span construction, retries to Braintrust | +| `bt trace hook` / daemon wire layer | Envelope fields, route resolution, journaling, per-session ordering | Agent-specific payload semantics | +| Agent translator | Correlation, deterministic IDs, trace hierarchy, span content | Credentials, HTTP delivery, plugin installation | +| Sink | Braintrust SDK objects, batching, merge delivery, flushing | Agent-native event interpretation | +| Setup/run/import commands | Installation and invocation UX, non-secret routing | A second trace-building implementation | + +A plugin therefore does **not** send arbitrary spans directly to the Rust SDK. +It forwards opaque native events. Its registered translator interprets those +events and emits the common `SpanOp`/`SpanRow` model; the sink is the only layer +that talks to Braintrust. + +Relevant implementation boundaries: + +```text +src/plugins// deployable capture adapter +bt-daemon/src/wire/envelope.rs common event and route contract +bt-daemon/src/translate/.rs agent-specific state machine +bt-daemon/src/translate/mod.rs translator and span interfaces +bt-daemon/src/sink/ delivery through Braintrust +bt-daemon/src/setup.rs persistent setup surfaces +bt-daemon/src/trace_runtime.rs managed-run surfaces +bt-daemon/src/transcript_import/ transcript import/attach surfaces +``` + +## 3. Conformance levels + +A capability marked “when observable” is required only if the agent exposes +reliable native evidence through hooks, an in-process API, or a vetted +transcript. A conformance report MUST mark unavailable capabilities **N/A** and +record the evidence for that conclusion. + +### 3.1 Core tracing + +Required before an integration is described as tracing-capable: + +- Stable source identity and registered translator. +- Thin, fail-open capture adapter. +- Session root span. +- Per-turn task spans with user input and final assistant output when available. +- Deterministic IDs, ordered handling, bounded state, and safe flush behavior. +- Journal-safe non-secret routing. +- Translator fixtures and an envelope-to-sink pipeline test. + +### 3.2 Full-fidelity tracing + +Required when the corresponding native evidence is observable: + +- LLM spans and token/cache metrics. +- Tool, web, and MCP spans. +- Permission requests and decisions. +- Subagent hierarchy. +- Compaction events. +- Failure and cancellation details. +- Session, version, workspace, and Git metadata. + +### 3.3 Product-complete integration + +Required before production release: + +- Persistent enable, disable, and doctor/status support. +- Project, experiment, and parent-span routing supported by the shared command + surfaces where applicable. +- Managed-run support when capture can be injected safely for one process tree. +- Transcript import and live attach when a stable, sufficiently complete + transcript exists. +- Reproducible packaging, validation, publishing, and rollback. +- An installed-artifact smoke test using the real agent. + +## 4. Source identity and registration + +Each integration MUST choose one lowercase canonical source identifier. The same +identifier MUST be used by: + +- capture commands; +- `TranslatorFactory::source()`; +- setup, run, status, and import registries; +- fixtures and pipeline tests; +- settings paths and documentation. + +Aliases MAY exist at user-facing command boundaries, but MUST resolve to the +canonical identifier before an envelope enters the daemon. + +The production translator registry MUST include the source. Unknown sources MAY +use the debug translator for diagnostics, but debug fallback does not constitute +support for an integration. + +## 5. Capture adapter + +### 5.1 Native events + +The adapter MUST register every native event needed by its translator. It SHOULD +register all documented lifecycle events even before all are translated, so +journals can reveal payload evolution and later features do not require a new +capture path. + +The adapter MUST forward the original native payload without renaming or +normalizing agent fields. Normalization belongs in the translator. + +Hook-provider requirements and preferred native fields are described in +[`trace-coding-agents-with-hooks.md`](./trace-coding-agents-with-hooks.md). + +### 5.2 Forwarding behavior + +A command-hook adapter MUST: + +- read one native payload from standard input; +- invoke `bt trace hook` with the canonical source and mappings for session ID, + event name, and transcript path where available; +- block until local daemon acceptance at ordering-sensitive boundaries; +- quote paths and payloads safely; +- never print or persist credentials; +- exit successfully if `bt`, daemon startup, or forwarding fails, unless running + in an explicit diagnostic/test mode. + +An in-process adapter MUST obey the same ownership and failure rules. It MAY +buffer events only if ordering and process-exit flushing are proven. + +### 5.3 Envelope contract + +Every accepted event becomes an `Envelope` containing: + +| Field | Requirement | +|---|---| +| `source` | Canonical source identity | +| `session_id` | Stable native session identity | +| `event` | Unmodified native event name | +| `ts_ms` | Epoch milliseconds stamped at capture time | +| `payload` | Raw native JSON payload | +| `source_version` | Agent version when available | +| `plugin_version` | Capture package version when available | +| `route` | Optional immutable, non-secret routing intent | +| `managed_run_id` | Set only by invocation-local managed runs | +| `capture` | Daemon-safe process ancestry, without command lines or environment | + +Credentials MUST NOT be serializable in an envelope or journal. They are +resolved by the embedding `bt` host at delivery time. + +## 6. Translator contract + +A translator is one stateful `AgentTranslator` instance per daemon session. It +MUST map native events into `SpanOp::Insert` and `SpanOp::Merge` values and MUST +remain independent of Braintrust network delivery. + +### 6.1 General rules + +A translator MUST: + +- tolerate unknown events and additive payload fields; +- tolerate a missing session-start event by opening the root on first useful + activity; +- use capture timestamps rather than translator wall-clock time; +- derive stable IDs from native stable identifiers and the daemon session; +- merge completion data into an existing span rather than creating a second + logical operation; +- preserve externally attached parent/root IDs from `SessionConfig`; +- merge `additional_metadata` without allowing it to override + integration-owned keys; +- close or defensibly finalize open spans during flush; +- bound deduplication, open-operation, and transcript state; +- return an error for malformed events only when continuing would corrupt trace + semantics; schema drift SHOULD otherwise degrade gracefully. + +A translator MUST NOT: + +- perform network I/O to Braintrust; +- resolve credentials; +- parse plugin installation state; +- depend on events being delivered exactly once; +- infer facts that are not supported by native evidence. + +### 6.2 Session root + +A session root MUST be a `task` span. Its ID MUST be deterministic for the +canonical source/session pair. It SHOULD contain: + +- native session ID and source; +- agent and plugin versions; +- cwd and workspace/repository context; +- execution or permission mode; +- transcript location when safe and useful; +- Git revision, branch, and dirty state when available; +- route-provided custom metadata. + +A process shutdown/session-end event closes the current process lifetime. Since +agents can resume sessions, the translator MUST remain replay-safe if later +activity reuses the native session identifier. + +### 6.3 Turns + +Each user request and the agent work attributable to it MUST form one child +`task` span. A turn SHOULD include: + +- the submitted user input; +- the final assistant output; +- native turn/prompt ID; +- start and end capture timestamps; +- completion, cancellation, or failure reason. + +Concurrent or overlapping turns MUST use native identifiers rather than a single +“current turn” slot. If an agent guarantees one active turn, the translator MAY +use a single slot and MUST test that assumption. + +### 6.4 Operations + +For every observable LLM, tool, web, MCP, permission, or subagent operation: + +- start and stop events MUST correlate by a native operation ID; +- the span MUST be parented to the native turn that spawned it; +- nested operations MUST preserve native ancestry; +- input MUST come from the start/request event; +- output, error, metrics, and end time MUST come from completion evidence; +- missing completion MUST be handled during turn/session close or flush without + fabricating successful output. + +Use `llm` for model calls, `tool` for tool/web/MCP execution, and `task` for +turns, permissions, compaction, subagents, and other orchestration. + +### 6.5 LLM metrics + +When observable, an LLM span SHOULD include model/provider identity, full native +request and response bodies subject to data policy, input/output/total tokens, +cache read/write tokens, time-to-first-token, and finish reason. + +Metric names and units MUST match the common sink conventions already used by +existing translators. Estimated token counts MUST be explicitly labeled and +MUST NOT replace native counts silently. + +### 6.6 Failures and cancellation + +A native failed operation MUST set `error` and preserve structured failure +fields in metadata or output as appropriate. Cancellation MUST not be labeled as +success. Translators SHOULD distinguish cancellation from failure with tags or +metadata when the common span model has no dedicated status field. + +## 7. Routing, flushing, and recovery + +Routes are immutable per session and contain only auth selection, destination, +flush mode, and additional metadata. + +Supported destination forms are: + +- project logs by project ID and/or name; +- an experiment by ID; +- an exported parent span. + +`fire_and_forget` is the normal interactive default. `flush_on_turn_end` SHOULD +be available for validation, short-lived processes, and workflows requiring +immediate visibility. + +The daemon journals accepted envelopes before relying on remote delivery. +Replaying a journal MUST converge on the same logical spans. This requires +stable span IDs and idempotent insert/merge behavior. Recovery and explicit +transcript import are separate mechanisms and MUST share the production +translator rather than separate trace builders. + +## 8. Product surfaces + +### 8.1 Persistent setup + +`bt trace enable ` MUST install or update capture idempotently and store +only non-secret route settings. It MUST preserve unrelated user configuration. + +`bt trace disable ` MUST remove only Braintrust-managed state and MUST be +safe when repeated. + +Doctor/status output SHOULD diagnose: + +- whether the agent is installed; +- whether capture is installed and enabled; +- source/daemon/plugin version compatibility; +- route validity and credential availability without exposing credentials; +- daemon reachability; +- agent-specific activation requirements such as restart or plugin reload. + +### 8.2 Managed run + +`bt trace run ` MUST isolate its route to one process tree, inject capture +without rewriting persistent setup, suppress inherited duplicate capture, and +preserve the child process's exit status and signal behavior. + +If safe invocation-local injection is impossible, managed run MUST be marked N/A +with evidence rather than approximated by mutating global configuration. + +### 8.3 Import and attach + +Import is permitted only when a native transcript has enough stable information +to produce the same trace semantics as live capture. Import and live attach +MUST use one parser and the production translator. Attach differs only by +incremental waiting/finalization. + +Session lookup MUST be deterministic and destination/parent overrides MUST not +rewrite persistent settings. Truncated records, resumed sessions, duplicate +records, and partial final writes MUST be tested. + +## 9. Security and data handling + +- Capture adapters MUST be credential-free. +- API keys and OAuth tokens MUST never appear in plugin settings, envelopes, + journals, logs, command lines, or span metadata. +- Journals, debug logs, transcript mirrors, and temporary configs MUST use + user-private permissions. +- Raw prompts, responses, tool payloads, and transcripts MUST be treated as + sensitive. +- Full raw payload logging MUST be opt-in and documented. +- An integration MUST document what content it captures and any redaction + limitations before release. +- Fail-open applies to tracing failures, not to bypassing the coding agent's own + permission or trust model. + +## 10. Verification requirements + +### 10.1 Translator fixtures + +Fixtures MUST cover: + +- normal root/turn lifecycle; +- every translated operation type; +- failures and cancellation; +- duplicate events; +- missing start and missing stop boundaries; +- unknown events and additive schema drift; +- external parent/root attachment; +- custom metadata precedence; +- deterministic replay and flush; +- bounded-state behavior where practical; +- relevant agent-version variants. + +### 10.2 Pipeline tests + +Tests MUST exercise: + +```text +native payload → adapter → Envelope → journal → translator → sink rows +``` + +They MUST verify route isolation, credential redaction, ordering, restart +recovery, and process-exit flush behavior. Debug-sink tests are necessary but do +not replace a Braintrust ingest test. + +### 10.3 Product tests + +A product-complete integration MUST test enable, repeated enable, route changes, +disable, doctor/status, and managed run independently. Import/attach MUST be +tested when supported. + +Before release, CI or documented release verification MUST install the packaged +artifact into an isolated agent home, run the real agent, and confirm the +resulting Braintrust trace hierarchy and content. + +Standard repository checks are: + +```bash +make build +make test +cargo test --manifest-path bt-daemon/Cargo.toml --all-features +cargo fmt --manifest-path bt-daemon/Cargo.toml --all -- --check +cargo clippy --manifest-path bt-daemon/Cargo.toml --all-targets --all-features -- -D warnings +``` + +## 11. Conformance report template + +Every integration SHOULD maintain this table in its plugin documentation or +tracking issue. Do not claim a row based only on code presence; link or name the +verification evidence. + +| Capability | Status | Evidence / blocker | +|---|---|---| +| Source registration | Not started | | +| Native event audit | Not started | | +| Fail-open capture | Not started | | +| Session root | Not started | | +| Turn spans | Not started | | +| LLM spans/metrics | Not started | | +| Tool/web/MCP spans | Not started | | +| Permission events | Not started | | +| Subagents | Not started | | +| Compaction | Not started | | +| Failures/cancellation | Not started | | +| Session/Git/custom metadata | Not started | | +| Deterministic replay and bounded state | Not started | | +| Project/experiment/parent routing | Not started | | +| Enable/disable/doctor | Not started | | +| Managed run | Not started | | +| Import/live attach | Not started | | +| Security and data-handling review | Not started | | +| Build/validation/publishing | Not started | | +| Translator and pipeline tests | Not started | | +| Installed real-agent smoke | Not started | | + +Allowed statuses are **Done**, **Partial**, **Not started**, **Blocked**, and +**N/A**. Partial, Blocked, and N/A entries MUST explain what remains or why the +capability cannot apply. + +## 12. Boundaries for new integrations + +Always: + +- preserve one source identity and one production translator; +- capture raw native evidence before designing mappings; +- add fixtures from real payloads; +- keep adapters thin, synchronous where ordering matters, and fail-open; +- use shared daemon routing, journaling, IDs, metadata, and sink abstractions. + +Ask first: + +- before changing the common `Envelope`, `SpanRow`, or route schema; +- before adding a second capture path or transcript-derived data to live hooks; +- before adding a dependency, credential flow, or cross-agent correlation rule; +- before declaring an observable feature N/A. + +Never: + +- put Braintrust SDK calls or trace construction in an agent plugin/hook; +- implement a second translator for import or managed run; +- persist credentials; +- infer unavailable model/tool/token data; +- let tracing failure block or alter the coding agent. + +## Appendix A: Coding-agent feature index (non-normative) + +This index gives coding-agent integrations a stable vocabulary and a suggested +implementation sequence. It intentionally does not track which agents implement +each feature; integration status and evidence belong in the conformance report +from section 11. The stages are guidance rather than hard dependencies, and +features that require native evidence remain conditional under section 3. + +| Stage | ID | Feature | Description / acceptance notes | +|-----------------------|----------|---------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 1. Feasibility | AUDIT-01 | Native event and transcript audit | Pin the agent version; inventory hooks, plugin APIs, transcripts, IDs, timestamps, ordering, recursion, and schema stability before designing spans. | +| 1. Feasibility | AUDIT-02 | Capability and fidelity report | Mark each observable feature supported, unavailable, or unknown, and record where fidelity is aggregate, reconstructed, delayed, or version-dependent. | +| 1. Foundation | ID-01 | Canonical source and production translator | One lowercase source identity selects one agent-specific production translator at every live, managed-run, and import ingress. | +| 1. Foundation | CAP-01 | Thin fail-open live capture | Native activity reaches the daemon without credentials or trace construction, and tracing failures never interrupt the agent. | +| 1. Foundation | CAP-02 | Complete native event forwarding | Capture registers every useful lifecycle event and forwards original payloads, stable IDs, timestamps, versions, transcript paths, and safe process context. | +| 1. Foundation | CAP-03 | Ordered blocking delivery | Ordering-sensitive boundaries wait for local daemon acceptance; quoting, reconnection, timeout, and process-exit behavior are safe. | +| 1. Foundation | SEC-01 | Credential-free capture and routes | Plugins, envelopes, journals, temporary configuration, command lines, and span metadata contain no API keys or OAuth tokens. | +| 1. Foundation | REL-01 | Durable event journal | Every accepted envelope is journaled before remote delivery using a route-safe representation suitable for deterministic recovery. | +| 1. Foundation | SINK-01 | Idempotent span delivery | Sink-neutral insert and merge operations support batching, late completion updates, retries, and stable remote identity without agent-specific network code. | +| 1. Foundation | ROUTE-01 | Project routing | A session can resolve and pin a Braintrust project destination without placing credentials in agent configuration. | +| 2. Minimum trace | TRACE-01 | Deterministic session root | A replay-safe task span represents the session, preserves external attachment, and can be extended after process shutdown or resume. | +| 2. Minimum trace | TRACE-02 | Per-turn task spans | Each user request has a child task span containing native input, final output, timing, and completion status when observable. | +| 2. Minimum trace | META-01 | Session, version, and workspace metadata | The root includes available native session ID, source and plugin versions, cwd, workspace, execution mode, and transcript provenance. | +| 2. Minimum trace | REL-02 | Deterministic IDs and duplicate handling | Stable native identities and deterministic fallbacks make duplicate delivery and journal replay converge on the same logical spans. | +| 2. Minimum trace | REL-03 | Bounded correlation state | Prompt, operation, transcript, and deduplication state have explicit bounds without dropping currently open work silently. | +| 2. Minimum trace | TRACE-14 | Concurrent activity correlation | Overlapping turns and operations correlate by native identity rather than a single implicit current slot; ambiguous ancestry fails safe. | +| 2. Minimum trace | TRACE-08 | Failure and cancellation closure | Failed, cancelled, interrupted, and incomplete turns close defensibly and are never labeled as successful. | +| 2. Verification | TEST-01 | Translator fixture suite | Fixtures cover normal root/turn flow, missing lifecycle boundaries, duplicates, attachment, metadata precedence, flush, replay, and schema drift. | +| 2. Verification | TEST-02 | Envelope-to-sink pipeline test | Tests exercise native payload capture, envelope creation, journaling, translation, sink rows, ordering, route isolation, and process-exit flush. | +| 3. Operations | TRACE-04 | Tool spans | Native tool IDs correlate start and completion; spans preserve input, output, timing, kind, failure, and missing-boundary behavior. | +| 3. Operations | TRACE-09 | Web and MCP classification | Web search, fetch, and MCP operations remain tool spans but retain their native transport, server, method, and operation classification when observable. | +| 3. Operations | TRACE-05 | Permission requests and decisions | Native permission requests and allow, deny, modification, or cancellation outcomes are represented without inferring unobserved decisions. | +| 3. Operations | TRACE-10 | Skill attribution | Explicit skill loads or invocations are represented as task spans or operation metadata and remain distinguishable from ordinary tools. | +| 3. Model fidelity | TRACE-03 | LLM call spans | Observable model-call boundaries become LLM spans with native model/provider identity and request/response content; unavailable inputs are explicit rather than reconstructed as fact. | +| 3. Model fidelity | TRACE-11 | Token and cache metrics | Native prompt, completion, total, cache-read, cache-write, and reasoning token counts use common metric names; aggregate usage is never guessed across calls. | +| 3. Model fidelity | TRACE-12 | Latency, finish, and cost metrics | Time-to-first-token, API duration, finish reason, estimated/native cost, retries, and model fingerprints are recorded only when native evidence exists. | +| 3. Orchestration | TRACE-06 | Subagent hierarchy | Child and recursively nested agent activity preserves native ancestry and nests beneath the turn or operation that spawned it. | +| 3. Orchestration | TRACE-13 | Background and asynchronous work | Background tools, tasks, workflows, or delayed completions retain identity and remain open until native completion, cancellation, or defensive flush. | +| 3. Orchestration | TRACE-07 | Compaction and session-tree spans | Native compaction, branching, and session-tree activity becomes orchestration spans with before/after context only when observable. | +| 3. Metadata | META-02 | Shared Git metadata | Repository root, revision, branch, remote, and dirty state are enriched through the shared Git metadata cache when a worktree is available. | +| 3. Metadata | META-03 | Route-provided custom metadata | Custom route metadata reaches the root while integration-owned keys retain precedence and internal routing keys stay hidden. | +| 3. Metadata | META-04 | Session summary metrics | Native totals such as turns, tool calls, tokens, duration, and cost can be merged into the root without double counting resumed activity. | +| 4. Transcript support | HIST-00 | Durable transcript mirroring | Mutable transcripts are copied incrementally into private daemon storage, and journaled high-water references bound live and replay observations. | +| 4. Transcript support | REL-04 | Incremental and partial-record parsing | Transcript readers tolerate append-only growth, partial final records, truncation, compaction, large catch-up batches, and additive schema changes. | +| 4. Reliability | REL-05 | Safe turn and session flushing | Interactive delivery may batch, while turn boundaries, managed runs, idle retirement, shutdown, and explicit flush drain all accepted work. | +| 4. Reliability | REL-06 | Restart recovery | Daemon restart replays journal and transcript mirrors without duplicate spans, lost completions, route drift, or resurrection of closed operations. | +| 4. Reliability | REL-07 | Session resume semantics | A resumed native session extends the existing trace safely after a prior process lifetime closed, including late records and repeated shutdown events. | +| 4. Routing | ROUTE-02 | Experiment and exported-parent routing | Sessions can target an experiment or attach beneath an exported span while preserving the externally supplied trace root. | +| 4. Routing | ROUTE-03 | Profile, organization, and route isolation | Concurrent sessions can use distinct profiles, organizations, destinations, and metadata without credentials or events crossing routes. | +| 4. Correlation | CORR-01 | Cross-agent parent-child linking | Process ancestry and native evidence can link child coding-agent sessions beneath spawning tools, including concurrent, recursive, and restart cases, while ambiguity fails safe. | +| 4. Operations | OPS-01 | Daemon status and delivery diagnostics | Operators can inspect daemon reachability, active sessions, pending delivery, last error, compatibility, and permalink without exposing credentials or payloads. | +| 4. Operations | OPS-02 | Opt-in sensitive diagnostics | Raw event and transcript diagnostics are private, bounded, clearly documented as sensitive, and disabled unless explicitly requested. | +| 5. Product | SETUP-01 | Persistent enable and disable | Setup installs or updates capture idempotently, stores only non-secret routing, preserves unrelated configuration, and removes only Braintrust-managed state. | +| 5. Product | SETUP-02 | Agent-specific doctor/status | Diagnostics cover agent installation, capture activation, plugin and daemon compatibility, route validity, credential availability, and restart or reload requirements. | +| 5. Product | RUN-01 | Invocation-local managed run | One process tree receives an isolated route and temporary capture without mutating persistent configuration or duplicating inherited capture. | +| 5. Product | HIST-01 | Historical transcript import | Stable native sessions can be located safely and replayed through the production translator with project, experiment, or parent overrides. | +| 5. Product | HIST-02 | Live transcript attach | The import parser can follow active sessions incrementally and finalize partial, cancelled, compacted, or resumed sessions without duplicates. | +| 5. Verification | TEST-03 | Product-surface tests | Enable, repeated enable, route changes, disable, doctor, managed run, import, and attach are tested independently in isolated agent homes. | +| 6. Security | SEC-02 | Sensitive-data policy and filesystem safety | Documentation states captured content and redaction limits; journals, transcript mirrors, logs, and temporary files use private permissions and safe path handling. | +| 6. Distribution | DIST-01 | Reproducible package and validation | A deterministic artifact contains every required manifest, hook, adapter, license, and document and validates independently of the source tree. | +| 6. Distribution | DIST-02 | Versioning, publishing, and rollback | Agent and plugin compatibility, release versions, generated distribution repositories or registries, release automation, and rollback are defined and reproducible. | +| 6. Verification | TEST-04 | Automated installed real-agent smoke | Automation installs the packaged artifact, runs the real agent, and asserts trace hierarchy, content, failures, and terminal delivery. | +| 6. Verification | TEST-05 | Real Braintrust ingest smoke | Release verification confirms the packaged integration can deliver and query a trace through a real Braintrust destination, not only a debug or mock sink. | + +The order favors a usable root-and-turn trace early, then adds operation fidelity, +reliability, product surfaces, and release evidence. Tests should be added with +each feature rather than deferred until the verification rows. When one row grows +to describe independently testable behavior, split it into stable feature IDs. diff --git a/scripts/set-plugin-version.py b/scripts/set-plugin-version.py index 6385474..266edcf 100644 --- a/scripts/set-plugin-version.py +++ b/scripts/set-plugin-version.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 -"""Set a release version in each of an agent's per-plugin manifests. +"""Set a release version on every distributed version surface for an agent. Versioning is per-plugin: every plugin under an agent carries its own -.-plugin/plugin.json with a `version` field. A release stamps the given -version into each of those manifests. The marketplace manifest is NOT touched. +.-plugin/plugin.json with a `version` field. Grok's hook adapter also +embeds the plugin version forwarded to the daemon, so its manifest and adapter +constant are stamped together. The marketplace manifest is NOT touched. -Only the version value is rewritten (surgical regex), so the manifests keep -their exact formatting. +Only version values are rewritten, so surrounding files keep their formatting. Usage: set-plugin-version.py (leading 'v' is stripped) """ @@ -19,9 +19,12 @@ MANIFEST_GLOBS = { "claude": "src/plugins/claude/content/plugins/*/.claude-plugin/plugin.json", "codex": "src/plugins/codex/content/plugins/*/.codex-plugin/plugin.json", + "grok": "src/plugins/grok/content/.grok-plugin/plugin.json", } VERSION_RE = re.compile(r'("version"\s*:\s*")[^"]*(")') +GROK_ADAPTER = "src/plugins/grok/content/hooks/forward.sh" +GROK_PLUGIN_VERSION_RE = re.compile(r'^(PLUGIN_VERSION=")[^"]*(")$', re.MULTILINE) def main() -> None: @@ -36,14 +39,26 @@ def main() -> None: if not manifests: sys.exit(f"no plugin manifests found for '{agent}' ({pattern})") - for path in manifests: - text = open(path).read() - new_text, n = VERSION_RE.subn(rf"\g<1>{version}\g<2>", text, count=1) - if n == 0: - sys.exit(f"no version field in {path}") + surfaces = [(path, VERSION_RE, "version") for path in manifests] + if agent == "grok": + surfaces.append((GROK_ADAPTER, GROK_PLUGIN_VERSION_RE, "PLUGIN_VERSION")) + + changes = [] + for path, version_re, label in surfaces: + with open(path) as f: + text = f.read() + new_text, n = version_re.subn( + lambda match: f"{match.group(1)}{version}{match.group(2)}", + text, + ) + if n != 1: + sys.exit(f"expected one {label} field in {path}, found {n}") + changes.append((path, label, new_text)) + + for path, label, new_text in changes: with open(path, "w") as f: f.write(new_text) - print(f"set {os.path.relpath(path)} version -> {version}") + print(f"set {os.path.relpath(path)} {label} -> {version}") if __name__ == "__main__": diff --git a/src/plugins/grok/TODO.md b/src/plugins/grok/TODO.md new file mode 100644 index 0000000..d282c89 --- /dev/null +++ b/src/plugins/grok/TODO.md @@ -0,0 +1,372 @@ +# Grok tracing conformance TODO + +Audit date: 2026-09-03 +Audited agent: `grok 1.0.13 (5e9a58528b76) [stable]` +Canonical source: `grok` + +## Verdict + +**Core trace prototype; not yet core-conformant or product-complete.** + +The repository has a thin hook forwarder, transcript mirroring, a registered +production translator, root/turn/LLM/tool rows, aggregate usage, deterministic +fixture replay, package validation, and release plumbing. The largest gaps are: + +1. The native contract has not been captured and versioned. Grok's documented + hooks do not expose LLM calls and do not guarantee ordered blocking delivery + for every lifecycle event. The translator depends on undocumented + `updates.jsonl` as its authoritative data stream, uses `events.jsonl` only + for optional tool enrichment, and snapshots `system_prompt.txt` for the + first reconstructed LLM input. +2. Current LLM spans are reconstructed from output stream boundaries, not + proven native LLM request/response boundaries. The system prompt is + observable, but full request bodies and per-call usage remain unavailable. +3. Session-end spelling, resume behavior, source namespacing, cancellation, and + bounded-state requirements are incomplete. +4. Permission, subagent, compaction, web/MCP, setup, doctor, managed run, + import/attach, and Grok-specific end-to-end tests are absent. +5. The distributed artifact still contains a placeholder license and has no + installed real-agent or real-ingest smoke test. + +Do not describe the integration as full-fidelity or production-ready until the +P0 items and the release gates below are complete. + +## Evidence inspected + +- Official Grok hooks documentation: + +- `src/plugins/grok/content/hooks/hooks.json` +- `src/plugins/grok/content/hooks/forward.sh` +- `src/plugins/grok/content/README.md` +- `src/plugins/grok/test/test_hook.sh` +- `src/plugins/grok/{build,validate,publish}.sh` +- `src/plugins/grok/local-dev.sh` +- `bt-daemon/src/translate/grok.rs` +- `bt-daemon/src/{dispatch,ids,journal,server,setup,trace_command,trace_runtime}.rs` +- `bt-daemon/src/transcript_import/` +- `bt-daemon/tests/grok_translator.rs` +- `bt-daemon/tests/fixtures/grok/transcript/` +- `.github/workflows/{ci,_release,release,test-release}.yml` + +The official hook documentation is not versioned to Grok 1.0.13. It documents +CamelCase lifecycle names and a common payload, but not timestamps, turn IDs, +shared operation IDs, cross-event ordering, LLM events, transcript paths, or the +`updates.jsonl`, `events.jsonl`, and `system_prompt.txt` formats. Repository +fixtures therefore prove observed behavior, not a stable compatibility +contract. + +## Native support matrix + +| # | Native requirement | Status | Evidence / gap | +|---|---|---|---| +| 1 | Millisecond timestamps | Partial | The daemon stamps hook capture time. Transcript fixtures contain `_meta.agentTimestampMs` and RFC3339 `ts`; the hook docs do not promise a native event timestamp. | +| 2 | Stable session, turn, and operation IDs | Partial | `sessionId`, `promptIndex`/`promptId`, and `toolCallId` are observed. No native LLM operation ID is present; the translator derives LLM identity from `streamStartMs`. | +| 3 | Turn attribution on every operation | Partial | Some transcript completions carry `promptId`; the fixture's tool start does not. The translator currently uses one implicit current turn. | +| 4 | Paired start/stop operations | Partial | Tool call/update records and user/turn-completed records exist. LLM spans are inferred from stream changes rather than paired native call events. | +| 5 | LLM call start/stop with full request/response | No | Grok documents no LLM hook. The first reconstructed LLM can include the native `system_prompt.txt` snapshot, but exact provider requests remain unavailable; output is agent stream content, not a proven full provider response. | +| 6 | Tool/web/MCP start/stop | Partial | Generic tool start/update data is observed. Web and MCP coverage/classification are unproven. | +| 7 | Turn start/stop | Partial | `user_message_chunk` and `turn_completed` are observed, but the stability and uniqueness of `promptIndex`/`promptId` across resume are not documented. | +| 8 | Recursive subagent events with ancestry | Unknown | `SubagentStart`/`SubagentStop` hooks are registered, but there is no captured payload, transcript fixture, recursive ancestry evidence, or translator support. | +| 9 | Session shutdown | Yes | `SessionEnd` is documented and registered. Translator/session flush semantics still need the fixes below. | +| 10 | Ordered blocking delivery | No evidence | Grok documents `PreToolUse` as the only blocking event. The adapter waits for daemon acknowledgement when invoked, but Grok does not document ordered blocking execution for all passive lifecycle hooks. | + +**Fallback verdict: fallback-viable for a core root/turn/tool trace, with fidelity +loss.** The undocumented session files currently supply the system prompt, +assistant stream output, tool activity, turn completion, and aggregate usage. +They do not prove full LLM request/response boundaries, and schema/version +stability remains unknown. + +## Section 11 conformance report + +| Capability | Status | Evidence / blocker | +|---|---|---| +| Source registration | Partial | `grok` is used by the adapter, settings path, translator factory, registry, and fixtures. Setup, doctor, run, and import enums omit Grok. IDs, delivery keys, and journal paths are not source-namespaced. | +| Native event audit | Partial | All documented lifecycle hooks are registered and one transcript schema is fixture-tested for 1.0.13. No real raw-hook fixture set, schema provenance, compatibility window, ordering guarantee, or transcript stability evidence exists. | +| Fail-open capture | Partial | Adapter forwards stdin synchronously and its test covers forwarding failure and missing `bt`/`curl`. It tries `curl | bash` installation inside a 5-second hook and does not forward source/plugin versions or test terminal flush/order/reconnection. | +| Session root | Partial | Deterministic task root, external attachment, and native/documented session-end spellings exist. Source is absent from the ID namespace, and a closed translator cannot reopen/extend the root safely on resume. | +| Turn spans | Partial | User input, user-visible assistant output without reasoning, timing, stop metadata, failure/cancellation handling, and sequential multi-turn fixtures exist. Concurrent turns and resumed sessions remain unproven. | +| LLM spans/metrics | Partial | Native model names and reconstructed stream output/timing exist. The first LLM includes the native system prompt and first user message; each LLM output separates observed reasoning from assistant response. Exact later-call provider inputs and serialization remain unavailable. Aggregate usage remains on the turn and is also attributed once to the final reconstructed LLM with explicit turn-scope metadata; true per-call usage remains unavailable. Provider/finish/TTFT and proven request boundaries are unavailable. Raw `costUsdTicks` is preserved without plugin-side estimation. | +| Tool/web/MCP spans | Partial | Generic native-ID tool spans include input/output/timing/kind and basic errors. Web/MCP classification, structured failure fields, cancellation tags, nested ancestry, and concurrency coverage are missing. | +| Permission events | Not started | `PermissionDenied` is captured but ignored by the translator; request/allow/modify evidence is unknown. | +| Subagents | Not started | Start/stop hooks are captured but ignored; no hierarchy fixture or recursive evidence exists. | +| Compaction | Not started | Pre/post hooks are captured but ignored; no transcript or payload fixture exists. | +| Failures/cancellation | Partial | Tool failure becomes `error`; dangling work is marked incomplete. Failed/cancelled turns and session hooks are not translated into error/status metadata, and cancellation is not distinguished from failure. | +| Session/Git/custom metadata | Partial | Source/session/cwd/workspace/permission/transcript and optional versions are supported; route metadata precedence is implemented. Live version forwarding, shared Git enrichment, execution mode normalization, and Grok-specific precedence tests are missing. | +| Deterministic replay and bounded state | Partial | Incremental/full replay equivalence and bounded recent-ID caches exist. Open tools, assistant/LLM output, LLM ID vectors, and per-boundary record vectors are unbounded; duplicate records, malformed lines, truncation/regrowth, and resume are not covered. | +| Project/experiment/parent routing | Partial | Shared route and sink types support all destinations and the translator consumes attached IDs. Grok has no setup/run/import surface or route/attachment pipeline test. | +| Enable/disable/doctor | Not started | `SetupAgent` and `DoctorAgent` omit Grok. The README explicitly refers to a future setup command. | +| Managed run | Not started | `RunSource` omits Grok. `local-dev.sh` demonstrates a development-only isolated home/hook injection path, not the public route-isolated product contract. | +| Import/live attach | Blocked | `ImportSource` and `transcript_import` support only Codex and Claude. Implementation is absent, and historical roots, authoritative session lookup, stable `updates.jsonl` tail semantics, and independent optional `events.jsonl` enrichment are not established. | +| Security and data-handling review | Partial | Credentials stay out of the adapter and shared routes/journals redact auth; daemon storage is private. The hook auto-installs remote code, raw payloads are logged whenever Grok translator debug logging is enabled, and redaction limitations/retention are not fully documented. | +| Build/validation/publishing | Partial | Build, validation, version stamping, dry-run deployment, production deployment, tags, and releases include Grok. The artifact license is `TODO Apache 2.0 (placeholder)`; compatibility, rollback, supported platforms, and installed-artifact verification are missing. | +| Translator and pipeline tests | Partial | Translator tests cover first-LLM system/user input, reasoning/response output, turn content separation, incremental replay, malformed/partial records, replacement, terminal spellings, and asymmetric missing-file recovery. The packaged adapter-to-journal-to-translator-to-sink test covers independent mirrors, system prompt capture, first-LLM input, restart replay, route isolation, terminal flush, and late tool enrichment. | +| Installed real-agent smoke | Not started | `local-dev.sh` is manual and does not assert an installed distribution trace. CI installs/tests other agents only; release smoke runs only Codex or Antigravity. | + +## P0: establish a trustworthy core trace + +### Native evidence and compatibility + +- [ ] **AUDIT-01 / AUDIT-02:** Capture provenance-labeled real hook payloads and + complete `updates.jsonl`, `events.jsonl`, and `system_prompt.txt` session + evidence from Grok 1.0.13 for a normal turn, tool + success/failure/cancellation, permission denial, subagent, compaction, + session exit, and resume. +- [ ] Record which fields are documented versus empirically observed, whether + passive hooks are ordered and awaited, transcript locations/rotation rules, + ID stability, timestamp units, and schema behavior across the minimum and + current supported Grok versions. +- [x] Preserve `streamStartMs` groups as the observable LLM/tool/LLM sequence, + while documenting that they are reconstructed stream boundaries rather than + proven native requests. Attribute aggregate usage once to the final group + with explicit turn-scope metadata; never assign it to every group. +- [ ] Treat full LLM request bodies, per-call native usage, web/MCP identity, + subagent ancestry, and skill/background semantics as **Blocked**, not N/A, + until the audit produces evidence or Grok documents their absence. + +### Capture and terminal behavior + +- [x] **CAP-01 / SEC-01:** Remove installation (`curl | bash`) from + `hooks/forward.sh`. Installation belongs to `bt trace enable grok`; a missing + CLI should produce a bounded fail-open diagnostic without changing the host. +- [ ] **CAP-02:** Forward Grok and plugin versions into the envelope. The plugin + manifest contains version `0.1.0`, but `run_hook` currently writes + `plugin_version: None` and the adapter supplies no `--source-version`. +- [x] Fix the daemon handshake to send `env.plugin_version` as + `client.plugin_version`; `forward_envelope` currently mislabels + `env.source_version` as the plugin version during both initialize attempts. +- [x] **CAP-03 / REL-05:** Define and test exact native event spelling. Official + docs and `hooks.json` use `SessionEnd`/`Stop`; the adapter test uses + snake_case, while `GrokTranslator::handle` checks only `session_end` and the + shared flush path checks `SessionEnd`, `Stop`, and `SubagentStop`. +- [x] Make the terminal timeout coherent. Every Grok hook is capped at 5 + seconds, but `flush_session` defaults to 10 seconds before startup/IPC + overhead, so Grok can terminate `SessionEnd` before the flush's own bound. +- [x] Capture independent high-water boundaries for `updates.jsonl`, + `events.jsonl`, and `system_prompt.txt`. Each sibling is mirrored + independently; a missing optional file omits only its reference and does not + block the authoritative updates stream. +- [ ] Prove passive-event ordering and process-exit delivery. If Grok cannot + await all relevant hooks, design transcript high-water/catch-up behavior that + cannot lose the final records; do not claim ordered blocking capture. +- [ ] Test paths with spaces/non-ASCII, daemon startup timeout, repeated missing + CLI diagnostics, forwarding timeout, and terminal flush failure. Preserve + Grok's exit and permission behavior in every case. + +### Identity, resume, and correlation + +- [ ] **ID-01 / REL-02:** Namespace span IDs by canonical source as well as + native session ID. `ids::span_id` currently hashes only `session_id + key`. +- [ ] Namespace daemon `DeliveryKey`, journal files, transcript mirrors, managed + run records, and import processor keys by source where collisions are + possible. Current delivery/journal identity uses session ID plus route but not + source. +- [ ] This identity work changes common daemon contracts; obtain approval before + changing the shared ID, journal, or route schema, and add migration/recovery + coverage for existing journals. +- [ ] **TRACE-01 / REL-07:** Replace permanent `root_closed` behavior with tested + resume semantics. A later process lifetime for the same native session must + extend the same logical root without leaving its end time before new child + work or duplicating the root. +- [ ] **TRACE-14:** Either prove Grok serializes turns and operations and lock the + assumption with fixtures/tests, or replace `current_turn`/`open_llm` with + native-ID keyed correlation. Ambiguous ancestry must fail safe. + +### Translator correctness and bounds + +- [x] **TRACE-03 / TRACE-11:** Keep aggregate usage on the turn and also copy it + once to the final reconstructed LLM for LLM-view visibility. Mark the copy + with `usage_scope: "turn"` and `usage_attribution: "last_llm"` so it is not + represented as true per-call evidence, especially when `usage.modelCalls` + exceeds one. +- [x] Preserve native `costUsdTicks` as `cost_usd_ticks`; do not calculate + `estimated_cost` in the plugin. +- [x] **TRACE-08:** Map native failed, cancelled, interrupted, and incomplete + turns/tools to explicit error and outcome metadata/tags. Never represent + cancellation as generic failure or success. +- [x] Deduplicate a `user_message_chunk` before closing the current turn or + incrementing `turn_seq`. The current order mutates state first, then rejects + an already-emitted turn key, which can drop all later output for the active + turn. +- [x] **REL-03:** Bound `open_tools`, assistant output, LLM output, + `llm_span_ids`, and records processed per call. Use `drain_pending` for bounded + catch-up instead of loading all new transcript records into a `Vec`. +- [ ] **REL-04:** Add explicit behavior for partial final writes, malformed + complete lines, file replacement/truncation/regrowth, missing mirror files, + additive fields, and version variants. One bad record must not silently stall + every later record forever. +- [x] Make cursor advancement asymmetric. Updates read and cursor advancement + remain transactional and authoritative. An unreadable or short events mirror + leaves the events cursor unchanged, logs a debug diagnostic, and cannot + discard update `SpanOp`s or delay terminal handling; a later hook retries it. +- [x] Close roots/turns at `max(hook timestamp, last observed transcript + timestamp)` so capture-time skew cannot place an ancestor end before its + child activity. +- [x] **OPS-02 / SEC-02:** Remove unconditional raw payload fields from normal + debug logging or put them behind a separate explicit sensitive-diagnostics + opt-in with private, bounded storage and documentation. + +### Core verification + +- [ ] **TEST-01:** Add Grok fixtures for multiple turns, exact `SessionEnd`, + flush, shutdown, resume, duplicates inside a transcript, missing turn/tool/LLM + boundaries, unknown records, additive fields, malformed/partial records, + attachment, custom metadata precedence, version variants, cancellation, and + state limits. +- [ ] Add assertions for root/turn/operation start and end times, remaining + output variants, errors, cancellation tags, metric names/units, stable IDs, + native system/user input provenance, reasoning/response separation, and no + invented later-call provider history or per-call usage. +- [x] **TEST-02:** Add a Grok-specific pipeline test covering packaged adapter + invocation, `HookArgs` mapping, envelope timestamps/versions/process capture, + independent transcript and system prompt mirroring, redacted journal, + production translator, route-isolated sink rows, restart replay, late + optional events enrichment, and terminal flush. +- [ ] Add a source-collision regression using the same native session ID for + Grok and another source on the same route. + +## P1: native operation fidelity + +- [ ] **TRACE-04:** Complete tool missing-start/missing-stop, duplicate, + reordered, concurrent, structured failure, cancellation, and defensive-flush + behavior using native `toolCallId`. +- [ ] **TRACE-09:** Classify observable web and MCP calls as tool spans while + retaining native transport/server/method metadata. Keep Blocked if the audit + cannot distinguish them. +- [ ] **TRACE-05:** Translate permission requests and decisions when observable. + Current capture has only documented `PermissionDenied`; do not invent allow + or modification decisions. +- [ ] **TRACE-10:** Represent explicit skill loads/invocations only if a native + payload or vetted transcript proves them. +- [ ] **TRACE-03 / TRACE-12:** Add provider, native request/response, finish + reason, TTFT, API duration, retry, fingerprint, and cost only where native + evidence exists. Document unavailable fields. +- [ ] **TRACE-06:** Add recursive subagent task spans keyed and parented by + native ancestry; cover sub-subagents and concurrent children. +- [ ] **TRACE-13:** Preserve observable background/asynchronous operation IDs + until native completion/cancellation or defensive flush. +- [ ] **TRACE-07:** Add compaction/session-tree spans from `PreCompact` and + `PostCompact` only after their payloads and correlation IDs are captured. + +## P2: metadata, reliability, and routing + +- [ ] **META-01 / META-02:** Pass live agent/plugin versions and use the shared + `GitMetadataCache` for repository root, revision, branch, remote, and dirty + state when a worktree is available. +- [ ] Normalize cwd/workspace/execution/permission/transcript provenance while + preserving raw native fields needed for drift diagnosis. +- [ ] **META-03:** Test that route metadata reaches the root, `_bt_*` keys are + hidden, and integration-owned source/session/version keys win. +- [ ] **META-04:** Add replay-safe native session summary metrics only when the + transcript exposes authoritative totals without double counting resume. +- [x] **HIST-00 / REL-06:** Add Grok pipeline recovery tests proving the + authoritative updates mirror, optional system prompt snapshot, and journal + boundary survive daemon restart without reading future bytes or duplicating + rows, while an optional events mirror may appear later and enrich the + existing deterministic tool span. +- [ ] **ROUTE-01 / ROUTE-02 / ROUTE-03:** Exercise project ID/name, experiment, + exported parent, profile, organization, metadata, and concurrent route + isolation through Grok entry points. +- [ ] **CORR-01:** Prove daemon-captured process ancestry links nested Grok and + cross-agent sessions only when unambiguous, including concurrent/restart + cases; otherwise keep the session standalone. +- [ ] **OPS-01:** Include Grok in daemon/doctor delivery diagnostics with active + sessions, pending delivery, last error, compatibility, activation/reload + requirement, and permalink without payloads or credentials. + +## P3: product surfaces + +### Persistent setup + +- [x] **SETUP-01:** Add Grok to `SetupAgent`, source/display mappings, public + help, and runtime dispatch. +- [x] Implement marketplace/plugin install or update, enablement, repeated + enable, route changes, and reversible disable/uninstall without touching + unrelated Grok plugins, hooks, trust, or config. +- [x] Preserve Grok's trust boundary. Do not use global `--trust` as an + unattended shortcut unless the product explicitly owns and documents that + decision. +- [x] Persist only the shared non-secret route at `~/.grok/braintrust.json` with + private atomic writes. +- [ ] **SETUP-02:** Add Grok doctor/status checks for executable/version, + installed/enabled plugin, daemon/plugin compatibility, route/auth readiness, + and Grok 1.0.13's required `/reload-plugins` activation workaround. + +### Managed run + +- [ ] **RUN-01:** Add Grok to `RunSource`, parsing, executable selection, source + mapping, and help. +- [ ] Replace the development-only `local-dev.sh` mutation path with a safe + invocation-local isolated home/plugin or hook injection that does not rewrite + persistent setup and does not copy credentials into a less-safe location. +- [ ] Suppress inherited Braintrust capture only for the managed process tree, + inject every required hook, preserve trust review, stdio, arguments, exit + status, and interruption semantics, and flush only accepted run sessions. + +### Import and attach + +- [ ] **HIST-01:** Establish documented active/archive roots and deterministic + session lookup for Grok's multi-file session directory before adding Grok to + `ImportSource`. +- [ ] Implement one Grok parser that emits synthetic `grok` envelopes with + bounded references to authoritative `updates.jsonl` evidence, the optional + `system_prompt.txt` snapshot, and optional `events.jsonl` enrichment, matching + live capture. Reuse `GrokTranslator`; do not build spans in the importer. +- [ ] **HIST-02:** Tail `updates.jsonl` incrementally with authoritative + high-water marks and tail `events.jsonl` independently as optional + enrichment; handle no-growth polls, partial writes, truncation, compaction, + cancellation, resume, and finalization without duplicates. +- [ ] Test project/experiment/parent overrides without changing persistent Grok + settings, plus invalid/ambiguous IDs and unsafe symlink traversal. + +## P4: distribution and release evidence + +- [ ] **SEC-02:** Document captured prompts, reasoning summaries, responses, + file paths, tool payloads, transcript mirroring, retention, redaction limits, + and the explicit sensitive-debug opt-in before release. +- [x] **DIST-01:** Replace `content/LICENSE` (`TODO Apache 2.0 (placeholder)`) + with the real license and make validation reject placeholders. +- [x] Document supported Grok versions, operating systems, shell requirements, + plugin activation/reload behavior, setup/run/import/disable commands, fidelity + limitations, and troubleshooting. The current Bash adapter has no Windows + compatibility statement or CI coverage. +- [ ] **DIST-02:** Add documented rollback and installed-version compatibility + checks to the existing build/version/publish/tag/release flow. +- [ ] **TEST-03:** Add isolated product tests for first/repeated enable, route + update, unrelated configuration preservation, disable, doctor, managed run, + import, and attach. +- [ ] Add Grok minimum/current compatibility jobs and make CI install/report the + Grok version rather than validating only JSON when Grok is absent. +- [ ] **TEST-04:** Add an installed-distribution real-Grok smoke that asserts + root/turn/tool hierarchy, failure/cancellation, versions/provenance, and final + delivery. A successful process exit alone is insufficient. +- [ ] **TEST-05:** Add release verification against a real Braintrust + destination and query the resulting trace. Keep credentials confined to the + host/release secret and skip cleanly when unavailable. + +## Existing implementation worth preserving + +- Raw stdin forwarding and fail-open exit behavior in + `content/hooks/forward.sh`. +- Registration of all currently documented Grok lifecycle events in + `content/hooks/hooks.json`. +- Daemon-owned incremental transcript mirrors and journaled high-water + references in `dispatch.rs`/`transcript_mirror.rs`. +- Sink-neutral `SpanOp::Insert`/`Merge` translation and shared Braintrust sink. +- Native `toolCallId` correlation and aggregate usage remaining on the turn when + multiple LLM rows are observed. +- Bounded recent completed-ID caches and deterministic incremental/full fixture + replay. +- Shared route redaction, private daemon data directory, build validation, + version stamping, dry-run publishing, and release tagging. + +## Verification snapshot + +- `make validate-grok`: **passed**; this covers package shape, hook + registration, raw forwarding, and fail-open adapter behavior. +- `cargo +1.88.0 test --manifest-path bt-daemon/Cargo.toml --all-features + --locked grok`: **passed** with 18 Grok tests across 16 test binaries. The + explicit 1.88.0 toolchain is required because this workstation's default + Rust 1.86.0 is older than the resolved `darling` dependency requires. +- Existing CI runs the complete Rust suite on stable Rust, but no CI job installs + or exercises Grok. Repository coverage includes the Grok-specific packaged + hook-to-journal-to-translator-to-sink pipeline test described above. diff --git a/src/plugins/grok/build.sh b/src/plugins/grok/build.sh new file mode 100755 index 0000000..79e0261 --- /dev/null +++ b/src/plugins/grok/build.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +TARGET_DIR="${1:?usage: build.sh }" +SRC_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +mkdir -p "$TARGET_DIR" +rsync -a --delete --exclude '.git' "$SRC_DIR/content/" "$TARGET_DIR/" +echo "Built grok dist into $TARGET_DIR." diff --git a/src/plugins/grok/content/.grok-plugin/plugin.json b/src/plugins/grok/content/.grok-plugin/plugin.json new file mode 100644 index 0000000..b0d3ad8 --- /dev/null +++ b/src/plugins/grok/content/.grok-plugin/plugin.json @@ -0,0 +1,8 @@ +{ + "name": "trace-grok", + "description": "Beta Braintrust tracing for Grok sessions with transcript-reconstructed LLM and tool spans.", + "version": "0.1.0", + "author": { + "name": "Braintrust" + } +} diff --git a/src/plugins/grok/content/LICENSE b/src/plugins/grok/content/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/src/plugins/grok/content/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/src/plugins/grok/content/README.md b/src/plugins/grok/content/README.md new file mode 100644 index 0000000..1e9961f --- /dev/null +++ b/src/plugins/grok/content/README.md @@ -0,0 +1,197 @@ +# Braintrust tracing for Grok (v1 beta) + +This plugin traces interactive Grok sessions to Braintrust. It is useful for +observing turns, model output, tool calls, timing, and native aggregate usage, +but it is a beta integration: it is neither full-fidelity nor production-ready. + +## Compatibility + +The supported evidence baseline is **Grok 1.0.13**. Hook registration follows +Grok's documented lifecycle names, and the repository's fixtures cover that +version's `updates.jsonl`, `events.jsonl`, and `system_prompt.txt` shape. Later +Grok versions may work, but are not yet compatibility-covered. + +The plugin supports **macOS and Linux** and requires **Bash**. Windows is not +supported. Install `grok` and a `bt` CLI version that includes +`bt trace enable grok`, and configure Braintrust authentication in `bt` before +enabling tracing. + +## Setup + +Install the published plugin, enable it in Grok, and save a non-secret +Braintrust route with: + +```bash +bt trace enable grok +``` + +The command is safe to repeat. It installs +`braintrustdata/braintrust-grok-plugin` with `--trust`, updates that published +plugin when already installed, enables it, and writes only the +Braintrust-owned tracing settings. A same-named local or forked `trace-grok` +plugin is replaced with the published package; every other Grok plugin and +configuration entry is preserved. + +Grok 1.0.13 discovers a newly installed or updated plugin but does not activate +its hooks in a session that is already open. In each active session, run this +before sending the next prompt: + +```text +/reload-plugins +``` + +Wait for Grok to report that the hooks were reloaded. New Grok sessions load the +installed plugin normally. + +### Manual plugin installation + +To inspect or install the published package yourself, run: + +```bash +grok plugin install braintrustdata/braintrust-grok-plugin --trust +grok plugin enable trace-grok +``` + +Manual Grok installation alone does not create a Braintrust destination. Run +`bt trace enable grok` afterward to reconcile the installed plugin and write +its route, then use `/reload-plugins` in any session that was already open. + +## Diagnose or disable + +Inspect the installed plugin, tracing route, authentication selection, and +known compatibility warnings without changing configuration: + +```bash +bt trace doctor grok +``` + +For Grok 1.0.13, doctor also reminds you when `/reload-plugins` is required. + +To uninstall the published `trace-grok` package and remove its Braintrust-owned +route while preserving unrelated Grok configuration: + +```bash +bt trace disable grok +``` + +Disable removes only a `trace-grok` whose recorded source matches +`braintrustdata/braintrust-grok-plugin`; it leaves a same-named local or forked +plugin untouched. Disabling does not delete traces already sent to Braintrust +or the daemon's existing local recovery records. + +## Trace shape and fidelity + +Hooks synchronously and fail-open forward native lifecycle payloads to the +local Braintrust daemon. The adapter contains no Braintrust credential and does +not send data directly to Braintrust. Hooks are control-plane wake-up and flush +signals, plus terminal fallback when the transcript lacks a terminal record. +`updates.jsonl` is the trace data source of truth. `events.jsonl` is optional, +independently tailed enrichment for completed tool duration and outcome; a +missing, unreadable, malformed, or lagging events stream never delays updates, +terminal handling, or flush. A later hook can merge recovered enrichment onto +the same deterministic tool span identity. The daemon also snapshots the +optional sibling `system_prompt.txt` and attaches it to the first reconstructed +LLM when available; prompt capture failure does not block trace construction. + +A traced session contains a `Grok` root span, turn task spans, reconstructed LLM +spans, and child tool spans. A turn span is the user-facing exchange: its input +is the user message and its output contains only observable assistant responses, +never reasoning. Tool spans retain transcript-observable input, output, timing, +and failure details. Turn spans also retain Grok's native aggregate tokens, +cache, reasoning, raw cost ticks, model-call count, and API duration when +present. + +Grok's transcript does not expose a complete provider request or a native LLM +span boundary. The translator reconstructs each observed model stream from +`streamStartMs` evidence. The first reconstructed LLM receives the available +conversation start as `system` and `user` messages: the native +`system_prompt.txt` snapshot when present, followed by the first turn's user +message. It is marked with `system_prompt_included`, `user_message_included`, +and an `input_scope` such as `"system_and_user"`. Each LLM output is one +standard assistant message so Braintrust renders it as an LLM response. Its +string `content` is the observed assistant response; an optional `reasoning` +field contains Grok thought chunks when emitted. Later calls do not claim an +exact reconstructed input +because Grok does not expose their serialized provider history. LLM spans +retain `input_unavailable: true` to make that limitation explicit, plus +`boundary_source: "streamStartMs"` and +`trace_source: "session_transcript"`. The visible LLM span remains named +`{model} call {sequence}` and has the Braintrust LLM span type. + +Usage is aggregate turn evidence, not per-call evidence. It remains on the turn +and is never divided among reconstructed calls. To expose the aggregate in LLM +views without double-attributing it across calls, the translator also copies it +to only the final reconstructed LLM span. That merge is labeled +`usage_scope: "turn"` and `usage_attribution: "last_llm"`; on a multi-call turn +its metrics describe the whole turn, not the final call alone. Native +`costUsdTicks` is reported as `cost_usd_ticks`; the plugin does not calculate +`estimated_cost`. + +The v1 beta does **not** provide: + +- `bt trace run grok` managed-run support; +- `bt trace import grok` or historical transcript import/attach support; +- dedicated subagent hierarchy or faithful subagent lifecycle spans; +- permission request/decision spans; +- compaction spans or compaction-aware transcript fidelity; +- faithful web-search, web-fetch, or MCP transport/server/method + classification (observable operations may appear as generic tool spans); +- complete provider requests or per-call usage for multi-call turns. + +Hook events for unsupported lifecycle areas may still be journaled for ordering +and recovery; their presence does not imply dedicated translated spans. + +## Sensitive data, redaction, and retention + +Treat Grok transcripts, daemon journals, transcript mirrors, and Braintrust +traces as sensitive. Depending on the session, captured content can include +system instructions, user prompts, assistant messages, reasoning summaries, +repository and file paths, tool inputs and outputs, command output, errors, and +source-control metadata. Secrets included in any of that content can be +captured too. + +Braintrust credentials are not stored in the plugin, hook command, route, +envelope, or journal. The daemon keeps local files private and redacts routing +credentials, but the beta does **not** comprehensively redact prompt, +response, reasoning, or tool content. It also provides no plugin-level custom +redaction rules. + +Local recovery journals and transcript mirrors are append-only while active and +become eligible for best-effort age collection after seven days; their contents +are not size-capped or truncated before collection. Do not treat that cleanup +as immediate or secure deletion. Data already delivered to Braintrust follows +the retention and deletion policy of the selected Braintrust +organization/project. Disabling the plugin is not a data-deletion operation. +Avoid placing secrets in agent input, restrict access to the host account, and +apply appropriate Braintrust retention and deletion controls. + +## Troubleshooting + +### No trace appears + +1. Run `bt trace doctor grok` and resolve any reported installation, route, or + authentication problem. +2. Run `grok plugin list --json` and confirm that `trace-grok` is installed and + enabled. +3. If setup or update happened while Grok was open, run `/reload-plugins` in + that session and wait for the reload confirmation before another prompt. +4. Re-run `bt trace enable grok` if `bt` was replaced, the route was removed, or + the plugin was disabled. + +Tracing is fail-open by design: a missing `bt`, unavailable daemon, forwarding +timeout, or translation failure must not interrupt Grok, so a normal Grok turn +does not by itself prove that tracing succeeded. + +### A trace is partial + +The daemon can translate only transcript records visible at a lifecycle +boundary. Keep the native transcript readable for the duration of the session, +let the turn finish, and exit Grok normally so the native `session_end` hook can +flush terminal state. Missing full LLM input, per-call metrics on multi-call +turns, dedicated permission/subagent/compaction spans, and web/MCP +classification are known fidelity limits rather than setup failures. + +### Hooks do not activate after an update + +This is a known Grok 1.0.13 activation behavior. Run `/reload-plugins` in every +already-open session, or start a new Grok session. diff --git a/src/plugins/grok/content/hooks/forward.sh b/src/plugins/grok/content/hooks/forward.sh new file mode 100755 index 0000000..ca0654a --- /dev/null +++ b/src/plugins/grok/content/hooks/forward.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# Thin, fail-open bridge from Grok hooks to the shared Braintrust daemon. + +PLUGIN_VERSION="0.1.0" + +resolve_bt() { + if [[ -n "${BT_BIN:-}" && -x "$BT_BIN" ]]; then + printf '%s\n' "$BT_BIN" + return 0 + fi + if command -v bt >/dev/null 2>&1; then + command -v bt + return 0 + fi + + local candidate + for candidate in \ + "${XDG_BIN_HOME:-${HOME:-}/.local/bin}/bt" \ + "${CARGO_HOME:-${HOME:-}/.cargo}/bin/bt"; do + if [[ -n "$candidate" && -x "$candidate" ]]; then + printf '%s\n' "$candidate" + return 0 + fi + done + return 1 +} + +BT_BIN="$(resolve_bt || true)" +if [[ -z "$BT_BIN" ]]; then + printf 'trace-grok: bt CLI not found; tracing skipped\n' >&2 + exit 0 +fi + +BT_ARGS=( + trace hook + --source grok + --plugin-version "$PLUGIN_VERSION" + --session-id-field sessionId + --event-field hookEventName + --transcript-path-field transcriptPath +) +if [[ -n "${GROK_VERSION:-}" ]]; then + BT_ARGS+=(--source-version "$GROK_VERSION") +fi + +"$BT_BIN" "${BT_ARGS[@]}" || true +exit 0 diff --git a/src/plugins/grok/content/hooks/hooks.json b/src/plugins/grok/content/hooks/hooks.json new file mode 100644 index 0000000..71d9c79 --- /dev/null +++ b/src/plugins/grok/content/hooks/hooks.json @@ -0,0 +1,49 @@ +{ + "hooks": { + "SessionStart": [ + { "hooks": [{ "type": "command", "command": "bash \"${GROK_PLUGIN_ROOT}/hooks/forward.sh\"", "timeout": 5 }] } + ], + "UserPromptSubmit": [ + { "hooks": [{ "type": "command", "command": "bash \"${GROK_PLUGIN_ROOT}/hooks/forward.sh\"", "timeout": 5 }] } + ], + "PreToolUse": [ + { "matcher": ".*", "hooks": [{ "type": "command", "command": "bash \"${GROK_PLUGIN_ROOT}/hooks/forward.sh\"", "timeout": 5 }] } + ], + "PostToolUse": [ + { "matcher": ".*", "hooks": [{ "type": "command", "command": "bash \"${GROK_PLUGIN_ROOT}/hooks/forward.sh\"", "timeout": 5 }] } + ], + "PostToolUseFailure": [ + { "matcher": ".*", "hooks": [{ "type": "command", "command": "bash \"${GROK_PLUGIN_ROOT}/hooks/forward.sh\"", "timeout": 5 }] } + ], + "PermissionDenied": [ + { "matcher": ".*", "hooks": [{ "type": "command", "command": "bash \"${GROK_PLUGIN_ROOT}/hooks/forward.sh\"", "timeout": 5 }] } + ], + "Stop": [ + { "hooks": [{ "type": "command", "command": "bash \"${GROK_PLUGIN_ROOT}/hooks/forward.sh\"", "timeout": 5 }] } + ], + "StopFailure": [ + { "matcher": ".*", "hooks": [{ "type": "command", "command": "bash \"${GROK_PLUGIN_ROOT}/hooks/forward.sh\"", "timeout": 5 }] } + ], + "StopCancelled": [ + { "matcher": ".*", "hooks": [{ "type": "command", "command": "bash \"${GROK_PLUGIN_ROOT}/hooks/forward.sh\"", "timeout": 5 }] } + ], + "Notification": [ + { "matcher": ".*", "hooks": [{ "type": "command", "command": "bash \"${GROK_PLUGIN_ROOT}/hooks/forward.sh\"", "timeout": 5 }] } + ], + "SubagentStart": [ + { "matcher": ".*", "hooks": [{ "type": "command", "command": "bash \"${GROK_PLUGIN_ROOT}/hooks/forward.sh\"", "timeout": 5 }] } + ], + "SubagentStop": [ + { "matcher": ".*", "hooks": [{ "type": "command", "command": "bash \"${GROK_PLUGIN_ROOT}/hooks/forward.sh\"", "timeout": 5 }] } + ], + "PreCompact": [ + { "matcher": ".*", "hooks": [{ "type": "command", "command": "bash \"${GROK_PLUGIN_ROOT}/hooks/forward.sh\"", "timeout": 5 }] } + ], + "PostCompact": [ + { "matcher": ".*", "hooks": [{ "type": "command", "command": "bash \"${GROK_PLUGIN_ROOT}/hooks/forward.sh\"", "timeout": 5 }] } + ], + "SessionEnd": [ + { "matcher": ".*", "hooks": [{ "type": "command", "command": "bash \"${GROK_PLUGIN_ROOT}/hooks/forward.sh\"", "timeout": 15 }] } + ] + } +} diff --git a/src/plugins/grok/local-dev.sh b/src/plugins/grok/local-dev.sh new file mode 100755 index 0000000..a661656 --- /dev/null +++ b/src/plugins/grok/local-dev.sh @@ -0,0 +1,478 @@ +#!/usr/bin/env bash +# Build and install trace-grok, run the standalone development daemon, and +# launch Grok with hooks routed to that daemon. The production bt CLI is not +# required for this local translator development loop. +set -euo pipefail +umask 077 + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" +DIST_DIR="$REPO_ROOT/dist/grok" +DEV_DIR="${GROK_LOCAL_DEV_DIR:-/tmp/braintrust-grok-local-dev-${UID:-user}-$$}" +CURRENT_LINK="/tmp/braintrust-grok-local-dev-${UID:-user}-current" +DAEMON_BIN="$REPO_ROOT/bt-daemon/target/debug/bt-daemon" +SOCKET="$DEV_DIR/daemon.sock" +DATA_DIR="$DEV_DIR/state" +CONFIG="$DEV_DIR/braintrust.json" +DAEMON_LOG="$DEV_DIR/daemon.log" +BT_WRAPPER="$DEV_DIR/bt" +GROK_HOME_DIR="$DEV_DIR/grok-home" +PLUGIN_NAME="trace-grok" +DAEMON_PID="" +REPORT_TO_BRAINTRUST=false +SKIP_PLUGIN_RELOAD=false +BRAINTRUST_PROJECT="${BRAINTRUST_DEFAULT_PROJECT:-grok-local-dev}" +GROK_ARGS=() +GROK_ARG_COUNT=0 + +fail() { + printf 'grok local dev: %s\n' "$*" >&2 + exit 1 +} + +find_probe_journal() { + local journal_dir="$1" + local session_id="$2" + local candidate + for candidate in "$journal_dir"/grok--"$session_id"--*.ndjson; do + [[ -s "$candidate" ]] || continue + printf '%s\n' "$candidate" + return 0 + done + return 1 +} + +if [[ "${BASH_SOURCE[0]}" != "$0" ]]; then + return 0 +fi + +while [[ $# -gt 0 ]]; do + case "$1" in + --braintrust) + REPORT_TO_BRAINTRUST=true + shift + ;; + --project) + [[ $# -ge 2 ]] || fail "--project requires a project name" + BRAINTRUST_PROJECT="$2" + shift 2 + ;; + --no-reload) + SKIP_PLUGIN_RELOAD=true + shift + ;; + --help|-h) + cat <<'USAGE' +Usage: local-dev.sh [--braintrust] [--project NAME] [--no-reload] [-- GROK_ARGS...] + + --braintrust Send translated spans to Braintrust instead of local NDJSON + --project NAME Braintrust project name (default: $BRAINTRUST_DEFAULT_PROJECT + or grok-local-dev) + --no-reload Inject hooks directly for this run; do not send /reload-plugins +USAGE + exit 0 + ;; + --) + shift + GROK_ARGS+=("$@") + GROK_ARG_COUNT=$((GROK_ARG_COUNT + $#)) + break + ;; + *) + GROK_ARGS+=("$1") + GROK_ARG_COUNT=$((GROK_ARG_COUNT + 1)) + shift + ;; + esac +done + +if [[ "$REPORT_TO_BRAINTRUST" == true && -z "${BRAINTRUST_API_KEY:-}" ]]; then + fail "--braintrust requires BRAINTRUST_API_KEY in the environment" +fi + +for command in cargo grok python3 rsync; do + command -v "$command" >/dev/null 2>&1 || fail "$command is required" +done + +cleanup_prior_runs() { + local old_dir socket pid process_command + for old_dir in /tmp/braintrust-grok-local-dev-"${UID:-user}"-[0-9]*; do + [[ -e "$old_dir" ]] || continue + socket="$old_dir/daemon.sock" + + # Stop only processes that demonstrably own a prior run's socket. This also + # cleans up production bt daemons left by older versions of this script. + if [[ -S "$socket" ]] && command -v lsof >/dev/null 2>&1; then + while IFS= read -r pid; do + [[ "$pid" =~ ^[0-9]+$ ]] || continue + process_command="$(ps -p "$pid" -o command= 2>/dev/null || true)" + if [[ "$process_command" == *"$socket"* ]]; then + kill "$pid" 2>/dev/null || true + fi + done < <(lsof -t "$socket" 2>/dev/null || true) + elif [[ -f "$old_dir/daemon.pid" ]]; then + pid="$(<"$old_dir/daemon.pid")" + if [[ "$pid" =~ ^[0-9]+$ ]]; then + process_command="$(ps -p "$pid" -o command= 2>/dev/null || true)" + if [[ "$process_command" == *"$socket"* ]]; then + kill "$pid" 2>/dev/null || true + fi + fi + fi + rm -rf -- "$old_dir" + done + rm -f -- "$CURRENT_LINK" +} + +printf '==> Cleaning prior Grok local-development runs\n' +cleanup_prior_runs + +printf '==> Building and validating the Grok plugin\n' +"$SCRIPT_DIR/build.sh" "$DIST_DIR" +"$SCRIPT_DIR/validate.sh" "$DIST_DIR" + +printf '==> Building the standalone development daemon\n' +if cargo +1.92.0 --version >/dev/null 2>&1; then + cargo +1.92.0 build \ + --manifest-path "$REPO_ROOT/bt-daemon/Cargo.toml" \ + --features cli \ + --bin bt-daemon +else + cargo build \ + --manifest-path "$REPO_ROOT/bt-daemon/Cargo.toml" \ + --features cli \ + --bin bt-daemon +fi + +installed_source="$(grok plugin list --json | python3 -c ' +import json, sys +for plugin in json.load(sys.stdin): + if plugin.get("name") == "trace-grok": + print(plugin.get("source", "")) + break +')" + +if [[ -z "$installed_source" ]]; then + printf '==> Installing %s from %s\n' "$PLUGIN_NAME" "$DIST_DIR" + grok plugin install "$DIST_DIR" --trust +else + installed_real="$(python3 -c 'import os,sys; print(os.path.realpath(sys.argv[1]))' "$installed_source")" + dist_real="$(python3 -c 'import os,sys; print(os.path.realpath(sys.argv[1]))' "$DIST_DIR")" + if [[ "$installed_real" != "$dist_real" ]]; then + fail "$PLUGIN_NAME is already installed from $installed_source; uninstall it with 'grok plugin uninstall $PLUGIN_NAME --confirm' before using this script" + fi + printf '==> Refreshing the existing local plugin installation\n' + grok plugin update "$PLUGIN_NAME" +fi +grok plugin enable "$PLUGIN_NAME" >/dev/null +installed_path="$(grok plugin list --json | python3 -c ' +import json, sys +for plugin in json.load(sys.stdin): + if plugin.get("name") == "trace-grok": + print(plugin.get("path", "")) + break +')" +[[ -n "$installed_path" ]] || fail "could not locate the installed $PLUGIN_NAME plugin" + +rm -rf "$DEV_DIR" +mkdir -p "$DEV_DIR" +ln -sfn "$DEV_DIR" "$CURRENT_LINK" +python3 - "$CONFIG" "$BRAINTRUST_PROJECT" "$REPORT_TO_BRAINTRUST" <<'PY' +import json +import sys + +path, project, report_to_braintrust = sys.argv[1:] +with open(path, "w") as f: + json.dump( + { + "trace_to_braintrust": True, + "route": { + "auth": {"source": "environment"}, + "destination": { + "type": "project_logs", + "project_name": project, + }, + "flush_mode": ( + "flush_on_turn_end" + if report_to_braintrust == "true" + else "fire_and_forget" + ), + }, + }, + f, + indent=2, + ) + f.write("\n") +PY + +# Isolate Grok's own plugin registry so unrelated Claude-compatible plugins +# cannot start a production bt daemon on the development socket. +mkdir -p "$GROK_HOME_DIR" +cp "${GROK_HOME:-$HOME/.grok}/auth.json" "$GROK_HOME_DIR/auth.json" +ln -s "$installed_path" "$GROK_HOME_DIR/trace-grok" +if [[ "$SKIP_PLUGIN_RELOAD" == true ]]; then + cat >"$GROK_HOME_DIR/config.toml" <<'TOML' +[plugins] +disabled = ["trace-grok", "trace-claude-code"] + +[compat.claude] +hooks = false + +[compat.cursor] +hooks = false +TOML +else + cat >"$GROK_HOME_DIR/config.toml" <"$BT_WRAPPER" <&2 + exit 2 +fi +shift 2 +exec "$DAEMON_BIN" hook "\$@" --socket "$SOCKET" --no-spawn +SH +chmod +x "$BT_WRAPPER" + +installed_hooks="$installed_path/hooks/hooks.json" +dist_hooks="$DIST_DIR/hooks/hooks.json" +installed_forward="$installed_path/hooks/forward.sh" +dist_forward="$DIST_DIR/hooks/forward.sh" +original_hooks="$DEV_DIR/hooks.json.original" +original_dist_hooks="$DEV_DIR/dist-hooks.json.original" +original_forward="$DEV_DIR/forward.sh.original" +original_dist_forward="$DEV_DIR/dist-forward.sh.original" + +# Grok's local-source refresh can leave an interrupted development run's +# temporary env injection in the installed copy. Reset it from the freshly +# built artifact before taking backups or applying this run's injection. +if [[ "$installed_hooks" != "$dist_hooks" ]]; then + cp "$dist_hooks" "$installed_hooks" +fi +if [[ "$installed_forward" != "$dist_forward" ]]; then + cp "$dist_forward" "$installed_forward" +fi +cp "$installed_hooks" "$original_hooks" +cp "$dist_hooks" "$original_dist_hooks" +cp "$installed_forward" "$original_forward" +cp "$dist_forward" "$original_dist_forward" + +restore_hooks() { + if [[ -f "$original_hooks" && -n "$installed_hooks" ]]; then + cp "$original_hooks" "$installed_hooks" 2>/dev/null || true + fi + if [[ -f "$original_dist_hooks" ]]; then + cp "$original_dist_hooks" "$dist_hooks" 2>/dev/null || true + fi + if [[ -f "$original_forward" ]]; then + cp "$original_forward" "$installed_forward" 2>/dev/null || true + fi + if [[ -f "$original_dist_forward" ]]; then + cp "$original_dist_forward" "$dist_forward" 2>/dev/null || true + fi +} + +cleanup() { + if [[ -n "$DAEMON_PID" ]] && kill -0 "$DAEMON_PID" 2>/dev/null; then + kill "$DAEMON_PID" 2>/dev/null || true + wait "$DAEMON_PID" 2>/dev/null || true + fi + restore_hooks +} +trap cleanup EXIT INT TERM + +# Grok's hook runner intentionally does not pass arbitrary parent-process +# environment variables through. Add the isolated local route explicitly to +# this installed copy, then restore the original file when the session exits. +python3 - "$installed_hooks" "$dist_hooks" "$BT_WRAPPER" "$DAEMON_BIN" "$SOCKET" "$CONFIG" <<'PY' +import json +import sys + +installed_path, dist_path, bt_bin, daemon_bin, socket, config = sys.argv[1:] +local_env = { + "BT_BIN": bt_bin, + "BT_DAEMON_BIN": daemon_bin, + "BT_DAEMON_SOCKET": socket, + "BT_DAEMON_CONFIG": config, +} +for path in (installed_path, dist_path): + with open(path) as f: + document = json.load(f) + for groups in document["hooks"].values(): + for group in groups: + for handler in group["hooks"]: + handler["env"] = {**handler.get("env", {}), **local_env} + with open(path, "w") as f: + json.dump(document, f, indent=2) + f.write("\n") +PY + +# Pin the adapter itself to the local wrapper. This remains effective even when +# Grok strips parent-process variables, and patching both copies survives its +# local-source refresh during /reload-plugins. +python3 - "$installed_forward" "$dist_forward" "$BT_WRAPPER" "$DEV_DIR/grok-adapter.stderr" <<'PY' +import shlex +import sys + +for path in sys.argv[1:3]: + with open(path) as f: + lines = f.readlines() + lines.insert(1, f"exec 2>>{shlex.quote(sys.argv[4])}\n") + lines.insert(1, f"export BT_BIN={shlex.quote(sys.argv[3])}\n") + with open(path, "w") as f: + f.writelines(lines) +PY + +if [[ "$SKIP_PLUGIN_RELOAD" == true ]]; then + # Grok 1.0.13 does not activate installed plugin hooks at process startup. + # For a clean demo, install this artifact's hooks into the isolated run home, + # where Grok discovers them as trusted global hooks without a slash command. + direct_hooks_dir="$GROK_HOME_DIR/hooks" + direct_forward="$direct_hooks_dir/forward.sh" + mkdir -p "$direct_hooks_dir" + cp "$dist_forward" "$direct_forward" + chmod +x "$direct_forward" + python3 - "$dist_hooks" "$direct_hooks_dir/braintrust.json" "$direct_forward" <<'PY' +import json +import shlex +import sys + +source, destination, forward = sys.argv[1:] +with open(source) as f: + document = json.load(f) +for groups in document["hooks"].values(): + for group in groups: + for handler in group["hooks"]: + handler["command"] = f"bash {shlex.quote(forward)}" +with open(destination, "w") as f: + json.dump(document, f, indent=2) + f.write("\n") +PY +fi + +if [[ "$REPORT_TO_BRAINTRUST" == true ]]; then + printf '==> Starting the standalone daemon with Braintrust project %s\n' "$BRAINTRUST_PROJECT" + DAEMON_ARGS=(serve) +else + printf '==> Starting the standalone daemon with the local debug sink\n' + DAEMON_ARGS=(serve --debug-sink) +fi +DAEMON_ARGS+=( + --socket "$SOCKET" + --data-dir "$DATA_DIR" + --idle-timeout-secs 0 +) +RUST_LOG="${GROK_DAEMON_RUST_LOG:-bt_daemon::translate::grok=debug}" \ +BT_DAEMON_CONFIG="$CONFIG" \ + "$DAEMON_BIN" "${DAEMON_ARGS[@]}" >"$DAEMON_LOG" 2>&1 & +DAEMON_PID=$! +printf '%s\n' "$DAEMON_PID" >"$DEV_DIR/daemon.pid" + +for _ in {1..100}; do + [[ -S "$SOCKET" ]] && break + kill -0 "$DAEMON_PID" 2>/dev/null || { + cat "$DAEMON_LOG" >&2 + fail "daemon exited before creating its socket" + } + sleep 0.05 +done +[[ -S "$SOCKET" ]] || fail "timed out waiting for daemon socket" + +# Prove the plugin adapter and local daemon are connected before Grok starts. +PROBE_SESSION="grok-local-dev-probe" +PROBE_JOURNAL="" +printf '%s' "{\"hookEventName\":\"local_dev_probe\",\"sessionId\":\"$PROBE_SESSION\"}" | \ + BT_BIN="$BT_WRAPPER" \ + BT_DAEMON_BIN="$DAEMON_BIN" \ + BT_DAEMON_SOCKET="$SOCKET" \ + BT_DAEMON_CONFIG="$CONFIG" \ + "$DIST_DIR/hooks/forward.sh" +for _ in {1..100}; do + PROBE_JOURNAL="$(find_probe_journal "$DATA_DIR/journal" "$PROBE_SESSION" || true)" + [[ -n "$PROBE_JOURNAL" ]] && break + sleep 0.05 +done +if [[ -z "$PROBE_JOURNAL" || ! -s "$PROBE_JOURNAL" ]]; then + cat "$DAEMON_LOG" >&2 + fail "hook-to-daemon probe was not journaled" +fi +if ! python3 - "$PROBE_JOURNAL" "$PROBE_SESSION" <<'PY' +import json +import sys + +path, session_id = sys.argv[1:] +with open(path) as f: + records = [json.loads(line) for line in f if line.strip()] +assert any( + record.get("source") == "grok" + and record.get("session_id") == session_id + and record.get("event") == "local_dev_probe" + for record in records +) +PY +then + cat "$DAEMON_LOG" >&2 + fail "hook-to-daemon probe journal was invalid" +fi +printf '==> Hook-to-daemon probe captured successfully\n' + +if [[ "$SKIP_PLUGIN_RELOAD" == true ]]; then + printf '\nGrok will start now with tracing hooks injected for this run.\n' +else + printf '\nGrok will start now and automatically run /reload-plugins. After the reload\n' + printf 'finishes, send prompts or invoke tools.\n' +fi +printf 'Exit Grok to print the daemon log.\n' +printf 'For a second live view, run in another terminal:\n' +printf ' tail -f %q/daemon.log\n\n' "$CURRENT_LINK" + +GROK_COMMAND=(grok --leader-socket "$DEV_DIR/grok-leader.sock") +if ((GROK_ARG_COUNT > 0)); then + GROK_COMMAND+=("${GROK_ARGS[@]}") +fi +if [[ "$SKIP_PLUGIN_RELOAD" == false ]]; then + GROK_COMMAND+=("/reload-plugins") +fi + +set +e +BT_BIN="$BT_WRAPPER" \ +BT_DAEMON_BIN="$DAEMON_BIN" \ +BT_DAEMON_SOCKET="$SOCKET" \ +BT_DAEMON_CONFIG="$CONFIG" \ +GROK_HOME="$GROK_HOME_DIR" \ +GROK_CLAUDE_HOOKS_ENABLED=false \ +GROK_CURSOR_HOOKS_ENABLED=false \ +GROK_HOOK_DEBUG=1 \ +GROK_HOOKS_LOG="$DEV_DIR/grok-hooks.log" \ + "${GROK_COMMAND[@]}" +grok_status=$? +set -e + +cleanup +DAEMON_PID="" +trap - EXIT INT TERM + +printf '\n==> Daemon log (%s)\n' "$DAEMON_LOG" +cat "$DAEMON_LOG" +printf '\nRaw journals are under: %s/journal\n' "$DATA_DIR" +if [[ "$REPORT_TO_BRAINTRUST" == true ]]; then + printf 'Translated spans were sent to Braintrust project: %s\n' "$BRAINTRUST_PROJECT" + printf 'Inspect them with: bt view logs --project %q\n' "$BRAINTRUST_PROJECT" +else + printf 'Translated spans are under: %s/spans\n' "$DATA_DIR" +fi + +exit "$grok_status" diff --git a/src/plugins/grok/publish.sh b/src/plugins/grok/publish.sh new file mode 100755 index 0000000..3202a71 --- /dev/null +++ b/src/plugins/grok/publish.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Build and deploy the Grok plugin to a generated distribution repository. +set -euo pipefail + +: "${DIST_REPO:?set DIST_REPO= (usually via PUBLISH_TARGETS)}" +SRC_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SLUG="$(printf '%s' "$DIST_REPO" | sed -E 's#^git@[^:]+:##; s#^https?://[^/]+/##; s#\.git$##')" + +case "$DIST_REPO" in + *://*|*@*) CLONE_URL="$DIST_REPO" ;; + *) + if [[ -n "${GH_TOKEN:-}" ]]; then + CLONE_URL="https://x-access-token:${GH_TOKEN}@github.com/${DIST_REPO}.git" + else + CLONE_URL="https://github.com/${DIST_REPO}.git" + fi + ;; +esac + +WORKTREE="$(mktemp -d)/dist" +cleanup() { rm -rf "$(dirname "$WORKTREE")"; } +trap cleanup EXIT + +git clone --depth 1 "$CLONE_URL" "$WORKTREE" 2>/dev/null \ + || git clone "$CLONE_URL" "$WORKTREE" +git -C "$WORKTREE" config user.name "github-actions[bot]" +git -C "$WORKTREE" config user.email "41898282+github-actions[bot]@users.noreply.github.com" +git -C "$WORKTREE" rm -rfq --ignore-unmatch . >/dev/null 2>&1 || true +"$SRC_DIR/build.sh" "$WORKTREE" +"$SRC_DIR/validate.sh" "$WORKTREE" + +git -C "$WORKTREE" add -A +if git -C "$WORKTREE" diff --cached --quiet; then + echo "==> $SLUG already up to date; nothing to publish." + exit 0 +fi + +SRC_SHA="$(git -C "$SRC_DIR" rev-parse --short HEAD 2>/dev/null || echo unknown)" +git -C "$WORKTREE" commit -q -m "build: deploy grok plugin from monorepo@${SRC_SHA}" +if [[ "${DRY_RUN:-}" == "1" ]]; then + echo "==> DRY_RUN=1: built + committed locally, skipping push." + git -C "$WORKTREE" --no-pager show --stat HEAD | head -30 + exit 0 +fi + +git -C "$WORKTREE" push origin HEAD:main +echo "==> Deployed grok plugin to $SLUG." diff --git a/src/plugins/grok/test/test_hook.sh b/src/plugins/grok/test/test_hook.sh new file mode 100755 index 0000000..b60a6ec --- /dev/null +++ b/src/plugins/grok/test/test_hook.sh @@ -0,0 +1,145 @@ +#!/usr/bin/env bash +set -euo pipefail + +HOOK="${1:?usage: test_hook.sh }" +HOOK_DIR="${HOOK%/*}" +[[ "$HOOK_DIR" != "$HOOK" ]] || HOOK_DIR="." +HOOKS_JSON="$HOOK_DIR/hooks.json" +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT + +cat >"$TMP/bt" <<'STUB' +#!/bin/sh +{ + printf '%s\n' "$#" + printf '%s\n' "$@" +} >"$BT_CAPTURE_ARGS" +cat >"$BT_CAPTURE_STDIN" +exit "${BT_STUB_STATUS:-0}" +STUB +chmod +x "$TMP/bt" + +payload='{"hookEventName":"pre_tool_use","sessionId":"test","transcriptPath":"/tmp/session.jsonl","toolName":"read_file"}' +payload_file="$TMP/payload" +args="$TMP/args" +stdin="$TMP/stdin" +expected_args="$TMP/expected-args" +printf '%s\n' "$payload" >"$payload_file" + +cat "$payload_file" | env \ + BT_BIN="$TMP/bt" \ + BT_CAPTURE_ARGS="$args" \ + BT_CAPTURE_STDIN="$stdin" \ + GROK_VERSION= \ + BT_STUB_STATUS=0 \ + "$HOOK" + +cat >"$expected_args" <<'ARGS' +12 +trace +hook +--source +grok +--plugin-version +0.1.0 +--session-id-field +sessionId +--event-field +hookEventName +--transcript-path-field +transcriptPath +ARGS +cmp -s "$expected_args" "$args" \ + || { echo "test: unexpected bt arguments" >&2; exit 1; } +cmp -s "$payload_file" "$stdin" \ + || { echo "test: hook payload changed" >&2; exit 1; } + +# Forward an already-available Grok version without launching Grok to discover it. +cat "$payload_file" | env \ + BT_BIN="$TMP/bt" \ + BT_CAPTURE_ARGS="$args" \ + BT_CAPTURE_STDIN="$stdin" \ + BT_STUB_STATUS=0 \ + GROK_VERSION="1.0.13 beta" \ + "$HOOK" +cat >"$expected_args" <<'ARGS' +14 +trace +hook +--source +grok +--plugin-version +0.1.0 +--session-id-field +sessionId +--event-field +hookEventName +--transcript-path-field +transcriptPath +--source-version +1.0.13 beta +ARGS +cmp -s "$expected_args" "$args" \ + || { echo "test: unexpected source-version arguments" >&2; exit 1; } +cmp -s "$payload_file" "$stdin" \ + || { echo "test: source-version forwarding changed the hook payload" >&2; exit 1; } + +# Forwarding failures must not interrupt Grok. +cat "$payload_file" | env \ + BT_BIN="$TMP/bt" \ + BT_CAPTURE_ARGS="$args" \ + BT_CAPTURE_STDIN="$stdin" \ + BT_STUB_STATUS=23 \ + GROK_VERSION= \ + "$HOOK" +cmp -s "$payload_file" "$stdin" \ + || { echo "test: forwarding failure changed the hook payload" >&2; exit 1; } + +# A missing host CLI must diagnose and return without attempting installation. +mkdir "$TMP/no-bt" "$TMP/host-home" +cat >"$TMP/no-bt/curl" <<'STUB' +#!/bin/sh +: >"$CURL_CALLED" +exit 0 +STUB +chmod +x "$TMP/no-bt/curl" +diagnostic="$TMP/diagnostic" +/usr/bin/env -u BT_BIN -u GROK_VERSION \ + PATH="$TMP/no-bt" \ + HOME="$TMP/host-home" \ + XDG_BIN_HOME="$TMP/host-home/bin" \ + CARGO_HOME="$TMP/host-home/cargo" \ + CURL_CALLED="$TMP/curl-called" \ + /bin/bash "$HOOK" <"$payload_file" 2>"$diagnostic" +[[ "$(<"$diagnostic")" == "trace-grok: bt CLI not found; tracing skipped" ]] \ + || { echo "test: unexpected missing-CLI diagnostic" >&2; exit 1; } +[[ ! -e "$TMP/curl-called" ]] \ + || { echo "test: missing bt attempted an installation" >&2; exit 1; } +shopt -s nullglob dotglob +host_files=("$TMP/host-home"/*) +shopt -u nullglob dotglob +(( ${#host_files[@]} == 0 )) \ + || { echo "test: missing bt mutated the host home" >&2; exit 1; } + +python3 - "$HOOKS_JSON" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as handle: + hooks = json.load(handle)["hooks"] + +assert "SessionEnd" in hooks, "terminal hook must use exact SessionEnd spelling" +assert "SessionStop" not in hooks, "non-native terminal spelling must not be registered" +for event, groups in hooks.items(): + expected_timeout = 15 if event == "SessionEnd" else 5 + timeouts = { + hook["timeout"] + for group in groups + for hook in group["hooks"] + } + assert timeouts == {expected_timeout}, ( + f"{event} timeout {sorted(timeouts)} != {expected_timeout}" + ) +PY + +echo "test: grok hook adapter OK" diff --git a/src/plugins/grok/test/test_local_dev.sh b/src/plugins/grok/test/test_local_dev.sh new file mode 100755 index 0000000..e35e00d --- /dev/null +++ b/src/plugins/grok/test/test_local_dev.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=../local-dev.sh +source "$SCRIPT_DIR/../local-dev.sh" + +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "$TMP_DIR"' EXIT +JOURNAL_DIR="$TMP_DIR/journal" +SESSION_ID="grok-local-dev-probe" +mkdir -p "$JOURNAL_DIR" + +legacy="$JOURNAL_DIR/$SESSION_ID.ndjson" +printf '%s\n' '{"event":"local_dev_probe"}' >"$legacy" +if find_probe_journal "$JOURNAL_DIR" "$SESSION_ID" >/dev/null; then + printf 'test: legacy journal path must not satisfy the source-qualified probe\n' >&2 + exit 1 +fi + +qualified="$JOURNAL_DIR/grok--$SESSION_ID--stable-id.ndjson" +printf '%s\n' '{"source":"grok","event":"local_dev_probe"}' >"$qualified" +observed="$(find_probe_journal "$JOURNAL_DIR" "$SESSION_ID")" +[[ "$observed" == "$qualified" ]] || { + printf 'test: expected %s, got %s\n' "$qualified" "$observed" >&2 + exit 1 +} + +printf 'test: grok local-dev journal lookup OK\n' diff --git a/src/plugins/grok/validate.sh b/src/plugins/grok/validate.sh new file mode 100755 index 0000000..90ba690 --- /dev/null +++ b/src/plugins/grok/validate.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +set -euo pipefail + +TARGET_DIR="${1:?usage: validate.sh }" +SRC_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +fail() { echo "validate: $*" >&2; exit 1; } + +[[ -x "$SRC_DIR/local-dev.sh" ]] || fail "local-dev.sh is not executable" +bash -n "$SRC_DIR/local-dev.sh" || fail "local-dev.sh has invalid shell syntax" + +for file in .grok-plugin/plugin.json hooks/hooks.json hooks/forward.sh README.md LICENSE; do + [[ -f "$TARGET_DIR/$file" ]] || fail "missing $file" +done + +python3 - "$TARGET_DIR/LICENSE" <<'PY' || fail "invalid or placeholder license" +import pathlib +import sys + +text = pathlib.Path(sys.argv[1]).read_text() +assert "Apache License" in text +assert "Version 2.0, January 2004" in text +assert "END OF TERMS AND CONDITIONS" in text +assert "placeholder" not in text.lower() +assert "todo" not in text.lower() +PY +[[ -x "$TARGET_DIR/hooks/forward.sh" ]] || fail "hook adapter is not executable" + +python3 - "$TARGET_DIR/hooks/hooks.json" <<'PY' || fail "invalid Grok hooks" +import json +import sys + +with open(sys.argv[1]) as f: + hooks = json.load(f)["hooks"] + +expected = { + "SessionStart", "UserPromptSubmit", "PreToolUse", "PostToolUse", + "PostToolUseFailure", "PermissionDenied", "Stop", "StopFailure", + "StopCancelled", "Notification", "SubagentStart", "SubagentStop", + "PreCompact", "PostCompact", "SessionEnd", +} +assert set(hooks) == expected +for event, groups in hooks.items(): + expected_timeout = 15 if event == "SessionEnd" else 5 + for group in groups: + for hook in group["hooks"]: + assert hook == { + "type": "command", + "command": 'bash "${GROK_PLUGIN_ROOT}/hooks/forward.sh"', + "timeout": expected_timeout, + } +PY + +if command -v grok >/dev/null 2>&1; then + grok plugin validate "$TARGET_DIR" >/dev/null || fail "Grok rejected plugin manifest" +else + python3 -m json.tool "$TARGET_DIR/.grok-plugin/plugin.json" >/dev/null \ + || fail "invalid plugin manifest" +fi + +TEST_LOG="$(mktemp -d)/grok-hook-data" +trap 'rm -rf "$(dirname "$TEST_LOG")"' EXIT +"$SRC_DIR/test/test_hook.sh" "$TARGET_DIR/hooks/forward.sh" "$TEST_LOG" +"$SRC_DIR/test/test_local_dev.sh" + +echo "validate: grok dist OK ($TARGET_DIR)"