feat(*): compress trace payloads, cold logs and message packs

CONTEXT.md

@@ -2548,13 +2548,30 @@ sha1 of `json.dumps(message, ensure_ascii=False, default=str)` — key order
 preserved, since sorting would make resolution hand back a reordered copy. Beside
 `_blobs/` and never inside it: `raven.tracing.compact` sweeps a blob whose link count
 is 1, and a Message Blob's is permanently 1 because a shell references it from its
-JSON text rather than by hard link. `compact` names both stores in
-`_CONTENT_STORES` and walks neither as an artifact tree -- their layout is
+JSON text rather than by hard link. `compact` names the three content stores
+(`_blobs/`, `_messages/`, `_message_packs/`) in `_CONTENT_STORES` and walks none
+of them as an artifact tree -- their layout is
 indistinguishable from `<kind>/<day>/<file>`, so a store left in the walk is rehashed
-whole on every run and hard-linked into `_blobs/`, breaking this boundary. No reclaim path yet; growth is bounded only by use.
+whole on every run and hard-linked into `_blobs/`, breaking this boundary. Its
+reclaim path is Message Pack: `raven tracing pack-messages` moves the files into
+packs; until that runs, growth is bounded only by use.
 _Avoid_: storing one under `_blobs/` — that directory holds whole-artifact
 payloads, and `compact` sweeps any member of it whose link count is 1.
 
+**Message Pack** (`raven/tracing/message_pack.py`):
+Message Blobs concatenated into multi-megabyte zstd frames under
+`_message_packs/<uuid>.pack.zst`, with `index.json` mapping each sha1 to
+`(pack, plaintext offset, length)`. A migration, not a writer change:
+`raven tracing pack-messages` writes the packs, replaces the index atomically, and
+only then deletes the message files, so a never-packed store, a half-packed one,
+and a crash at any step all leave every address readable (a reader that finds no
+file reads its slice). The index travels with an imported store and is untrusted:
+a pack name is a generated basename and the range fields are non-negative
+integers, validated before any path is built. A reader keeps at most 32 MB of
+decompressed packs; a larger pack is streamed per read.
+_Avoid_: "archive" — that names the rotated-log directory; a pack is not a
+whole-store backup.
+
 **Artifact Shell** (`raven/observability/semconv.py`):
 An `llm.input` artifact in `audit.artifact.v2` form: the call's identity plus one
 `{"$msg": "<sha1>"}` reference per message, with `systemPrompt` and `prompt` aliasing

docs-site/docs/commands.md

@@ -36,6 +36,8 @@ lists its subcommands and options.
 | `raven sandbox` | List sandbox VMs, run commands, or open a shell; requires `tools.sandbox.debug.enabled=true` |
 | `raven tracing` | Open the local trace dashboard |
 | `raven tracing compact` | Consolidate duplicate trace artifacts to free disk space |
+| `raven tracing compress [--logs]` | Rewrite trace payloads as zstd (with `--logs`, the rotated span and event logs too) to free disk space |
+| `raven tracing pack-messages` | Pack content-addressed message blobs into a few Message Packs to free disk space |
 | `raven trajectory` | Save, replay, redact, label, and preserve execution trajectories for debugging, and run the bug report receiver |
 
 Run `raven --help` for the command list, or `raven <command> --help` for details

docs-site/docs/commands.zh.md

@@ -35,6 +35,8 @@
 | `raven sandbox` | 列出沙箱虚拟机、执行命令或打开 Shell；需要设置 `tools.sandbox.debug.enabled=true` |
 | `raven tracing` | 打开本地追踪面板 |
 | `raven tracing compact` | 合并重复的追踪产物，回收磁盘空间 |
+| `raven tracing compress [--logs]` | 以 zstd 重写追踪载荷（加 `--logs` 时还包括轮转后的日志），回收磁盘空间 |
+| `raven tracing pack-messages` | 将内容寻址的消息载荷打包成少量消息包（Message Pack），回收磁盘空间 |
 | `raven trajectory` | 保存、回放、脱敏、标注和保留执行轨迹，用于调试；也运行问题报告接收器 |
 
 运行 `raven --help` 查看命令列表，或运行 `raven <command> --help` 查看指定命令的详细说明。

pyproject.toml

@@ -122,6 +122,7 @@ dependencies = [
     # `AsyncOpenAI`, the only client LiteLLM's OpenAI-SDK route adopts. It
     # already arrives with litellm, and the range is litellm's own.
     "openai>=2.20.0,<3.0.0",
+    "zstandard>=0.25.0",
 ]
 
 [project.optional-dependencies]

raven/cli/tracing_commands.py

@@ -42,6 +42,18 @@
 
 console = Console()
 
+#: zstd decoding landed in Node's zlib in 22.15 and 23.8, and the viewer reads
+#: payloads compressed by default. Its floor is therefore higher than the TUI's.
+_MIN_VIEWER_NODE_VERSION = (22, 15, 0)
+_FIRST_VIEWER_NODE_23_VERSION = (23, 8, 0)
+
+
+def _node_supports_zstd(version: tuple[int, ...]) -> bool:
+    """Whether this Node has zlib zstd: 22.15+, 23.8+, or any later major."""
+    if version[0] == 23:
+        return version >= _FIRST_VIEWER_NODE_23_VERSION
+    return version >= _MIN_VIEWER_NODE_VERSION
+
 
 def _viewer_dir() -> Path:
     return Path(__file__).resolve().parent / "tracing_viewer"
@@ -190,15 +202,17 @@ def _stop_viewer() -> None:
 
 
 def _resolve_node() -> str:
-    from raven.cli.tui_commands import _MIN_NODE_VERSION, find_node
-
-    node, version = find_node()
-    # find_node returns the best node it saw even below the minimum, so the
-    # version check is the caller's -- the TUI's own launch does the same.
-    if not node or version is None or version < _MIN_NODE_VERSION:
+    from raven.cli.tui_commands import find_node
+
+    # The viewer's floor is stricter than the TUI's, so the search has to use
+    # it: a PATH node the TUI accepts would otherwise hide a zstd-capable one
+    # later in the order. find_node still returns the best node it saw when
+    # none qualifies, so the version check stays the caller's.
+    node, version = find_node(_node_supports_zstd)
+    if not node or version is None or not _node_supports_zstd(version):
         console.print(
-            "[red]Node (>= 22) not found.[/red] The tracing dashboard needs the "
-            "same Node runtime as the TUI.\n"
+            "[red]Node with zstd support not found (22.15+, or 23.8+).[/red] The tracing dashboard reads "
+            "zstd-compressed payloads and needs a Node with zlib zstd support.\n"
             "Install: https://nodejs.org/  or  brew install node@22  or  nvm install 22"
         )
         raise typer.Exit(1)
@@ -306,6 +320,70 @@ def _run_compact(*, dry_run: bool) -> None:
         console.print(f"[yellow]... and {hidden} more[/yellow]")
 
 
+def _run_compress(*, dry_run: bool, logs: bool = False) -> None:
+    from raven.tracing.compress import compress, compress_logs
+    from raven.tracing.store import TraceStore
+
+    store = TraceStore(tracing_config.state_dir())
+    if dry_run:
+        console.print("[dim]dry run - nothing is written[/dim]")
+    artifacts_dir = store.artifacts_dir
+    if artifacts_dir.is_dir():
+        result = compress(artifacts_dir, dry_run=dry_run)
+        console.print(
+            f"blobs {result.blobs}: compressed {result.compressed}, already {result.already}, relinked {result.relinked}"
+        )
+        console.print(f"messages {result.messages}: compressed {result.messages_compressed}")
+        console.print(
+            f"{result.bytes_before / (1024 * 1024):.1f} MB -> {result.bytes_after / (1024 * 1024):.1f} MB, "
+            f"saved {result.saved / (1024 * 1024):.1f} MB"
+        )
+        _print_skipped(result.errors)
+    else:
+        console.print(f"[yellow]No artifacts at {escape(str(artifacts_dir))}.[/yellow]")
+    if logs:
+        log_result = compress_logs(store.logs_dir, dry_run=dry_run)
+        console.print(
+            f"logs {log_result.files}: compressed {log_result.compressed}, already {log_result.already}, "
+            f"left for the next rotation {log_result.newest}"
+        )
+        console.print(
+            f"{log_result.bytes_before / (1024 * 1024):.1f} MB -> {log_result.bytes_after / (1024 * 1024):.1f} MB, "
+            f"saved {log_result.saved / (1024 * 1024):.1f} MB"
+        )
+        _print_skipped(log_result.errors)
+
+
+def _run_pack_messages(*, dry_run: bool) -> None:
+    from raven.tracing.message_pack import pack_messages
+    from raven.tracing.store import TraceStore
+
+    artifacts_dir = TraceStore(tracing_config.state_dir()).artifacts_dir
+    if dry_run:
+        console.print("[dim]dry run - nothing is written[/dim]")
+    if not artifacts_dir.is_dir():
+        console.print(f"[yellow]No artifacts at {escape(str(artifacts_dir))}.[/yellow]")
+        return
+    result = pack_messages(artifacts_dir, dry_run=dry_run)
+    console.print(
+        f"messages {result.scanned}: packed {result.packed}, already {result.already}, "
+        f"packs {result.packs}, removed files {result.removed}"
+    )
+    console.print(
+        f"{result.bytes_before / (1024 * 1024):.1f} MB -> {result.bytes_after / (1024 * 1024):.1f} MB, "
+        f"saved {result.saved / (1024 * 1024):.1f} MB"
+    )
+    _print_skipped(result.errors)
+
+
+def _print_skipped(errors: list[str]) -> None:
+    for message in errors[:10]:
+        console.print(f"[yellow]skipped:[/yellow] {escape(message)}")
+    hidden = max(len(errors) - 10, 0)
+    if hidden:
+        console.print(f"[yellow]... and {hidden} more[/yellow]")
+
+
 def register(app: typer.Typer) -> None:
     """Attach the ``tracing`` command to ``app``.
 
@@ -320,26 +398,40 @@ def tracing(
             None,
             help=(
                 "Optional action: 'stop' shuts down the background viewer; "
-                "'compact' folds duplicate artifacts (terminal only, not available from the TUI)."
+                "'compact' folds duplicate artifacts; 'compress' rewrites payloads as zstd "
+                "(--logs adds the rotated span and event logs); 'pack-messages' moves message blobs "
+                "into packs. All but 'stop' are terminal only, not available from the TUI."
             ),
         ),
         port: int = typer.Option(None, "--port", "-p", help="Port to bind (default: config or 4318)."),
         foreground: bool = typer.Option(
             False, "--foreground", "-f", help="Run the viewer in the foreground (blocks; Ctrl-C to stop)."
         ),
         dry_run: bool = typer.Option(
-            False, "--dry-run", help="With 'compact': report what would change without writing."
+            False,
+            "--dry-run",
+            help="With 'compact', 'compress' or 'pack-messages': report what would change without writing.",
         ),
+        logs: bool = typer.Option(False, "--logs", help="With 'compress': also compress rotated span and event logs."),
     ) -> None:
         """Open the tracing dashboard (captured LLM/tool/memory spans)."""
         if action == "compact":
             _run_compact(dry_run=dry_run)
             return
+        if action == "compress":
+            _run_compress(dry_run=dry_run, logs=logs)
+            return
+        if action == "pack-messages":
+            _run_pack_messages(dry_run=dry_run)
+            return
         if action == "stop":
             _stop_viewer()
             return
         if action is not None:
-            console.print(f"[red]Unknown action '{escape(action)}'.[/red] Supported actions: stop, compact")
+            console.print(
+                f"[red]Unknown action '{escape(action)}'.[/red] "
+                "Supported actions: stop, compact, compress, pack-messages"
+            )
             raise typer.Exit(2)
         bind_port = port if port is not None else tracing_config.port()
         if foreground:

raven/cli/tracing_viewer/log-store.js

@@ -1,8 +1,40 @@
 const fs = require('fs');
 const os = require('os');
 const path = require('path');
+const zlib = require('zlib');
 
 const DEFAULT_MAX_BYTES = 50 * 1024 * 1024;
+const MAX_STORED_BYTES = 256 * 1024 * 1024;
+const ZSTD_MAGIC = Buffer.from([0x28, 0xb5, 0x2f, 0xfd]);
+
+function readStored(filePath) {
+  const data = fs.readFileSync(filePath);
+  if (data.length >= 4 && data.subarray(0, 4).equals(ZSTD_MAGIC)) {
+    // zstd landed in Node's zlib in 22.15 / 23.8. An older viewer can still
+    // read a plain store; a compressed one is reported rather than parsed as
+    // mojibake, because JSON.parse of the frame would only say "invalid".
+    if (typeof zlib.zstdDecompressSync !== 'function') {
+      console.warn('tracing viewer: this Node has no zstd support; compressed files need Node >= 22.15');
+      return Buffer.alloc(0);
+    }
+    try {
+      return zlib.zstdDecompressSync(data, { maxOutputLength: MAX_STORED_BYTES });
+    } catch (err) {
+      if (err && err.code === 'ERR_BUFFER_TOO_LARGE') {
+        // A compressed file can be imported with a store; a frame that expands
+        // past the read limit is treated as unreadable rather than allocated.
+        console.warn('tracing viewer: a compressed file expands past the read limit; treating it as empty');
+        return Buffer.alloc(0);
+      }
+      return data;
+    }
+  }
+  return data;
+}
+
+function readStoredText(filePath) {
+  return readStored(filePath).toString('utf8');
+}
 
 const KIND_FILES = {
   events: 'audit-events.log',
@@ -65,8 +97,7 @@ function statOrNull(filePath) {
 
 function readJsonlFile(filePath) {
   if (!fs.existsSync(filePath)) return [];
-  return fs
-    .readFileSync(filePath, 'utf8')
+  return readStoredText(filePath)
     .split('\n')
     .map((line) => line.trim())
     .filter(Boolean)
@@ -199,6 +230,8 @@ module.exports = {
   ensureDir,
   readJsonl,
   readJsonlFile,
+  readStored,
+  readStoredText,
   statLogFiles,
   appendJsonl,
   rotateIfNeeded,