From 394ed2f0fde1e58f63f0c4f7de1256339715a93d Mon Sep 17 00:00:00 2001 From: Dizhan Xue Date: Thu, 29 Oct 2026 14:30:32 +0000 Subject: [PATCH 01/23] feat(tracing): store trace payloads compressed in place Artifacts and content-addressed messages are written as zstd frames under the paths they already had. A reader tells the encoding from the frame, so a store written before compression stays readable and a half-migrated one stays coherent. `raven tracing compress` migrates a store in place, re-linking each artifact onto the compressed blob; `--dry-run` reports the real saving without writing. --- pyproject.toml | 1 + raven/cli/tracing_commands.py | 36 +++- raven/cli/tracing_viewer/server.js | 34 ++- raven/node/install.py | 4 +- raven/node/protocol.py | 7 +- raven/rpc/methods/cli_dispatch.py | 27 +-- raven/tracing/artifact_v2.py | 17 +- raven/tracing/codec.py | 79 +++++++ raven/tracing/compact.py | 23 +- raven/tracing/compress.py | 162 ++++++++++++++ raven/tracing/store.py | 51 +++-- raven/trajectory/bundle.py | 26 ++- raven/trajectory/conversation.py | 13 +- raven/trajectory/details.py | 6 +- raven/trajectory/selection.py | 6 +- tests/integration/test_tracing_viewer_e2e.py | 19 ++ tests/test_cli_tracing_commands.py | 40 ++++ tests/test_rpc_cli_dispatch.py | 30 ++- tests/test_subagent_acp.py | 14 +- tests/test_tracing_api.py | 45 ++-- tests/test_tracing_store_compression.py | 215 +++++++++++++++++++ tests/test_trajectory_conversation.py | 8 +- tests/test_utils_mirrors.py | 6 +- uv.lock | 59 +++++ 24 files changed, 837 insertions(+), 91 deletions(-) create mode 100644 raven/tracing/codec.py create mode 100644 raven/tracing/compress.py create mode 100644 tests/test_tracing_store_compression.py diff --git a/pyproject.toml b/pyproject.toml index 2636c4bff..03aa46b9e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -122,6 +122,7 @@ dependencies = [ # `AsyncOpenAI`, the only client LiteLLM's OpenAI-SDK route adopts. It # already arrives with litellm, and the range is litellm's own. "openai>=2.20.0,<3.0.0", + "zstandard>=0.25.0", ] [project.optional-dependencies] diff --git a/raven/cli/tracing_commands.py b/raven/cli/tracing_commands.py index 1656eb01b..e276b6fad 100644 --- a/raven/cli/tracing_commands.py +++ b/raven/cli/tracing_commands.py @@ -306,6 +306,32 @@ def _run_compact(*, dry_run: bool) -> None: console.print(f"[yellow]... and {hidden} more[/yellow]") +def _run_compress(*, dry_run: bool) -> None: + from raven.tracing.compress import compress + from raven.tracing.store import TraceStore + + artifacts_dir = TraceStore(tracing_config.state_dir()).artifacts_dir + if not artifacts_dir.is_dir(): + console.print(f"[yellow]No artifacts at {escape(str(artifacts_dir))}.[/yellow]") + return + result = compress(artifacts_dir, dry_run=dry_run) + if dry_run: + console.print("[dim]dry run - nothing is written[/dim]") + console.print( + f"blobs {result.blobs}: compressed {result.compressed}, already {result.already}, relinked {result.relinked}" + ) + console.print(f"messages {result.messages}: compressed {result.messages_compressed}") + console.print( + f"{result.bytes_before / (1024 * 1024):.1f} MB -> {result.bytes_after / (1024 * 1024):.1f} MB, " + f"saved {result.saved / (1024 * 1024):.1f} MB" + ) + for message in result.errors[:10]: + console.print(f"[yellow]skipped:[/yellow] {escape(message)}") + hidden = max(len(result.errors) - 10, 0) + if hidden: + console.print(f"[yellow]... and {hidden} more[/yellow]") + + def register(app: typer.Typer) -> None: """Attach the ``tracing`` command to ``app``. @@ -320,7 +346,8 @@ def tracing( None, help=( "Optional action: 'stop' shuts down the background viewer; " - "'compact' folds duplicate artifacts (terminal only, not available from the TUI)." + "'compact' folds duplicate artifacts; 'compress' rewrites payloads as zstd " + "(both terminal only, not available from the TUI)." ), ), port: int = typer.Option(None, "--port", "-p", help="Port to bind (default: config or 4318)."), @@ -328,18 +355,21 @@ def tracing( False, "--foreground", "-f", help="Run the viewer in the foreground (blocks; Ctrl-C to stop)." ), dry_run: bool = typer.Option( - False, "--dry-run", help="With 'compact': report what would change without writing." + False, "--dry-run", help="With 'compact' or 'compress': report what would change without writing." ), ) -> None: """Open the tracing dashboard (captured LLM/tool/memory spans).""" if action == "compact": _run_compact(dry_run=dry_run) return + if action == "compress": + _run_compress(dry_run=dry_run) + return if action == "stop": _stop_viewer() return if action is not None: - console.print(f"[red]Unknown action '{escape(action)}'.[/red] Supported actions: stop, compact") + console.print(f"[red]Unknown action '{escape(action)}'.[/red] Supported actions: stop, compact, compress") raise typer.Exit(2) bind_port = port if port is not None else tracing_config.port() if foreground: diff --git a/raven/cli/tracing_viewer/server.js b/raven/cli/tracing_viewer/server.js index bd01a385b..a487d111c 100644 --- a/raven/cli/tracing_viewer/server.js +++ b/raven/cli/tracing_viewer/server.js @@ -4,6 +4,7 @@ const http = require('http'); const fs = require('fs'); const path = require('path'); const os = require('os'); +const zlib = require('zlib'); const { URL } = require('url'); const { applyStateDirArg } = require('./state-dir'); applyStateDirArg(); @@ -18,6 +19,33 @@ const STATE_DIR = path.join(os.homedir(), '.openclaw'); const LOGS_DIR = getLogsDir(); const ARTIFACTS_DIR = path.join(LOGS_DIR, 'audit-artifacts'); +const ZSTD_MAGIC = Buffer.from([0x28, 0xb5, 0x2f, 0xfd]); + +// Artifacts and message blobs are stored zstd-compressed under the same paths +// they always had; the frame, never the name, says which encoding a file +// carries. A plain file written before compression reads unchanged. +function readStored(file) { + const data = fs.readFileSync(file); + if (data.length >= 4 && data.subarray(0, 4).equals(ZSTD_MAGIC)) { + // zstd landed in Node's zlib in 22.15 / 23.8. An older viewer can still + // read a plain store; a compressed one is reported rather than shown as + // mojibake, because JSON.parse of the frame would only say "invalid". + if (typeof zlib.zstdDecompressSync !== 'function') { + console.warn('tracing viewer: this Node has no zstd support; compressed files need Node >= 22.15'); + return Buffer.alloc(0); + } + try { + return zlib.zstdDecompressSync(data); + } catch { + return data; + } + } + return data; +} + +function readStoredText(file) { + return readStored(file).toString('utf8'); +} const STATIC_DIR = path.join(__dirname, 'ui'); const SHELL_HTML = require('./ui/shell.js'); const BUNDLED_DESCRIPTORS_DIR = path.join(__dirname, 'descriptors'); @@ -1094,7 +1122,7 @@ function resolvePayload(payload) { const file = path.join(MESSAGES_DIR, sha1.slice(0, 2), `${sha1}.json`); let value; try { - value = JSON.parse(fs.readFileSync(file, 'utf8')); + value = JSON.parse(readStoredText(file)); } catch { value = { role: 'unknown', content: `[message blob missing: ${sha1}]` }; } @@ -1128,7 +1156,7 @@ function resolvePayload(payload) { function readArtifact(filePath) { if (!filePath || !isSafeArtifactPath(filePath)) return null; if (!fs.existsSync(filePath)) return null; - const content = fs.readFileSync(filePath, 'utf8'); + const content = readStoredText(filePath); let parsed = null; try { parsed = resolvePayload(JSON.parse(content)); @@ -1213,7 +1241,7 @@ function artifactTextForSearch(filePath) { let text = ''; if (stat.isFile() && stat.size <= MAX_ARTIFACT_SEARCH_BYTES) { try { - text = fs.readFileSync(filePath, 'utf8'); + text = readStoredText(filePath).slice(0, MAX_ARTIFACT_SEARCH_BYTES); } catch { text = ''; } diff --git a/raven/node/install.py b/raven/node/install.py index 11d61c67b..09d82a2c9 100644 --- a/raven/node/install.py +++ b/raven/node/install.py @@ -65,8 +65,8 @@ from raven.utils import mirrors #: Roughly what a node takes, for the free-space check: Raven's code with -#: pillow, loguru and ripgrep-bin (measured 2026-10-10), a Python 3.12 when -#: the machine has none of its own, and uv when the machine has none. +#: pillow, loguru, zstandard and ripgrep-bin, a Python 3.12 when the machine +#: has none of its own, and uv when the machine has none. _NODE_KB = 60 * 1024 _PYTHON_KB = 80 * 1024 _UV_KB = 25 * 1024 diff --git a/raven/node/protocol.py b/raven/node/protocol.py index 8ca0b3267..39e20b158 100644 --- a/raven/node/protocol.py +++ b/raven/node/protocol.py @@ -123,9 +123,10 @@ TASK_KEEP_S = 7 * 24 * 3600 #: Third-party packages the daemon cannot run without, beside Raven's own -#: code: the only ones its file tools import (measured 2026-10-10), pinned to -#: the host's versions. -REQUIREMENTS = ("loguru", "pillow") +#: code: the file tools' two, plus zstandard, which the tracing store the +#: daemon links imports for its payload frames (measured 2026-10-29), pinned +#: to the host's versions. +REQUIREMENTS = ("loguru", "pillow", "zstandard") #: Installed when the machine can get it, not required. ``ripgrep-bin`` puts the #: same ``rg`` beside the daemon's Python that the host has beside its own, so a diff --git a/raven/rpc/methods/cli_dispatch.py b/raven/rpc/methods/cli_dispatch.py index 1e77039b6..3f85bc543 100644 --- a/raven/rpc/methods/cli_dispatch.py +++ b/raven/rpc/methods/cli_dispatch.py @@ -98,9 +98,11 @@ # backing thread is not cancelled on timeout — it would keep running # detached after the RPC call returns, risking Ink reconciler corruption. # Bare ``tracing`` and ``tracing stop`` are unaffected: only this action - # is blacklisted. ``compact`` is a positional action, so this prefix - # catches it only at argv[1] — _is_tracing_compact covers the rest. + # is blacklisted. The maintenance actions are positional arguments, so this + # prefix catches only the argv[1] spelling — _is_tracing_maintenance covers + # every other order. ("tracing", "compact"), # full-store hash walk can outrun the RPC timeout — terminal-only + ("tracing", "compress"), # same walk (and, with --logs, a whole archive rewrite) # ``agent`` (no -m) — handled specially in _is_dispatch_compatible } @@ -117,22 +119,21 @@ def _is_agent_repl(argv: list[str]) -> bool: return not any(a in ("-m", "--message") for a in argv[1:]) -def _is_tracing_compact(argv: list[str]) -> bool: - """Return True for a ``tracing compact`` invocation, at any argv position. +def _is_tracing_maintenance(argv: list[str]) -> bool: + """Return True for a ``tracing compact`` or ``tracing compress`` invocation. - ``compact`` is a positional argument of the ``tracing`` leaf command, so - Click accepts it after any option: ``tracing --dry-run compact`` and - ``tracing --port 1 compact`` both run it. The ``_DISPATCH_BLACKLIST`` - prefix match compares fixed positions and therefore only sees the action - at ``argv[1]``, so it alone would let those forms reach dispatch. + Both are positional arguments of the ``tracing`` leaf command, so Click + accepts them after any option: ``tracing --dry-run compact`` and + ``tracing --port 1 compress`` both run. The ``_DISPATCH_BLACKLIST`` prefix + match compares fixed positions and therefore only sees the action at + ``argv[1]``, so it alone would let those forms reach dispatch. - ``raven tracing compact`` -> blacklist hit - ``raven tracing --dry-run compact`` -> blacklist hit (this predicate) + ``raven tracing compact`` / ``tracing compress`` -> terminal-only ``raven tracing`` / ``tracing stop`` -> OK """ if not argv or argv[0] != "tracing": return False - return "compact" in argv[1:] + return any(action in argv[1:] for action in ("compact", "compress")) # Default timeout if caller did not override. @@ -198,7 +199,7 @@ def _is_dispatch_compatible(argv: list[str]) -> bool: return False if _is_agent_repl(argv): return False - if _is_tracing_compact(argv): + if _is_tracing_maintenance(argv): return False # Reflection-based positive check (replaces the v0.0.2 hardcoded # _DISPATCH_WHITELIST 19-tuple set). The helper module also serves diff --git a/raven/tracing/artifact_v2.py b/raven/tracing/artifact_v2.py index 4f6852a44..ea01c8d85 100644 --- a/raven/tracing/artifact_v2.py +++ b/raven/tracing/artifact_v2.py @@ -6,8 +6,9 @@ the viewer's. The canonical serialization, the envelope key, the reference validation and the per-field resolution rules all live here and nowhere else. -Stdlib only, like :mod:`raven.tracing.store`, so the tracing package stays -installable on its own. +The wire shape itself is stdlib; the default reader decodes the store's zstd +frames through :mod:`raven.tracing.codec`, imported lazily so this module +stays cheap to import. """ from __future__ import annotations @@ -18,6 +19,8 @@ from pathlib import Path from typing import Any, Callable +from .codec import read_plaintext + ARTIFACT_FORMAT = "audit.artifact.v2" MESSAGES_DIR_NAME = "_messages" REF_KEY = "$msg" @@ -108,6 +111,14 @@ def placeholder(sha1: str) -> dict[str, str]: return {"role": "unknown", "content": f"[message blob missing: {sha1}]"} +def _read_stored_text(path: Path) -> str: + """A blob's plaintext, whichever encoding it was stored under.""" + plain = read_plaintext(path.read_bytes()) + if plain is None: + raise ValueError("corrupt compressed message blob") + return plain.decode("utf-8") + + def resolve_payload( payload: Any, artifacts_dir: Path | str, @@ -133,7 +144,7 @@ def load(sha1: str) -> Any: if sha1 not in cache: try: path = message_path(artifacts_dir, sha1) - text = path.read_text(encoding="utf-8") if read is None else read(path) + text = _read_stored_text(path) if read is None else read(path) if text is None: raise OSError(f"blob {sha1} could not be read") cache[sha1] = json.loads(text) diff --git a/raven/tracing/codec.py b/raven/tracing/codec.py new file mode 100644 index 000000000..b92171db7 --- /dev/null +++ b/raven/tracing/codec.py @@ -0,0 +1,79 @@ +"""The zstd framing every stored payload shares. + +A payload is written as a zstd frame under the path it always had, so a reader +tells the encoding from the frame's magic and never from the file name. That +is what keeps a store written before compression readable and a half-migrated +one coherent. + +Only the encoding lives here: paths, dedupe and migration stay in +``store``/``compress``, so both can import this module without reaching back +into each other. +""" + +from __future__ import annotations + +import io +from pathlib import Path + +import zstandard + +#: A zstd frame always opens with these four bytes. A JSON or text payload +#: cannot start with them (0xB5 is not a UTF-8 leading byte), which is what +#: makes the sniff safe for everything the store writes. +ZSTD_MAGIC = b"\x28\xb5\x2f\xfd" +COMPRESSION_LEVEL = 3 + + +def is_compressed(data: bytes) -> bool: + return data[:4] == ZSTD_MAGIC + + +def compress_bytes(data: bytes) -> bytes: + # A fresh compressor per call: ZstdCompressor instances are not documented + # as thread safe, and the store writes from more than one worker thread. + return zstandard.ZstdCompressor(level=COMPRESSION_LEVEL).compress(data) + + +def decompress_bytes(data: bytes, limit: int | None = None) -> bytes: + """The plaintext in ``data``, at most ``limit`` bytes of it. + + ``limit`` bounds the output, not just the returned slice: a caller that + only wants a preview must not pay for a whole multi-megabyte payload. + """ + if not data: + return b"" + with zstandard.ZstdDecompressor().stream_reader(io.BytesIO(data), read_across_frames=False) as reader: + return reader.read() if limit is None else reader.read(limit) + + +def read_plaintext(data: bytes) -> bytes | None: + """The payload's plaintext, or None when a compressed frame is unreadable.""" + try: + return decompress_bytes(data) if is_compressed(data) else data + except zstandard.ZstdError: + return None + + +def plaintext_size(path: Path) -> int | None: + """The payload's uncompressed length, from the frame header when possible.""" + try: + with path.open("rb") as handle: + head = handle.read(4) + if not is_compressed(head): + return path.stat().st_size + handle.seek(0) + size = zstandard.frame_content_size(handle.read(18)) + except (OSError, zstandard.ZstdError): + return None + return None if size < 0 else int(size) + + +__all__ = [ + "COMPRESSION_LEVEL", + "ZSTD_MAGIC", + "compress_bytes", + "decompress_bytes", + "is_compressed", + "plaintext_size", + "read_plaintext", +] diff --git a/raven/tracing/compact.py b/raven/tracing/compact.py index 0ecfbe992..ab985a68b 100644 --- a/raven/tracing/compact.py +++ b/raven/tracing/compact.py @@ -49,7 +49,10 @@ from dataclasses import dataclass, field from pathlib import Path +import zstandard + from raven.tracing import artifact_v2 +from raven.tracing.codec import ZSTD_MAGIC from raven.tracing.store import BLOBS_DIR_NAME # Directories under ``audit-artifacts/`` that are content-addressed stores, not @@ -110,10 +113,26 @@ def _record_error(result: CompactResult, message: str) -> None: def _sha1_of(path: Path) -> str: + """The plaintext digest: a compressed artifact hashes what it decodes to. + + A blob's name is the digest of the payload, never of its zstd frame, so a + compressed artifact has to be decoded before it can be compared with, or + folded onto, the blob it belongs to. + """ digest = hashlib.sha1() with path.open("rb") as handle: - for chunk in iter(lambda: handle.read(_READ_CHUNK), b""): - digest.update(chunk) + head = handle.read(4) + handle.seek(0) + if head == ZSTD_MAGIC: + try: + with zstandard.ZstdDecompressor().stream_reader(handle, read_across_frames=False) as reader: + for chunk in iter(lambda: reader.read(_READ_CHUNK), b""): + digest.update(chunk) + except zstandard.ZstdError as exc: + raise OSError(f"corrupt compressed artifact: {exc}") from exc + else: + for chunk in iter(lambda: handle.read(_READ_CHUNK), b""): + digest.update(chunk) return digest.hexdigest() diff --git a/raven/tracing/compress.py b/raven/tracing/compress.py new file mode 100644 index 000000000..5ee4d3558 --- /dev/null +++ b/raven/tracing/compress.py @@ -0,0 +1,162 @@ +"""Compress a trace store's payloads in place, under the paths they already have. + +A span records its artifact's path when it is written, and those records are +the audit trail: a migration cannot rename them. So this keeps every path and +changes only the bytes, then re-points each artifact hard link at the one +compressed copy of its payload. Readers pick the encoding from the zstd frame, +never from the name, which is also what lets a half-migrated store read. + +Idempotent: a blob or message already compressed is counted and left alone, so +a second pass writes nothing. A dry run compresses in memory to report the real +saving and writes no file at all.""" + +from __future__ import annotations + +import os +import uuid +from dataclasses import dataclass, field +from pathlib import Path +from typing import Iterator + +from . import artifact_v2 +from .codec import compress_bytes, is_compressed +from .store import BLOBS_DIR_NAME + + +@dataclass +class CompressResult: + """What one pass saw, did, and saved.""" + + blobs: int = 0 + compressed: int = 0 + already: int = 0 + relinked: int = 0 + messages: int = 0 + messages_compressed: int = 0 + bytes_before: int = 0 + bytes_after: int = 0 + errors: list[str] = field(default_factory=list) + + @property + def saved(self) -> int: + return max(self.bytes_before - self.bytes_after, 0) + + +def _files(root: Path) -> Iterator[Path]: + for directory, _, names in os.walk(root): + for name in names: + if name.startswith("."): + continue + path = Path(directory) / name + if path.is_file(): + yield path + + +def _replace(path: Path, payload: bytes) -> None: + """Swap ``path``'s bytes without ever leaving the name pointing at nothing.""" + tmp = path.parent / f".{path.name}.{os.getpid()}.{uuid.uuid4().hex}.compress.tmp" + try: + tmp.write_bytes(payload) + os.replace(tmp, path) + finally: + tmp.unlink(missing_ok=True) + + +def _relink(blob: Path, artifact: Path, old_inode: int) -> bool: + """Point ``artifact`` at ``blob``, leaving it readable across the swap. + + A hard link is staged beside the artifact and renamed over it, so the path + never passes through a state where it does not exist. The inode check + keeps a path that something else has already replaced from being clobbered. + """ + try: + if artifact.stat().st_ino != old_inode: + return False + staged = artifact.parent / f".{artifact.name}.{os.getpid()}.{uuid.uuid4().hex}.relink" + try: + os.link(blob, staged) + os.replace(staged, artifact) + finally: + staged.unlink(missing_ok=True) + return True + except OSError: + return False + + +def compress(artifacts_dir: Path, *, dry_run: bool = False) -> CompressResult: + """Compress every plain payload under ``artifacts_dir``; see the module doc.""" + result = CompressResult() + artifacts_dir = Path(artifacts_dir) + if not artifacts_dir.is_dir(): + return result + blobs_dir = artifacts_dir / BLOBS_DIR_NAME + messages_dir = artifact_v2.messages_dir(artifacts_dir) + + # One walk to learn which artifact paths share each blob inode; the links + # must be moved after the blob's own name is swapped. + links: dict[int, list[Path]] = {} + for path in _files(artifacts_dir): + if path.is_relative_to(blobs_dir) or path.is_relative_to(messages_dir): + continue + try: + inode = path.stat().st_ino + except OSError as exc: + result.errors.append(f"{path}: {exc}") + continue + links.setdefault(inode, []).append(path) + + for blob in _files(blobs_dir): + result.blobs += 1 + try: + data = blob.read_bytes() + inode = blob.stat().st_ino + except OSError as exc: + result.errors.append(f"{blob}: {exc}") + continue + if is_compressed(data): + result.already += 1 + continue + packed = compress_bytes(data) + result.bytes_before += len(data) + result.bytes_after += len(packed) + result.compressed += 1 + if dry_run: + continue + # Stage beside the blob first, move the artifact links onto the staged + # inode, and only then give the blob name that inode: a crash in the + # middle leaves every path readable, one encoding or the other. + staged = blob.parent / f".{blob.name}.{os.getpid()}.{uuid.uuid4().hex}.compress.tmp" + try: + staged.write_bytes(packed) + for artifact in links.get(inode, []): + if _relink(staged, artifact, inode): + result.relinked += 1 + os.replace(staged, blob) + except OSError as exc: + result.errors.append(f"{blob}: {exc}") + finally: + staged.unlink(missing_ok=True) + + for path in _files(messages_dir): + result.messages += 1 + try: + data = path.read_bytes() + except OSError as exc: + result.errors.append(f"{path}: {exc}") + continue + if is_compressed(data): + continue + packed = compress_bytes(data) + result.bytes_before += len(data) + result.bytes_after += len(packed) + result.messages_compressed += 1 + if dry_run: + continue + try: + _replace(path, packed) + except OSError as exc: + result.errors.append(f"{path}: {exc}") + return result + + +__all__ = ["CompressResult", "compress"] diff --git a/raven/tracing/store.py b/raven/tracing/store.py index 9ad59e20f..78ff2afac 100644 --- a/raven/tracing/store.py +++ b/raven/tracing/store.py @@ -1,13 +1,14 @@ """JSONL + artifact storage for the trace core (audit.span.v1). -Dependency-free (stdlib only). Copied from the shared tracing-plugin core so -this package stays self-contained for a clean ``pip install``. +The one third-party dependency is ``zstandard``: a payload is stored as a zstd +frame under the path it always had, so readers tell the encoding from the +frame and a store written before compression stays readable. Layout under the state dir:: /logs/audit-events.log # one JSON event record per line /logs/audit-spans.log # one JSON span per line - /logs/audit-artifacts/_blobs//. # one copy per payload + /logs/audit-artifacts/_blobs//. # one zstd copy per payload /logs/audit-artifacts///... # hard links to those blobs /logs/archive//... # rotated logs """ @@ -23,7 +24,10 @@ from pathlib import Path from typing import Any +import zstandard + from . import artifact_v2 +from .codec import compress_bytes, is_compressed DEFAULT_MAX_BYTES = 50 * 1024 * 1024 @@ -114,7 +118,8 @@ def _rotate_if_needed(self, kind: str, next_text: str) -> Path: day_dir = self._ensure(self.archive_dir / current_day) suffix = datetime.now(timezone.utc).strftime("%H%M%S%f") base = _KIND_FILES[kind].replace(".log", "") - path.rename(day_dir / f"{base}-{current_day}-{suffix}.log") + rotated = day_dir / f"{base}-{current_day}-{suffix}.log" + path.rename(rotated) return path def append(self, kind: str, record: dict[str, Any]) -> None: @@ -165,15 +170,25 @@ def _blob_is_intact(self, blob: Path, sha1: str) -> bool: return True digest = hashlib.sha1() with blob.open("rb") as handle: - for chunk in iter(lambda: handle.read(_BLOB_READ_CHUNK), b""): - digest.update(chunk) + head = handle.read(4) + handle.seek(0) + if is_compressed(head): + try: + with zstandard.ZstdDecompressor().stream_reader(handle, read_across_frames=False) as reader: + for chunk in iter(lambda: reader.read(_BLOB_READ_CHUNK), b""): + digest.update(chunk) + except zstandard.ZstdError: + return False + else: + for chunk in iter(lambda: handle.read(_BLOB_READ_CHUNK), b""): + digest.update(chunk) if digest.hexdigest() != sha1: return False self._verified_blobs[blob.name] = token return True - def _publish_blob(self, blob: Path, text: str, sha1: str) -> None: - """Create ``blob`` holding ``text``, losing gracefully to a racing writer. + def _publish_blob(self, blob: Path, payload: bytes, sha1: str) -> None: + """Create ``blob`` holding ``payload``, losing gracefully to a racing writer. The temp file is named with a ``uuid4`` (not just the pid), so two writers in the same process publishing the same new payload never @@ -186,7 +201,7 @@ def _publish_blob(self, blob: Path, text: str, sha1: str) -> None: self._ensure(blob.parent) tmp = blob.parent / f"{sha1}.{os.getpid()}.{uuid.uuid4().hex}.tmp" try: - tmp.write_text(text, encoding="utf-8") + tmp.write_bytes(payload) os.link(tmp, blob) self._remember_blob(blob) except FileExistsError: @@ -194,7 +209,7 @@ def _publish_blob(self, blob: Path, text: str, sha1: str) -> None: finally: tmp.unlink(missing_ok=True) - def _repair_blob(self, blob: Path, text: str, sha1: str) -> None: + def _repair_blob(self, blob: Path, payload: bytes, sha1: str) -> None: """Point ``blob`` back at the bytes its name claims. ``os.replace`` here, unlike the publish path: swapping the directory @@ -206,14 +221,14 @@ def _repair_blob(self, blob: Path, text: str, sha1: str) -> None: """ tmp = blob.parent / f"{sha1}.{os.getpid()}.{uuid.uuid4().hex}.tmp" try: - tmp.write_text(text, encoding="utf-8") + tmp.write_bytes(payload) os.replace(tmp, blob) self._remember_blob(blob) finally: tmp.unlink(missing_ok=True) def _materialize(self, file_path: Path, text: str, sha1: str, extension: str) -> None: - """Link ``file_path`` at the single stored copy of ``text``. + """Link ``file_path`` at the single stored copy of ``text``'s compressed form. Hard link, not copy: the payload occupies its blocks once however many spans reference it, and the link count the filesystem keeps is the @@ -228,17 +243,18 @@ def _materialize(self, file_path: Path, text: str, sha1: str, extension: str) -> per process, not per span, and none at all when the cached (inode, mtime, size) still matches. """ + payload = compress_bytes(text.encode("utf-8")) blob = self._blob_path(sha1, extension) try: if not blob.exists(): - self._publish_blob(blob, text, sha1) + self._publish_blob(blob, payload, sha1) if blob.exists() and not self._blob_is_intact(blob, sha1): - self._repair_blob(blob, text, sha1) + self._repair_blob(blob, payload, sha1) os.link(blob, file_path) except FileExistsError: pass except OSError: - file_path.write_text(text, encoding="utf-8") + file_path.write_bytes(payload) def persist_artifact( self, @@ -296,10 +312,11 @@ def address_items(self, items: list[Any]) -> list[Any]: text = artifact_v2.message_text(item) sha1 = artifact_v2.message_sha1(item) blob = artifact_v2.message_path(self.artifacts_dir, sha1) + payload = compress_bytes(text.encode("utf-8")) if not blob.exists(): - self._publish_blob(blob, text, sha1) + self._publish_blob(blob, payload, sha1) if blob.exists() and not self._blob_is_intact(blob, sha1): - self._repair_blob(blob, text, sha1) + self._repair_blob(blob, payload, sha1) refs.append(artifact_v2.make_ref(sha1)) except OSError: refs.append(item) diff --git a/raven/trajectory/bundle.py b/raven/trajectory/bundle.py index 03907fdeb..4c511dcf1 100644 --- a/raven/trajectory/bundle.py +++ b/raven/trajectory/bundle.py @@ -40,6 +40,7 @@ from raven.config.paths import get_workspace_path from raven.tracing import artifact_v2 from raven.tracing import config as tracing_config +from raven.tracing.codec import read_plaintext from raven.trajectory.selection import read_store_file from raven.trajectory.store import attempt_alias_ids, iter_spans, pin, pin_attempt, resolve_attempt_id from raven.trajectory.verdict import Verdict, read_verdicts @@ -49,6 +50,16 @@ _ARTIFACTS_DIR = "artifacts" +def _stored_bytes(path: Path) -> bytes | None: + """The payload's plaintext: a bundle leaves this machine, so it must not + carry the store's on-disk encoding.""" + try: + data = path.read_bytes() + except OSError: + return None + return read_plaintext(data) + + def _copy_resolved( src: Path, target: Path, @@ -65,8 +76,13 @@ def _copy_resolved( ``read`` bounds how the shell's blobs are read. """ try: - payload = json.loads(src.read_text(encoding="utf-8") if text is None else text) - except (OSError, ValueError): + if text is None: + stored = _stored_bytes(src) + if stored is None: + return False + text = stored.decode("utf-8") + payload = json.loads(text) + except (OSError, ValueError, UnicodeDecodeError): return False if not artifact_v2.is_v2(payload): return False @@ -149,7 +165,11 @@ def _import_artifact( if text is None or not _copy_resolved(src, target, store_artifacts, missing_messages, text=text, read=read): target.write_bytes(data) elif not _copy_resolved(src, target, store_artifacts, missing_messages): - shutil.copy2(src, target) + plain = _stored_bytes(src) + if plain is None: + missing.append(source) + return None + target.write_bytes(plain) rel = f"{_ARTIFACTS_DIR}/{name}" copied[source] = rel return rel diff --git a/raven/trajectory/conversation.py b/raven/trajectory/conversation.py index e1c3d9a09..1c38b9c1a 100644 --- a/raven/trajectory/conversation.py +++ b/raven/trajectory/conversation.py @@ -42,8 +42,11 @@ from pathlib import Path from typing import Any, Sequence +import zstandard + from raven.tracing import artifact_v2 from raven.tracing import config as tracing_config +from raven.tracing.codec import ZSTD_MAGIC from raven.trajectory import store as tstore _log = logging.getLogger("raven.trajectory.conversation") @@ -149,9 +152,17 @@ def _read_artifact(state: Path, path: str, limit: int = _ARTIFACT_LIMIT) -> tupl if target.exists() and not target.is_file(): return None, "artifact is not a regular file" with target.open("rb") as handle: - data = handle.read(limit + 1) + head = handle.read(4) + handle.seek(0) + if head == ZSTD_MAGIC: + with zstandard.ZstdDecompressor().stream_reader(handle, read_across_frames=False) as reader: + data = reader.read(limit + 1) + else: + data = handle.read(limit + 1) except OSError: return None, "artifact missing" + except zstandard.ZstdError: + return None, "artifact unreadable: corrupt compressed content" except ValueError: # A pointer with an embedded NUL cannot name a file. return None, "artifact unreadable: the path is not valid" diff --git a/raven/trajectory/details.py b/raven/trajectory/details.py index 609890d92..85336efbb 100644 --- a/raven/trajectory/details.py +++ b/raven/trajectory/details.py @@ -40,6 +40,7 @@ from typing import Any, Callable, Sequence from raven.tracing import artifact_v2 +from raven.tracing import codec as trace_codec from raven.trajectory import conversation as _conv from raven.trajectory import entries as _entries from raven.trajectory import store as tstore @@ -491,10 +492,7 @@ def blob_prefix(self, sha1: str, limit: int) -> tuple[str | None, int | None] | if not self._allowed(True): self.exhausted = True return None - try: - size: int | None = path.stat().st_size - except OSError: - size = None + size: int | None = trace_codec.plaintext_size(path) text, _ = _conv._read_artifact(self.state, str(path), limit) if size is not None else (None, None) self.blobs += 1 self.bytes += len(text.encode("utf-8", errors="replace")) if text else 0 diff --git a/raven/trajectory/selection.py b/raven/trajectory/selection.py index 4a9e15be1..416ba986e 100644 --- a/raven/trajectory/selection.py +++ b/raven/trajectory/selection.py @@ -359,7 +359,11 @@ def read_store_file(path: Path, root: Path) -> tuple[bytes, float] | None: return None with os.fdopen(fd, "rb") as handle: fd = -1 - return handle.read(), info.st_mtime + data = handle.read() + from raven.tracing.codec import read_plaintext + + plain = read_plaintext(data) + return (plain, info.st_mtime) if plain is not None else None except OSError: return None finally: diff --git a/tests/integration/test_tracing_viewer_e2e.py b/tests/integration/test_tracing_viewer_e2e.py index 17c045af9..df3a78c95 100644 --- a/tests/integration/test_tracing_viewer_e2e.py +++ b/tests/integration/test_tracing_viewer_e2e.py @@ -42,6 +42,7 @@ import subprocess import time import urllib.error +import urllib.parse import urllib.request from contextlib import closing, contextmanager from pathlib import Path @@ -161,6 +162,24 @@ def test_ignores_a_log_planted_outside_the_active_and_archive_paths(tmp_path): assert _session_ids(_get(port, "/api/data")) == {"live-session"} +def test_artifact_endpoint_decodes_a_compressed_payload(tmp_path): + from raven.tracing.codec import compress_bytes + + logs = tmp_path / "logs" + artifact = logs / "audit-artifacts" / "llm.input" / "2026-08-01" / "a.json" + artifact.parent.mkdir(parents=True) + payload = {"messages": [{"role": "user", "content": "hi" * 100}]} + artifact.write_bytes(compress_bytes(json.dumps(payload).encode("utf-8"))) + span = _span("live-session", "span-live") + span["attributes"]["llm.input.artifact_path"] = str(artifact) + _write_spans(logs / "audit-spans.log", [span]) + + with _viewer(tmp_path) as port: + found = _get(port, "/api/artifact?path=" + urllib.parse.quote(str(artifact), safe="")) + + assert found["parsed"] == payload + + def test_repeated_read_is_served_without_rebuilding(tmp_path): _write_spans(tmp_path / "logs" / "audit-spans.log", [_span("live-session", "span-live")]) diff --git a/tests/test_cli_tracing_commands.py b/tests/test_cli_tracing_commands.py index c69e41071..51674f713 100644 --- a/tests/test_cli_tracing_commands.py +++ b/tests/test_cli_tracing_commands.py @@ -12,6 +12,7 @@ from __future__ import annotations +import hashlib import json import os import signal @@ -402,3 +403,42 @@ def test_a_below_minimum_node_is_refused(monkeypatch): monkeypatch.setattr("raven.cli.tui_commands.find_node", lambda: ("/usr/bin/node", (20, 0, 0))) with pytest.raises(typer.Exit): tc._resolve_node() + + +def _plain_blob_with_link(state_dir: Path, text: str) -> tuple[Path, Path]: + sha1 = hashlib.sha1(text.encode("utf-8")).hexdigest() + blob = state_dir / "logs" / "audit-artifacts" / "_blobs" / sha1[:2] / f"{sha1}.json" + blob.parent.mkdir(parents=True) + blob.write_text(text, encoding="utf-8") + artifact = _aged_artifact(state_dir, "a.json", text) + artifact.unlink() + os.link(blob, artifact) + stamp = time.time() - 3600 + os.utime(blob, (stamp, stamp)) + os.utime(artifact, (stamp, stamp)) + return artifact, blob + + +def test_tracing_compress_dry_run_reports_and_writes_nothing(tmp_path, monkeypatch): + monkeypatch.setenv("RAVEN_TRACING_DIR", str(tmp_path)) + text = '{"m":"same"}' + artifact, _ = _plain_blob_with_link(tmp_path, text) + + r = runner.invoke(app, ["tracing", "compress", "--dry-run"]) + + assert r.exit_code == 0 + assert "compressed 1" in r.output + assert "saved" in r.output + assert artifact.read_bytes() == text.encode("utf-8") + + +def test_tracing_compress_rewrites_the_blob_and_keeps_the_link(tmp_path, monkeypatch): + monkeypatch.setenv("RAVEN_TRACING_DIR", str(tmp_path)) + text = '{"m":"same"}' + artifact, blob = _plain_blob_with_link(tmp_path, text) + + r = runner.invoke(app, ["tracing", "compress"]) + + assert r.exit_code == 0 + assert artifact.read_bytes().startswith(b"\x28\xb5\x2f\xfd") + assert artifact.stat().st_ino == blob.stat().st_ino diff --git a/tests/test_rpc_cli_dispatch.py b/tests/test_rpc_cli_dispatch.py index c1c3cf2d9..bc7c316e4 100644 --- a/tests/test_rpc_cli_dispatch.py +++ b/tests/test_rpc_cli_dispatch.py @@ -258,13 +258,13 @@ def test_channels_login_normal_channels_dispatch(): def test_blacklist_full_coverage(): - """9-entry blacklist (5 original + tui + onboard + upgrade + tracing compact). + """10-entry blacklist (5 original + tui + onboard + upgrade + 2 tracing maintenance). `tui` blocks recursive Ink+Node spawn; `onboard` blocks prompt_toolkit wizard stdin hijack; `upgrade` blocks replacing the active Raven process; - `tracing compact` blocks a full-store hash walk that can outrun the RPC - timeout with no cancellation of the backing thread. All are necessary - because reflection will otherwise let them through. + the `tracing` maintenance actions block a full-store walk that can outrun + the RPC timeout with no cancellation of the backing thread. All are + necessary because reflection will otherwise let them through. """ from raven.rpc.methods.cli_dispatch import _DISPATCH_BLACKLIST @@ -278,9 +278,10 @@ def test_blacklist_full_coverage(): ("onboard",), ("upgrade",), ("tracing", "compact"), + ("tracing", "compress"), } assert _DISPATCH_BLACKLIST == expected_entries, ( - f"blacklist drift; expected 9 prefix tuples (+ agent REPL special-case), got {_DISPATCH_BLACKLIST}" + f"blacklist drift; expected 10 prefix tuples (+ agent REPL special-case), got {_DISPATCH_BLACKLIST}" ) # Hard-reject probe (prefix match) assert _is_dispatch_compatible(["tui"]) is False @@ -291,8 +292,9 @@ def test_blacklist_full_coverage(): assert _is_dispatch_compatible(["upgrade", "--check"]) is False # prefix assert _is_dispatch_compatible(["tracing", "compact"]) is False assert _is_dispatch_compatible(["tracing", "compact", "--dry-run"]) is False # prefix + assert _is_dispatch_compatible(["tracing", "compress"]) is False # Bare `tracing` and its other action stay dispatchable -- only the - # specific `compact` action is terminal-only. + # maintenance actions are terminal-only. assert _is_dispatch_compatible(["tracing"]) is True assert _is_dispatch_compatible(["tracing", "stop"]) is True @@ -303,7 +305,7 @@ def test_tracing_compact_is_rejected_whatever_the_option_order(): The blacklist compares fixed argv positions, so the prefix tuple alone only catches the action at argv[1]: `tracing --port 1 compact` would otherwise reach dispatch and run a real, writing compaction detached - past the timeout. `_is_tracing_compact` closes every order. + past the timeout. `_is_tracing_maintenance` closes every order. """ for argv in ( ["tracing", "--dry-run", "compact"], @@ -325,9 +327,17 @@ def test_tracing_compact_is_rejected_whatever_the_option_order(): assert _is_dispatch_compatible(argv) is True, argv -# --------------------------------------------------------------------------- -# Reflection-based dispatch compat (_DISPATCH_WHITELIST deleted) -# --------------------------------------------------------------------------- +def test_tracing_compress_is_rejected_whatever_the_option_order(): + """``compress`` walks the whole artifact store and writes in place, like + ``compact``; it must never run detached from a TUI slash command.""" + for argv in ( + ["tracing", "compress"], + ["tracing", "compress", "--dry-run"], + ["tracing", "--dry-run", "compress"], + ["tracing", "--port", "1", "compress"], + ["tracing", "-p", "1", "compress"], + ): + assert _is_dispatch_compatible(argv) is False, argv def test_dispatch_compatible_via_reflection_newly_active(): diff --git a/tests/test_subagent_acp.py b/tests/test_subagent_acp.py index 19afa60df..cb97b038f 100644 --- a/tests/test_subagent_acp.py +++ b/tests/test_subagent_acp.py @@ -3715,6 +3715,14 @@ async def test_a_failed_turn_marks_the_span_and_still_stores_the_transcript(trac # ---- the cli lane records too ----------------------------------------------- +def _stored_text(path) -> str: + """A stored artifact's plaintext, whichever encoding it was written under.""" + from raven.tracing.codec import decompress_bytes, is_compressed + + data = Path(path).read_bytes() + return (decompress_bytes(data) if is_compressed(data) else data).decode("utf-8") + + def _cli_stub_config(name: str = "clistub", *, exit_code: int = 0) -> ThirdPartyCliSubagentConfig: script = ( "import sys; " @@ -3748,7 +3756,7 @@ async def test_the_cli_lane_records_the_same_span_shape(trace_dir, tmp_path: Pat assert "subagent.external.update_counts" not in attrs assert not span["events"] - stored = json.loads(Path(attrs["subagent.external.transcript.artifact_path"]).read_text(encoding="utf-8")) + stored = json.loads(_stored_text(attrs["subagent.external.transcript.artifact_path"])) assert stored["invocations"][0]["stdout"].strip() == "hello from the cli" assert stored["invocations"][0]["stderr"].strip() == "some diagnostics" @@ -3764,9 +3772,7 @@ async def test_a_failed_cli_invocation_keeps_its_transcript(trace_dir, tmp_path: span = next(s for s in _spans_written(trace_dir) if s.get("name") == SPAN_NAME) assert span["status"]["code"] == "ERROR" assert span["attributes"]["subagent.external.exit_code"] == 3 - stored = json.loads( - Path(span["attributes"]["subagent.external.transcript.artifact_path"]).read_text(encoding="utf-8") - ) + stored = json.loads(_stored_text(span["attributes"]["subagent.external.transcript.artifact_path"])) assert stored["invocations"][0]["stdout"].strip() == "hello from the cli" diff --git a/tests/test_tracing_api.py b/tests/test_tracing_api.py index 5d4ce0331..320b7c41d 100644 --- a/tests/test_tracing_api.py +++ b/tests/test_tracing_api.py @@ -16,6 +16,16 @@ from raven.tracing import spans as _spans from raven.tracing import store as _store_mod + + +def _stored_text(path): + """A stored payload's plaintext, whichever encoding it was written under.""" + from raven.tracing.codec import decompress_bytes, is_compressed + + data = Path(path).read_bytes() + return (decompress_bytes(data) if is_compressed(data) else data).decode("utf-8") + + from raven.tracing import trace @@ -198,7 +208,7 @@ def request_generation(self, **asked): assert attrs["llm.request.temperature"] == 0.95 assert attrs["llm.request.effort_sent"] == "reasoning.effort=high" assert "Z.AI" in attrs["llm.request.provider_fence"] - payload = json.loads(Path(attrs["llm.input.artifact_path"]).read_text(encoding="utf-8")) + payload = json.loads(_stored_text(attrs["llm.input.artifact_path"])) assert payload["generation"]["reasoning"] == {"effort": "high"} assert "secret" not in json.dumps(attrs) and "openrouter.ai" not in json.dumps(attrs) @@ -344,8 +354,7 @@ def test_skill_gate_extractor_records_the_subagent_roster(trace_dir): None, ) a = _spans_written(trace_dir)[0]["attributes"] - with open(a["skill.gate.input.artifact_path"], encoding="utf-8") as fh: - payload = json.load(fh) + payload = json.loads(_stored_text(a["skill.gate.input.artifact_path"])) assert payload["available_subagents"] == "Scribe [stateless] (writes decks)" @@ -508,7 +517,7 @@ def test_a_mutated_artifact_does_not_corrupt_the_next_record(trace_dir): Path(first["path"]).write_text("CORRUPT", encoding="utf-8") second = store.persist_artifact("tool.output", {"traceId": "t2"}, payload) - text = Path(second["path"]).read_text(encoding="utf-8") + text = _stored_text(second["path"]) assert json.loads(text) == payload assert hashlib.sha1(text.encode("utf-8")).hexdigest() == second["sha1"] @@ -548,7 +557,7 @@ def test_an_intact_blob_is_not_rehashed_for_every_span(trace_dir, monkeypatch): real_open = Path.open def counting_open(self, mode="r", *args, **kwargs): - if _store_mod.BLOBS_DIR_NAME in self.parts and "b" in mode: + if _store_mod.BLOBS_DIR_NAME in self.parts and "b" in mode and not self.name.endswith(".tmp"): blob_reads.append(self) return real_open(self, mode, *args, **kwargs) @@ -569,8 +578,8 @@ def test_identical_payloads_share_one_inode(trace_dir): p1, p2 = Path(first["path"]), Path(second["path"]) assert p1 != p2 assert p1.stat().st_ino == p2.stat().st_ino - assert json.loads(p1.read_text(encoding="utf-8")) == payload - assert json.loads(p2.read_text(encoding="utf-8")) == payload + assert json.loads(_stored_text(p1)) == payload + assert json.loads(_stored_text(p2)) == payload assert first["sha1"] == second["sha1"] assert first["bytes"] == second["bytes"] @@ -613,7 +622,7 @@ def _no_links(*_args, **_kwargs): art = store.persist_artifact("tool.input", {"traceId": "t1"}, payload) path = Path(art["path"]) - assert json.loads(path.read_text(encoding="utf-8")) == payload + assert json.loads(_stored_text(path)) == payload assert path.stat().st_nlink == 1 assert art.get("error") is None assert not list(store.blobs_dir.rglob("*.tmp")) @@ -621,7 +630,7 @@ def _no_links(*_args, **_kwargs): def test_failed_temp_write_leaves_no_orphan_tmp_file(trace_dir, monkeypatch): store = _spans._get_store() - real_write_text = Path.write_text + real_write_bytes = Path.write_bytes calls = {"n": 0} def _fail_first_call(self, data, *args, **kwargs): @@ -629,17 +638,17 @@ def _fail_first_call(self, data, *args, **kwargs): if calls["n"] == 1: # Simulate a write that reaches disk before it fails (e.g. ENOSPC # mid-write), so a leftover temp file would prove the bug. - real_write_text(self, data, *args, **kwargs) + real_write_bytes(self, data, *args, **kwargs) raise OSError("simulated disk full") - return real_write_text(self, data, *args, **kwargs) + return real_write_bytes(self, data, *args, **kwargs) - monkeypatch.setattr(Path, "write_text", _fail_first_call) + monkeypatch.setattr(Path, "write_bytes", _fail_first_call) payload = {"m": "diskfull"} art = store.persist_artifact("tool.input", {"traceId": "t1"}, payload) path = Path(art["path"]) - assert json.loads(path.read_text(encoding="utf-8")) == payload + assert json.loads(_stored_text(path)) == payload assert path.stat().st_nlink == 1 assert art.get("error") is None assert not list(store.blobs_dir.rglob("*.tmp")) @@ -652,7 +661,7 @@ def test_string_payloads_keep_the_txt_extension(trace_dir): path = Path(art["path"]) assert path.suffix == ".txt" - assert path.read_text(encoding="utf-8") == "raw text" + assert _stored_text(path) == "raw text" assert (store.blobs_dir / art["sha1"][:2] / f"{art['sha1']}.txt").exists() @@ -781,7 +790,7 @@ class _Resp: assert by["llm.call"]["attributes"]["llm.model"] assert by["tool.call"]["attributes"]["tool.name"] == "grep" output_path = Path(by["llm.call"]["attributes"]["llm.output.artifact_path"]) - assert json.loads(output_path.read_text(encoding="utf-8"))["thinking_blocks"] == _Resp.thinking_blocks + assert json.loads(_stored_text(output_path))["thinking_blocks"] == _Resp.thinking_blocks def test_llm_span_carries_the_request_size_and_the_transport_facts(trace_dir): @@ -820,13 +829,13 @@ def test_llm_span_carries_the_request_size_and_the_transport_facts(trace_dir): assert attrs["llm.request_image_bytes"] == 300 assert attrs["llm.request_bytes"] > 400 - input_payload = json.loads(Path(attrs["llm.input.artifact_path"]).read_text(encoding="utf-8")) + input_payload = json.loads(_stored_text(attrs["llm.input.artifact_path"])) assert input_payload["request"] == { "bytes": attrs["llm.request_bytes"], "images": 1, "imageBytes": 300, } - output_payload = json.loads(Path(attrs["llm.output.artifact_path"]).read_text(encoding="utf-8")) + output_payload = json.loads(_stored_text(attrs["llm.output.artifact_path"])) assert output_payload["call"]["body"] == '{"choices": []}' @@ -1179,7 +1188,7 @@ def test_address_items_repairs_a_mutated_message_blob(trace_dir): store_2 = _store_mod.TraceStore(trace_dir) store_2.address_items([message]) - assert blob.read_text(encoding="utf-8") == v2.message_text(message) + assert _stored_text(blob) == v2.message_text(message) def test_spans_address_items_returns_the_input_when_the_store_fails(trace_dir, monkeypatch): diff --git a/tests/test_tracing_store_compression.py b/tests/test_tracing_store_compression.py new file mode 100644 index 000000000..73d99901d --- /dev/null +++ b/tests/test_tracing_store_compression.py @@ -0,0 +1,215 @@ +"""Compressed artifact storage: one zstd copy per payload, paths unchanged. + +The store keeps every payload under the same path it always used; only the +bytes change. That is what lets an old span's recorded ``artifact_path`` keep +pointing at a readable file after a migration, and why every reader decides by +the frame, never by the file name. +""" + +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path + +import zstandard + +from raven.tracing import artifact_v2 +from raven.tracing.codec import plaintext_size +from raven.tracing.compact import compact +from raven.tracing.compress import compress +from raven.tracing.store import TraceStore +from raven.trajectory import details +from raven.trajectory.conversation import _read_artifact + +MAGIC = b"\x28\xb5\x2f\xfd" +PAYLOAD = {"messages": [{"role": "user", "content": "hi " * 4000}]} + + +def _persist(state: Path, payload: object = PAYLOAD) -> dict: + return TraceStore(state).persist_artifact("llm.input", {"traceId": "t", "sessionKey": "s"}, payload) + + +def _plain(data: bytes) -> bytes: + return zstandard.ZstdDecompressor().decompressobj().decompress(data) + + +def test_persist_artifact_writes_a_compressed_blob_under_the_plain_path(tmp_path: Path) -> None: + artifact = _persist(tmp_path) + path = Path(artifact["path"]) + assert path.suffix == ".json" and not path.name.endswith(".zst") + + data = path.read_bytes() + assert data.startswith(MAGIC) + plain = _plain(data) + assert json.loads(plain) == PAYLOAD + + sha1 = hashlib.sha1(plain).hexdigest() + assert artifact["sha1"] == sha1 + assert artifact["bytes"] == len(plain) + + blob = tmp_path / "logs" / "audit-artifacts" / "_blobs" / sha1[:2] / f"{sha1}.json" + assert blob.read_bytes().startswith(MAGIC) + assert blob.stat().st_ino == path.stat().st_ino + assert plaintext_size(blob) == len(plain) + + +def test_identical_payloads_share_one_blob(tmp_path: Path) -> None: + first = _persist(tmp_path) + second = _persist(tmp_path) + assert Path(first["path"]).stat().st_ino == Path(second["path"]).stat().st_ino + + +def test_the_reader_decompresses_and_still_bounds_what_it_reads(tmp_path: Path) -> None: + artifact = _persist(tmp_path) + path = str(artifact["path"]) + + text, reason = _read_artifact(tmp_path, path) + assert reason is None + assert json.loads(text) == PAYLOAD + + limited, note = _read_artifact(tmp_path, path, 64) + assert limited == text[:64] + assert note is not None and "truncated" in note + + +def test_plain_artifacts_from_before_compression_still_read(tmp_path: Path) -> None: + plain = tmp_path / "logs" / "llm.input" / "2026-01-01" / "old.json" + plain.parent.mkdir(parents=True) + plain.write_text('{"old": true}', encoding="utf-8") + + text, reason = _read_artifact(tmp_path, str(plain)) + + assert reason is None + assert text == '{"old": true}' + + +def test_messages_are_compressed_and_addressable(tmp_path: Path) -> None: + message = {"role": "user", "content": "x" * 500} + store = TraceStore(tmp_path) + + refs = store.address_items([message]) + + sha1 = artifact_v2.message_sha1(message) + assert refs == [{"$msg": sha1}] + path = artifact_v2.message_path(tmp_path / "logs" / "audit-artifacts", sha1) + assert path.read_bytes().startswith(MAGIC) + text, reason = _read_artifact(tmp_path, str(path)) + assert reason is None and json.loads(text) == message + + +def test_compact_reads_compressed_artifacts(tmp_path: Path) -> None: + artifact = _persist(tmp_path) + artifacts = tmp_path / "logs" / "audit-artifacts" + + result = compact(artifacts) + + assert result.errors == [] + assert result.scanned == 1 + assert result.folded == 0 + assert Path(artifact["path"]).read_bytes().startswith(MAGIC) + + +def test_compact_folds_a_second_copy_of_the_same_payload(tmp_path: Path) -> None: + artifact = _persist(tmp_path) + source = Path(artifact["path"]) + copy = source.parent / "copy.json" + copy.write_bytes(source.read_bytes()) + os.utime(copy, (0, 0)) + + result = compact(tmp_path / "logs" / "audit-artifacts") + + assert result.folded == 1 + assert copy.stat().st_ino == source.stat().st_ino + + +def _plant_plain_store(state: Path) -> tuple[Path, Path, str]: + """A pre-compression store: a plain blob, its artifact link, and a message.""" + text = '{"legacy": "payload"}' + sha1 = hashlib.sha1(text.encode("utf-8")).hexdigest() + artifacts = state / "logs" / "audit-artifacts" + blob = artifacts / "_blobs" / sha1[:2] / f"{sha1}.json" + blob.parent.mkdir(parents=True) + blob.write_text(text, encoding="utf-8") + artifact = artifacts / "llm.input" / "2026-01-01" / "old.json" + artifact.parent.mkdir(parents=True) + os.link(blob, artifact) + message = {"role": "user", "content": "m" * 300} + msha = artifact_v2.message_sha1(message) + mpath = artifact_v2.message_path(artifacts, msha) + mpath.parent.mkdir(parents=True) + mpath.write_text(artifact_v2.message_text(message), encoding="utf-8") + return artifact, mpath, sha1 + + +def test_compress_migrates_plain_blobs_and_keeps_every_path(tmp_path: Path) -> None: + artifact, message_path, sha1 = _plant_plain_store(tmp_path) + artifacts = tmp_path / "logs" / "audit-artifacts" + + result = compress(artifacts) + + assert result.compressed == 1 + assert result.errors == [] + assert artifact.read_bytes().startswith(MAGIC) + assert message_path.read_bytes().startswith(MAGIC) + blob = artifacts / "_blobs" / sha1[:2] / f"{sha1}.json" + assert artifact.stat().st_ino == blob.stat().st_ino + text, reason = _read_artifact(tmp_path, str(artifact)) + assert reason is None and text == '{"legacy": "payload"}' + text, reason = _read_artifact(tmp_path, str(message_path)) + assert reason is None and json.loads(text)["content"] == "m" * 300 + + +def test_compress_is_idempotent_and_dry_run_writes_nothing(tmp_path: Path) -> None: + artifact, message_path, _ = _plant_plain_store(tmp_path) + artifacts = tmp_path / "logs" / "audit-artifacts" + before = artifact.read_bytes() + + dry = compress(artifacts, dry_run=True) + assert dry.compressed == 1 + assert artifact.read_bytes() == before + + once = compress(artifacts) + assert once.compressed == 1 + again = compress(artifacts) + assert again.compressed == 0 + assert again.already >= 1 + assert artifact.read_bytes().startswith(MAGIC) + assert message_path.read_bytes().startswith(MAGIC) + + +def test_blob_prefix_reports_the_plaintext_size(tmp_path: Path) -> None: + message = {"role": "user", "content": "x" * 1000} + TraceStore(tmp_path).address_items([message]) + sha1 = artifact_v2.message_sha1(message) + plain = artifact_v2.message_text(message) + reader = details._Reader(tmp_path, max_artifacts=None, max_blobs=None, max_bytes=None) + + text, size = reader.blob_prefix(sha1, 10) + + assert text == plain[:10] + assert size == len(plain.encode("utf-8")) + + +def test_a_corrupt_frame_is_reported_not_raised(tmp_path: Path) -> None: + bad = tmp_path / "logs" / "llm.input" / "2026-01-01" / "bad.json" + bad.parent.mkdir(parents=True) + bad.write_bytes(MAGIC + b" not a frame") + + text, reason = _read_artifact(tmp_path, str(bad)) + + assert text is None + assert reason is not None and "corrupt" in reason + + +def test_compact_records_a_corrupt_frame(tmp_path: Path) -> None: + bad = tmp_path / "logs" / "audit-artifacts" / "llm.input" / "2026-01-01" / "bad.json" + bad.parent.mkdir(parents=True) + bad.write_bytes(MAGIC + b" not a frame") + os.utime(bad, (0, 0)) + + result = compact(tmp_path / "logs" / "audit-artifacts") + + assert result.errors + assert bad.exists() diff --git a/tests/test_trajectory_conversation.py b/tests/test_trajectory_conversation.py index 9f42b76a5..0d5231169 100644 --- a/tests/test_trajectory_conversation.py +++ b/tests/test_trajectory_conversation.py @@ -685,6 +685,9 @@ def read(self, n=-1): reads.append(n) return self._handle.read(n) + def __getattr__(self, name): + return getattr(self._handle, name) + def __enter__(self): return self @@ -699,7 +702,10 @@ def spy_open(self, *args, **kwargs): text, degraded = tconv._read_artifact(state, str(big)) assert degraded == "content over 512 KiB — truncated" assert len(text) == tconv._ARTIFACT_LIMIT - assert reads == [tconv._ARTIFACT_LIMIT + 1] + # The reader sniffs the first four bytes for a compressed frame, then reads + # the bounded window; no read may exceed that window. + assert max(reads) == tconv._ARTIFACT_LIMIT + 1 + assert reads.count(tconv._ARTIFACT_LIMIT + 1) == 1 fifo = state / "logs" / "audit-artifacts" / "pipe" os.mkfifo(fifo) diff --git a/tests/test_utils_mirrors.py b/tests/test_utils_mirrors.py index 14916d71c..22ef5a1d1 100644 --- a/tests/test_utils_mirrors.py +++ b/tests/test_utils_mirrors.py @@ -165,8 +165,8 @@ def test_a_proxys_refusal_of_the_tunnel_answers_for_the_host(self, error: BaseEx assert mirrors.unreachable(error) is reached def test_the_question_needs_no_httpx(self, monkeypatch: pytest.MonkeyPatch) -> None: - """A Raven node runs with loguru and pillow and nothing else, and starts - agents through npx all the same.""" + """A Raven node runs with loguru, pillow and zstandard and nothing else, + and starts agents through npx all the same.""" monkeypatch.setitem(sys.modules, "httpx", None) assert mirrors.unreachable(urllib.error.URLError(ConnectionRefusedError(111, "connection refused"))) @@ -1402,7 +1402,7 @@ def test_a_refused_connection_is_not(self) -> None: assert not mirrors._reachable(f"http://127.0.0.1:{_closed_port()}/") def test_the_probe_needs_no_httpx(self, monkeypatch: pytest.MonkeyPatch, answering_host: str) -> None: - """A Raven node has loguru and pillow, and starts npx agents too.""" + """A Raven node has loguru, pillow and zstandard, and starts npx agents too.""" monkeypatch.setitem(sys.modules, "httpx", None) assert mirrors._reachable(answering_host) diff --git a/uv.lock b/uv.lock index 762155570..eb9c681d1 100644 --- a/uv.lock +++ b/uv.lock @@ -4254,6 +4254,7 @@ dependencies = [ { name = "truststore" }, { name = "typer" }, { name = "watchfiles" }, + { name = "zstandard" }, ] [package.optional-dependencies] @@ -4459,6 +4460,7 @@ requires-dist = [ { name = "websockets", marker = "extra == 'channel-discord'", specifier = ">=16.0,<17.0" }, { name = "websockets", marker = "extra == 'channel-whatsapp'", specifier = ">=16.0,<17.0" }, { name = "wecom-aibot-sdk-python", marker = "extra == 'channel-wecom'", specifier = ">=0.1.5" }, + { name = "zstandard", specifier = ">=0.25.0" }, ] provides-extras = ["channel-telegram", "channel-slack", "channel-discord", "channel-whatsapp", "channel-matrix", "channel-feishu", "channel-wecom", "channel-mochat", "channel-qq", "channel-dingtalk", "channel-email", "channel-weixin", "channels", "tools", "sandbox", "browser", "dev", "eval", "segmenter"] @@ -5807,3 +5809,60 @@ sdist = { url = "https://files.pythonhosted.org/packages/30/21/093488dfc7cc8964d wheels = [ { url = "https://files.pythonhosted.org/packages/08/8a/0861bec20485572fbddf3dfba2910e38fe249796cb73ecdeb74e07eeb8d3/zipp-3.23.1-py3-none-any.whl", hash = "sha256:0b3596c50a5c700c9cb40ba8d86d9f2cc4807e9bedb06bcdf7fac85633e444dc", size = 10378, upload-time = "2026-04-13T23:21:45.386Z" }, ] + +[[package]] +name = "zstandard" +version = "0.25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/aa/3e0508d5a5dd96529cdc5a97011299056e14c6505b678fd58938792794b1/zstandard-0.25.0.tar.gz", hash = "sha256:7713e1179d162cf5c7906da876ec2ccb9c3a9dcbdffef0cc7f70c3667a205f0b", size = 711513, upload-time = "2025-09-14T22:15:54.002Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/fc/f26eb6ef91ae723a03e16eddb198abcfce2bc5a42e224d44cc8b6765e57e/zstandard-0.25.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7b3c3a3ab9daa3eed242d6ecceead93aebbb8f5f84318d82cee643e019c4b73b", size = 795738, upload-time = "2025-09-14T22:16:56.237Z" }, + { url = "https://files.pythonhosted.org/packages/aa/1c/d920d64b22f8dd028a8b90e2d756e431a5d86194caa78e3819c7bf53b4b3/zstandard-0.25.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:913cbd31a400febff93b564a23e17c3ed2d56c064006f54efec210d586171c00", size = 640436, upload-time = "2025-09-14T22:16:57.774Z" }, + { url = "https://files.pythonhosted.org/packages/53/6c/288c3f0bd9fcfe9ca41e2c2fbfd17b2097f6af57b62a81161941f09afa76/zstandard-0.25.0-cp312-cp312-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:011d388c76b11a0c165374ce660ce2c8efa8e5d87f34996aa80f9c0816698b64", size = 5343019, upload-time = "2025-09-14T22:16:59.302Z" }, + { url = "https://files.pythonhosted.org/packages/1e/15/efef5a2f204a64bdb5571e6161d49f7ef0fffdbca953a615efbec045f60f/zstandard-0.25.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dffecc361d079bb48d7caef5d673c88c8988d3d33fb74ab95b7ee6da42652ea", size = 5063012, upload-time = "2025-09-14T22:17:01.156Z" }, + { url = "https://files.pythonhosted.org/packages/b7/37/a6ce629ffdb43959e92e87ebdaeebb5ac81c944b6a75c9c47e300f85abdf/zstandard-0.25.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:7149623bba7fdf7e7f24312953bcf73cae103db8cae49f8154dd1eadc8a29ecb", size = 5394148, upload-time = "2025-09-14T22:17:03.091Z" }, + { url = "https://files.pythonhosted.org/packages/e3/79/2bf870b3abeb5c070fe2d670a5a8d1057a8270f125ef7676d29ea900f496/zstandard-0.25.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:6a573a35693e03cf1d67799fd01b50ff578515a8aeadd4595d2a7fa9f3ec002a", size = 5451652, upload-time = "2025-09-14T22:17:04.979Z" }, + { url = "https://files.pythonhosted.org/packages/53/60/7be26e610767316c028a2cbedb9a3beabdbe33e2182c373f71a1c0b88f36/zstandard-0.25.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5a56ba0db2d244117ed744dfa8f6f5b366e14148e00de44723413b2f3938a902", size = 5546993, upload-time = "2025-09-14T22:17:06.781Z" }, + { url = "https://files.pythonhosted.org/packages/85/c7/3483ad9ff0662623f3648479b0380d2de5510abf00990468c286c6b04017/zstandard-0.25.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:10ef2a79ab8e2974e2075fb984e5b9806c64134810fac21576f0668e7ea19f8f", size = 5046806, upload-time = "2025-09-14T22:17:08.415Z" }, + { url = "https://files.pythonhosted.org/packages/08/b3/206883dd25b8d1591a1caa44b54c2aad84badccf2f1de9e2d60a446f9a25/zstandard-0.25.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:aaf21ba8fb76d102b696781bddaa0954b782536446083ae3fdaa6f16b25a1c4b", size = 5576659, upload-time = "2025-09-14T22:17:10.164Z" }, + { url = "https://files.pythonhosted.org/packages/9d/31/76c0779101453e6c117b0ff22565865c54f48f8bd807df2b00c2c404b8e0/zstandard-0.25.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1869da9571d5e94a85a5e8d57e4e8807b175c9e4a6294e3b66fa4efb074d90f6", size = 4953933, upload-time = "2025-09-14T22:17:11.857Z" }, + { url = "https://files.pythonhosted.org/packages/18/e1/97680c664a1bf9a247a280a053d98e251424af51f1b196c6d52f117c9720/zstandard-0.25.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:809c5bcb2c67cd0ed81e9229d227d4ca28f82d0f778fc5fea624a9def3963f91", size = 5268008, upload-time = "2025-09-14T22:17:13.627Z" }, + { url = "https://files.pythonhosted.org/packages/1e/73/316e4010de585ac798e154e88fd81bb16afc5c5cb1a72eeb16dd37e8024a/zstandard-0.25.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:f27662e4f7dbf9f9c12391cb37b4c4c3cb90ffbd3b1fb9284dadbbb8935fa708", size = 5433517, upload-time = "2025-09-14T22:17:16.103Z" }, + { url = "https://files.pythonhosted.org/packages/5b/60/dd0f8cfa8129c5a0ce3ea6b7f70be5b33d2618013a161e1ff26c2b39787c/zstandard-0.25.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:99c0c846e6e61718715a3c9437ccc625de26593fea60189567f0118dc9db7512", size = 5814292, upload-time = "2025-09-14T22:17:17.827Z" }, + { url = "https://files.pythonhosted.org/packages/fc/5f/75aafd4b9d11b5407b641b8e41a57864097663699f23e9ad4dbb91dc6bfe/zstandard-0.25.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:474d2596a2dbc241a556e965fb76002c1ce655445e4e3bf38e5477d413165ffa", size = 5360237, upload-time = "2025-09-14T22:17:19.954Z" }, + { url = "https://files.pythonhosted.org/packages/ff/8d/0309daffea4fcac7981021dbf21cdb2e3427a9e76bafbcdbdf5392ff99a4/zstandard-0.25.0-cp312-cp312-win32.whl", hash = "sha256:23ebc8f17a03133b4426bcc04aabd68f8236eb78c3760f12783385171b0fd8bd", size = 436922, upload-time = "2025-09-14T22:17:24.398Z" }, + { url = "https://files.pythonhosted.org/packages/79/3b/fa54d9015f945330510cb5d0b0501e8253c127cca7ebe8ba46a965df18c5/zstandard-0.25.0-cp312-cp312-win_amd64.whl", hash = "sha256:ffef5a74088f1e09947aecf91011136665152e0b4b359c42be3373897fb39b01", size = 506276, upload-time = "2025-09-14T22:17:21.429Z" }, + { url = "https://files.pythonhosted.org/packages/ea/6b/8b51697e5319b1f9ac71087b0af9a40d8a6288ff8025c36486e0c12abcc4/zstandard-0.25.0-cp312-cp312-win_arm64.whl", hash = "sha256:181eb40e0b6a29b3cd2849f825e0fa34397f649170673d385f3598ae17cca2e9", size = 462679, upload-time = "2025-09-14T22:17:23.147Z" }, + { url = "https://files.pythonhosted.org/packages/35/0b/8df9c4ad06af91d39e94fa96cc010a24ac4ef1378d3efab9223cc8593d40/zstandard-0.25.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ec996f12524f88e151c339688c3897194821d7f03081ab35d31d1e12ec975e94", size = 795735, upload-time = "2025-09-14T22:17:26.042Z" }, + { url = "https://files.pythonhosted.org/packages/3f/06/9ae96a3e5dcfd119377ba33d4c42a7d89da1efabd5cb3e366b156c45ff4d/zstandard-0.25.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a1a4ae2dec3993a32247995bdfe367fc3266da832d82f8438c8570f989753de1", size = 640440, upload-time = "2025-09-14T22:17:27.366Z" }, + { url = "https://files.pythonhosted.org/packages/d9/14/933d27204c2bd404229c69f445862454dcc101cd69ef8c6068f15aaec12c/zstandard-0.25.0-cp313-cp313-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:e96594a5537722fdfb79951672a2a63aec5ebfb823e7560586f7484819f2a08f", size = 5343070, upload-time = "2025-09-14T22:17:28.896Z" }, + { url = "https://files.pythonhosted.org/packages/6d/db/ddb11011826ed7db9d0e485d13df79b58586bfdec56e5c84a928a9a78c1c/zstandard-0.25.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bfc4e20784722098822e3eee42b8e576b379ed72cca4a7cb856ae733e62192ea", size = 5063001, upload-time = "2025-09-14T22:17:31.044Z" }, + { url = "https://files.pythonhosted.org/packages/db/00/87466ea3f99599d02a5238498b87bf84a6348290c19571051839ca943777/zstandard-0.25.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:457ed498fc58cdc12fc48f7950e02740d4f7ae9493dd4ab2168a47c93c31298e", size = 5394120, upload-time = "2025-09-14T22:17:32.711Z" }, + { url = "https://files.pythonhosted.org/packages/2b/95/fc5531d9c618a679a20ff6c29e2b3ef1d1f4ad66c5e161ae6ff847d102a9/zstandard-0.25.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:fd7a5004eb1980d3cefe26b2685bcb0b17989901a70a1040d1ac86f1d898c551", size = 5451230, upload-time = "2025-09-14T22:17:34.41Z" }, + { url = "https://files.pythonhosted.org/packages/63/4b/e3678b4e776db00f9f7b2fe58e547e8928ef32727d7a1ff01dea010f3f13/zstandard-0.25.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8e735494da3db08694d26480f1493ad2cf86e99bdd53e8e9771b2752a5c0246a", size = 5547173, upload-time = "2025-09-14T22:17:36.084Z" }, + { url = "https://files.pythonhosted.org/packages/4e/d5/ba05ed95c6b8ec30bd468dfeab20589f2cf709b5c940483e31d991f2ca58/zstandard-0.25.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3a39c94ad7866160a4a46d772e43311a743c316942037671beb264e395bdd611", size = 5046736, upload-time = "2025-09-14T22:17:37.891Z" }, + { url = "https://files.pythonhosted.org/packages/50/d5/870aa06b3a76c73eced65c044b92286a3c4e00554005ff51962deef28e28/zstandard-0.25.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:172de1f06947577d3a3005416977cce6168f2261284c02080e7ad0185faeced3", size = 5576368, upload-time = "2025-09-14T22:17:40.206Z" }, + { url = "https://files.pythonhosted.org/packages/5d/35/398dc2ffc89d304d59bc12f0fdd931b4ce455bddf7038a0a67733a25f550/zstandard-0.25.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3c83b0188c852a47cd13ef3bf9209fb0a77fa5374958b8c53aaa699398c6bd7b", size = 4954022, upload-time = "2025-09-14T22:17:41.879Z" }, + { url = "https://files.pythonhosted.org/packages/9a/5c/36ba1e5507d56d2213202ec2b05e8541734af5f2ce378c5d1ceaf4d88dc4/zstandard-0.25.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:1673b7199bbe763365b81a4f3252b8e80f44c9e323fc42940dc8843bfeaf9851", size = 5267889, upload-time = "2025-09-14T22:17:43.577Z" }, + { url = "https://files.pythonhosted.org/packages/70/e8/2ec6b6fb7358b2ec0113ae202647ca7c0e9d15b61c005ae5225ad0995df5/zstandard-0.25.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:0be7622c37c183406f3dbf0cba104118eb16a4ea7359eeb5752f0794882fc250", size = 5433952, upload-time = "2025-09-14T22:17:45.271Z" }, + { url = "https://files.pythonhosted.org/packages/7b/01/b5f4d4dbc59ef193e870495c6f1275f5b2928e01ff5a81fecb22a06e22fb/zstandard-0.25.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:5f5e4c2a23ca271c218ac025bd7d635597048b366d6f31f420aaeb715239fc98", size = 5814054, upload-time = "2025-09-14T22:17:47.08Z" }, + { url = "https://files.pythonhosted.org/packages/b2/e5/fbd822d5c6f427cf158316d012c5a12f233473c2f9c5fe5ab1ae5d21f3d8/zstandard-0.25.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4f187a0bb61b35119d1926aee039524d1f93aaf38a9916b8c4b78ac8514a0aaf", size = 5360113, upload-time = "2025-09-14T22:17:48.893Z" }, + { url = "https://files.pythonhosted.org/packages/8e/e0/69a553d2047f9a2c7347caa225bb3a63b6d7704ad74610cb7823baa08ed7/zstandard-0.25.0-cp313-cp313-win32.whl", hash = "sha256:7030defa83eef3e51ff26f0b7bfb229f0204b66fe18e04359ce3474ac33cbc09", size = 436936, upload-time = "2025-09-14T22:17:52.658Z" }, + { url = "https://files.pythonhosted.org/packages/d9/82/b9c06c870f3bd8767c201f1edbdf9e8dc34be5b0fbc5682c4f80fe948475/zstandard-0.25.0-cp313-cp313-win_amd64.whl", hash = "sha256:1f830a0dac88719af0ae43b8b2d6aef487d437036468ef3c2ea59c51f9d55fd5", size = 506232, upload-time = "2025-09-14T22:17:50.402Z" }, + { url = "https://files.pythonhosted.org/packages/d4/57/60c3c01243bb81d381c9916e2a6d9e149ab8627c0c7d7abb2d73384b3c0c/zstandard-0.25.0-cp313-cp313-win_arm64.whl", hash = "sha256:85304a43f4d513f5464ceb938aa02c1e78c2943b29f44a750b48b25ac999a049", size = 462671, upload-time = "2025-09-14T22:17:51.533Z" }, + { url = "https://files.pythonhosted.org/packages/3d/5c/f8923b595b55fe49e30612987ad8bf053aef555c14f05bb659dd5dbe3e8a/zstandard-0.25.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:e29f0cf06974c899b2c188ef7f783607dbef36da4c242eb6c82dcd8b512855e3", size = 795887, upload-time = "2025-09-14T22:17:54.198Z" }, + { url = "https://files.pythonhosted.org/packages/8d/09/d0a2a14fc3439c5f874042dca72a79c70a532090b7ba0003be73fee37ae2/zstandard-0.25.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:05df5136bc5a011f33cd25bc9f506e7426c0c9b3f9954f056831ce68f3b6689f", size = 640658, upload-time = "2025-09-14T22:17:55.423Z" }, + { url = "https://files.pythonhosted.org/packages/5d/7c/8b6b71b1ddd517f68ffb55e10834388d4f793c49c6b83effaaa05785b0b4/zstandard-0.25.0-cp314-cp314-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:f604efd28f239cc21b3adb53eb061e2a205dc164be408e553b41ba2ffe0ca15c", size = 5379849, upload-time = "2025-09-14T22:17:57.372Z" }, + { url = "https://files.pythonhosted.org/packages/a4/86/a48e56320d0a17189ab7a42645387334fba2200e904ee47fc5a26c1fd8ca/zstandard-0.25.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:223415140608d0f0da010499eaa8ccdb9af210a543fac54bce15babbcfc78439", size = 5058095, upload-time = "2025-09-14T22:17:59.498Z" }, + { url = "https://files.pythonhosted.org/packages/f8/ad/eb659984ee2c0a779f9d06dbfe45e2dc39d99ff40a319895df2d3d9a48e5/zstandard-0.25.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2e54296a283f3ab5a26fc9b8b5d4978ea0532f37b231644f367aa588930aa043", size = 5551751, upload-time = "2025-09-14T22:18:01.618Z" }, + { url = "https://files.pythonhosted.org/packages/61/b3/b637faea43677eb7bd42ab204dfb7053bd5c4582bfe6b1baefa80ac0c47b/zstandard-0.25.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ca54090275939dc8ec5dea2d2afb400e0f83444b2fc24e07df7fdef677110859", size = 6364818, upload-time = "2025-09-14T22:18:03.769Z" }, + { url = "https://files.pythonhosted.org/packages/31/dc/cc50210e11e465c975462439a492516a73300ab8caa8f5e0902544fd748b/zstandard-0.25.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e09bb6252b6476d8d56100e8147b803befa9a12cea144bbe629dd508800d1ad0", size = 5560402, upload-time = "2025-09-14T22:18:05.954Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ae/56523ae9c142f0c08efd5e868a6da613ae76614eca1305259c3bf6a0ed43/zstandard-0.25.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a9ec8c642d1ec73287ae3e726792dd86c96f5681eb8df274a757bf62b750eae7", size = 4955108, upload-time = "2025-09-14T22:18:07.68Z" }, + { url = "https://files.pythonhosted.org/packages/98/cf/c899f2d6df0840d5e384cf4c4121458c72802e8bda19691f3b16619f51e9/zstandard-0.25.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:a4089a10e598eae6393756b036e0f419e8c1d60f44a831520f9af41c14216cf2", size = 5269248, upload-time = "2025-09-14T22:18:09.753Z" }, + { url = "https://files.pythonhosted.org/packages/1b/c0/59e912a531d91e1c192d3085fc0f6fb2852753c301a812d856d857ea03c6/zstandard-0.25.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f67e8f1a324a900e75b5e28ffb152bcac9fbed1cc7b43f99cd90f395c4375344", size = 5430330, upload-time = "2025-09-14T22:18:11.966Z" }, + { url = "https://files.pythonhosted.org/packages/a0/1d/7e31db1240de2df22a58e2ea9a93fc6e38cc29353e660c0272b6735d6669/zstandard-0.25.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:9654dbc012d8b06fc3d19cc825af3f7bf8ae242226df5f83936cb39f5fdc846c", size = 5811123, upload-time = "2025-09-14T22:18:13.907Z" }, + { url = "https://files.pythonhosted.org/packages/f6/49/fac46df5ad353d50535e118d6983069df68ca5908d4d65b8c466150a4ff1/zstandard-0.25.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4203ce3b31aec23012d3a4cf4a2ed64d12fea5269c49aed5e4c3611b938e4088", size = 5359591, upload-time = "2025-09-14T22:18:16.465Z" }, + { url = "https://files.pythonhosted.org/packages/c2/38/f249a2050ad1eea0bb364046153942e34abba95dd5520af199aed86fbb49/zstandard-0.25.0-cp314-cp314-win32.whl", hash = "sha256:da469dc041701583e34de852d8634703550348d5822e66a0c827d39b05365b12", size = 444513, upload-time = "2025-09-14T22:18:20.61Z" }, + { url = "https://files.pythonhosted.org/packages/3a/43/241f9615bcf8ba8903b3f0432da069e857fc4fd1783bd26183db53c4804b/zstandard-0.25.0-cp314-cp314-win_amd64.whl", hash = "sha256:c19bcdd826e95671065f8692b5a4aa95c52dc7a02a4c5a0cac46deb879a017a2", size = 516118, upload-time = "2025-09-14T22:18:17.849Z" }, + { url = "https://files.pythonhosted.org/packages/f0/ef/da163ce2450ed4febf6467d77ccb4cd52c4c30ab45624bad26ca0a27260c/zstandard-0.25.0-cp314-cp314-win_arm64.whl", hash = "sha256:d7541afd73985c630bafcd6338d2518ae96060075f9463d7dc14cfb33514383d", size = 476940, upload-time = "2025-09-14T22:18:19.088Z" }, +] From b68d7d31885a4300c024aae2f7f5f9750c121330 Mon Sep 17 00:00:00 2001 From: Dizhan Xue Date: Thu, 29 Oct 2026 14:31:22 +0000 Subject: [PATCH 02/23] feat(tracing): compress rotated span and event logs Rotation is the one moment a log is closed for good, so the rotated file is rewritten as a zstd frame under the name it already has. Readers of the span log chain decode the frame and keep working in plaintext byte offsets, which is the only coordinate a log cursor and the shard index record. `raven tracing compress --logs` migrates archived logs in place. ACP frame journals stay plaintext: a span records a plaintext byte range into one, and re-encoding it would invalidate that audit record. --- docs-site/docs/commands.md | 1 + docs-site/docs/commands.zh.md | 1 + raven/cli/tracing_commands.py | 52 +++++++----- raven/cli/tracing_viewer/log-store.js | 30 ++++++- raven/cli/tracing_viewer/server.js | 29 +------ raven/tracing/codec.py | 51 +++++++++++- raven/tracing/compress.py | 68 ++++++++++++++- raven/tracing/store.py | 22 +++++ raven/trajectory/index.py | 54 +++++++++--- raven/trajectory/selection.py | 7 +- raven/trajectory/store.py | 7 +- tests/integration/test_tracing_viewer_e2e.py | 13 +++ tests/test_cli_tracing_commands.py | 13 +++ tests/test_tracing_store_compression.py | 87 +++++++++++++++++++- tests/test_trajectory_index.py | 28 +++++++ 15 files changed, 389 insertions(+), 74 deletions(-) diff --git a/docs-site/docs/commands.md b/docs-site/docs/commands.md index b4c3b279e..67757668e 100644 --- a/docs-site/docs/commands.md +++ b/docs-site/docs/commands.md @@ -36,6 +36,7 @@ lists its subcommands and options. | `raven sandbox` | List sandbox VMs, run commands, or open a shell; requires `tools.sandbox.debug.enabled=true` | | `raven tracing` | Open the local trace dashboard | | `raven tracing compact` | Consolidate duplicate trace artifacts to free disk space | +| `raven tracing compress [--logs]` | Rewrite trace payloads as zstd (with `--logs`, the rotated span and event logs too) to free disk space | | `raven trajectory` | Save, replay, redact, label, and preserve execution trajectories for debugging, and run the bug report receiver | Run `raven --help` for the command list, or `raven --help` for details diff --git a/docs-site/docs/commands.zh.md b/docs-site/docs/commands.zh.md index 90440dee0..1671bed25 100644 --- a/docs-site/docs/commands.zh.md +++ b/docs-site/docs/commands.zh.md @@ -35,6 +35,7 @@ | `raven sandbox` | 列出沙箱虚拟机、执行命令或打开 Shell；需要设置 `tools.sandbox.debug.enabled=true` | | `raven tracing` | 打开本地追踪面板 | | `raven tracing compact` | 合并重复的追踪产物，回收磁盘空间 | +| `raven tracing compress [--logs]` | 以 zstd 重写追踪载荷（加 `--logs` 时还包括轮转后的日志），回收磁盘空间 | | `raven trajectory` | 保存、回放、脱敏、标注和保留执行轨迹，用于调试；也运行问题报告接收器 | 运行 `raven --help` 查看命令列表，或运行 `raven --help` 查看指定命令的详细说明。 diff --git a/raven/cli/tracing_commands.py b/raven/cli/tracing_commands.py index e276b6fad..73fcea15f 100644 --- a/raven/cli/tracing_commands.py +++ b/raven/cli/tracing_commands.py @@ -306,28 +306,41 @@ def _run_compact(*, dry_run: bool) -> None: console.print(f"[yellow]... and {hidden} more[/yellow]") -def _run_compress(*, dry_run: bool) -> None: - from raven.tracing.compress import compress +def _run_compress(*, dry_run: bool, logs: bool = False) -> None: + from raven.tracing.compress import compress, compress_logs from raven.tracing.store import TraceStore - artifacts_dir = TraceStore(tracing_config.state_dir()).artifacts_dir - if not artifacts_dir.is_dir(): - console.print(f"[yellow]No artifacts at {escape(str(artifacts_dir))}.[/yellow]") - return - result = compress(artifacts_dir, dry_run=dry_run) + store = TraceStore(tracing_config.state_dir()) if dry_run: console.print("[dim]dry run - nothing is written[/dim]") - console.print( - f"blobs {result.blobs}: compressed {result.compressed}, already {result.already}, relinked {result.relinked}" - ) - console.print(f"messages {result.messages}: compressed {result.messages_compressed}") - console.print( - f"{result.bytes_before / (1024 * 1024):.1f} MB -> {result.bytes_after / (1024 * 1024):.1f} MB, " - f"saved {result.saved / (1024 * 1024):.1f} MB" - ) - for message in result.errors[:10]: + artifacts_dir = store.artifacts_dir + if artifacts_dir.is_dir(): + result = compress(artifacts_dir, dry_run=dry_run) + console.print( + f"blobs {result.blobs}: compressed {result.compressed}, already {result.already}, relinked {result.relinked}" + ) + console.print(f"messages {result.messages}: compressed {result.messages_compressed}") + console.print( + f"{result.bytes_before / (1024 * 1024):.1f} MB -> {result.bytes_after / (1024 * 1024):.1f} MB, " + f"saved {result.saved / (1024 * 1024):.1f} MB" + ) + _print_skipped(result.errors) + else: + console.print(f"[yellow]No artifacts at {escape(str(artifacts_dir))}.[/yellow]") + if logs: + log_result = compress_logs(store.logs_dir, dry_run=dry_run) + console.print(f"logs {log_result.files}: compressed {log_result.compressed}, already {log_result.already}") + console.print( + f"{log_result.bytes_before / (1024 * 1024):.1f} MB -> {log_result.bytes_after / (1024 * 1024):.1f} MB, " + f"saved {log_result.saved / (1024 * 1024):.1f} MB" + ) + _print_skipped(log_result.errors) + + +def _print_skipped(errors: list[str]) -> None: + for message in errors[:10]: console.print(f"[yellow]skipped:[/yellow] {escape(message)}") - hidden = max(len(result.errors) - 10, 0) + hidden = max(len(errors) - 10, 0) if hidden: console.print(f"[yellow]... and {hidden} more[/yellow]") @@ -347,7 +360,7 @@ def tracing( help=( "Optional action: 'stop' shuts down the background viewer; " "'compact' folds duplicate artifacts; 'compress' rewrites payloads as zstd " - "(both terminal only, not available from the TUI)." + "(--logs adds the rotated span and event logs; both terminal only, not available from the TUI)." ), ), port: int = typer.Option(None, "--port", "-p", help="Port to bind (default: config or 4318)."), @@ -357,13 +370,14 @@ def tracing( dry_run: bool = typer.Option( False, "--dry-run", help="With 'compact' or 'compress': report what would change without writing." ), + logs: bool = typer.Option(False, "--logs", help="With 'compress': also compress rotated span and event logs."), ) -> None: """Open the tracing dashboard (captured LLM/tool/memory spans).""" if action == "compact": _run_compact(dry_run=dry_run) return if action == "compress": - _run_compress(dry_run=dry_run) + _run_compress(dry_run=dry_run, logs=logs) return if action == "stop": _stop_viewer() diff --git a/raven/cli/tracing_viewer/log-store.js b/raven/cli/tracing_viewer/log-store.js index 8777152fd..7a4988178 100644 --- a/raven/cli/tracing_viewer/log-store.js +++ b/raven/cli/tracing_viewer/log-store.js @@ -1,8 +1,33 @@ const fs = require('fs'); const os = require('os'); const path = require('path'); +const zlib = require('zlib'); const DEFAULT_MAX_BYTES = 50 * 1024 * 1024; +const ZSTD_MAGIC = Buffer.from([0x28, 0xb5, 0x2f, 0xfd]); + +function readStored(filePath) { + const data = fs.readFileSync(filePath); + if (data.length >= 4 && data.subarray(0, 4).equals(ZSTD_MAGIC)) { + // zstd landed in Node's zlib in 22.15 / 23.8. An older viewer can still + // read a plain store; a compressed one is reported rather than parsed as + // mojibake, because JSON.parse of the frame would only say "invalid". + if (typeof zlib.zstdDecompressSync !== 'function') { + console.warn('tracing viewer: this Node has no zstd support; compressed files need Node >= 22.15'); + return Buffer.alloc(0); + } + try { + return zlib.zstdDecompressSync(data); + } catch { + return data; + } + } + return data; +} + +function readStoredText(filePath) { + return readStored(filePath).toString('utf8'); +} const KIND_FILES = { events: 'audit-events.log', @@ -65,8 +90,7 @@ function statOrNull(filePath) { function readJsonlFile(filePath) { if (!fs.existsSync(filePath)) return []; - return fs - .readFileSync(filePath, 'utf8') + return readStoredText(filePath) .split('\n') .map((line) => line.trim()) .filter(Boolean) @@ -199,6 +223,8 @@ module.exports = { ensureDir, readJsonl, readJsonlFile, + readStored, + readStoredText, statLogFiles, appendJsonl, rotateIfNeeded, diff --git a/raven/cli/tracing_viewer/server.js b/raven/cli/tracing_viewer/server.js index a487d111c..2c23886c3 100644 --- a/raven/cli/tracing_viewer/server.js +++ b/raven/cli/tracing_viewer/server.js @@ -4,11 +4,10 @@ const http = require('http'); const fs = require('fs'); const path = require('path'); const os = require('os'); -const zlib = require('zlib'); const { URL } = require('url'); const { applyStateDirArg } = require('./state-dir'); applyStateDirArg(); -const { readJsonl, readJsonlFile, statLogFiles, getLogsDir } = require('./log-store'); +const { readJsonl, readJsonlFile, readStored, readStoredText, statLogFiles, getLogsDir } = require('./log-store'); const everosDeposits = require('./everos-deposits'); const shardIndex = require('./shard-index'); @@ -19,33 +18,7 @@ const STATE_DIR = path.join(os.homedir(), '.openclaw'); const LOGS_DIR = getLogsDir(); const ARTIFACTS_DIR = path.join(LOGS_DIR, 'audit-artifacts'); -const ZSTD_MAGIC = Buffer.from([0x28, 0xb5, 0x2f, 0xfd]); - -// Artifacts and message blobs are stored zstd-compressed under the same paths -// they always had; the frame, never the name, says which encoding a file -// carries. A plain file written before compression reads unchanged. -function readStored(file) { - const data = fs.readFileSync(file); - if (data.length >= 4 && data.subarray(0, 4).equals(ZSTD_MAGIC)) { - // zstd landed in Node's zlib in 22.15 / 23.8. An older viewer can still - // read a plain store; a compressed one is reported rather than shown as - // mojibake, because JSON.parse of the frame would only say "invalid". - if (typeof zlib.zstdDecompressSync !== 'function') { - console.warn('tracing viewer: this Node has no zstd support; compressed files need Node >= 22.15'); - return Buffer.alloc(0); - } - try { - return zlib.zstdDecompressSync(data); - } catch { - return data; - } - } - return data; -} -function readStoredText(file) { - return readStored(file).toString('utf8'); -} const STATIC_DIR = path.join(__dirname, 'ui'); const SHELL_HTML = require('./ui/shell.js'); const BUNDLED_DESCRIPTORS_DIR = path.join(__dirname, 'descriptors'); diff --git a/raven/tracing/codec.py b/raven/tracing/codec.py index b92171db7..32cd40901 100644 --- a/raven/tracing/codec.py +++ b/raven/tracing/codec.py @@ -14,6 +14,7 @@ import io from pathlib import Path +from typing import BinaryIO import zstandard @@ -62,10 +63,52 @@ def plaintext_size(path: Path) -> int | None: if not is_compressed(head): return path.stat().st_size handle.seek(0) - size = zstandard.frame_content_size(handle.read(18)) - except (OSError, zstandard.ZstdError): + try: + size = zstandard.frame_content_size(handle.read(18)) + except zstandard.ZstdError: + return None + if size >= 0: + return int(size) + # A frame written without a content size is only measurable by + # decoding it; the store's own writer always records the size. + data = handle.read() + except OSError: return None - return None if size < 0 else int(size) + plain = read_plaintext(data) + return None if plain is None else len(plain) + + +def read_plaintext_file(path: Path) -> bytes | None: + """A stored file's plaintext, or None when it cannot be read or decoded.""" + try: + return read_plaintext(path.read_bytes()) + except OSError: + return None + + +def open_plaintext(path: Path) -> BinaryIO: + """Open a stored file as a plaintext byte stream. + + A caller that seeks or sizes the stream works in plaintext offsets, which + is the only coordinate the store records (a span log's cursor, an + artifact's range). A compressed payload is decompressed whole: these files + are bounded by the writer's own rotation cap. + """ + handle = path.open("rb") + try: + head = handle.read(4) + handle.seek(0) + if not is_compressed(head): + return handle + data = handle.read() + except BaseException: + handle.close() + raise + handle.close() + try: + return io.BytesIO(decompress_bytes(data)) + except zstandard.ZstdError as exc: + raise OSError(f"corrupt compressed payload: {path}") from exc __all__ = [ @@ -74,6 +117,8 @@ def plaintext_size(path: Path) -> int | None: "compress_bytes", "decompress_bytes", "is_compressed", + "open_plaintext", "plaintext_size", "read_plaintext", + "read_plaintext_file", ] diff --git a/raven/tracing/compress.py b/raven/tracing/compress.py index 5ee4d3558..8a840efcd 100644 --- a/raven/tracing/compress.py +++ b/raven/tracing/compress.py @@ -8,7 +8,13 @@ Idempotent: a blob or message already compressed is counted and left alone, so a second pass writes nothing. A dry run compresses in memory to report the real -saving and writes no file at all.""" +saving and writes no file at all. + +The rotated span/event logs are the same story one level up: closed files under +``archive/``, re-encoded in place. The active log and the ACP frame journals +are not touched -- the first is still being appended to, and a span records a +plaintext byte range into the second. +""" from __future__ import annotations @@ -159,4 +165,62 @@ def compress(artifacts_dir: Path, *, dry_run: bool = False) -> CompressResult: return result -__all__ = ["CompressResult", "compress"] +def _is_rotated_log(name: str) -> bool: + return (name.startswith("audit-spans-") or name.startswith("audit-events-")) and name.endswith(".log") + + +@dataclass +class LogCompressResult: + """What one pass over the rotated logs saw, did, and saved.""" + + files: int = 0 + compressed: int = 0 + already: int = 0 + bytes_before: int = 0 + bytes_after: int = 0 + errors: list[str] = field(default_factory=list) + + @property + def saved(self) -> int: + return max(self.bytes_before - self.bytes_after, 0) + + +def compress_logs(logs_dir: Path, *, dry_run: bool = False) -> LogCompressResult: + """Compress rotated span and event logs in place. + + Only ``archive/`` is touched: the active log is still being appended to. + ACP frame journals are left alone -- a span records a plaintext byte range + into one, so re-encoding it would invalidate that audit record. + + A rotated log is closed and never appended to again, which is why it can be + rewritten here and why a running writer needs no coordination. Idempotent: + a compressed file is counted and left alone; a dry run writes nothing. + """ + result = LogCompressResult() + archive = Path(logs_dir) / "archive" + if not archive.is_dir(): + return result + for path in sorted(p for p in _files(archive) if _is_rotated_log(p.name)): + result.files += 1 + try: + data = path.read_bytes() + except OSError as exc: + result.errors.append(f"{path}: {exc}") + continue + if is_compressed(data): + result.already += 1 + continue + packed = compress_bytes(data) + result.bytes_before += len(data) + result.bytes_after += len(packed) + result.compressed += 1 + if dry_run: + continue + try: + _replace(path, packed) + except OSError as exc: + result.errors.append(f"{path}: {exc}") + return result + + +__all__ = ["CompressResult", "LogCompressResult", "compress", "compress_logs"] diff --git a/raven/tracing/store.py b/raven/tracing/store.py index 78ff2afac..407d61529 100644 --- a/raven/tracing/store.py +++ b/raven/tracing/store.py @@ -120,8 +120,30 @@ def _rotate_if_needed(self, kind: str, next_text: str) -> Path: base = _KIND_FILES[kind].replace(".log", "") rotated = day_dir / f"{base}-{current_day}-{suffix}.log" path.rename(rotated) + self._compress_rotated(rotated) return path + def _compress_rotated(self, path: Path) -> None: + """Store a just-rotated log compressed under the name it already has. + + Rotation is the safe moment: the file is closed and never appended to + again, so rewriting it cannot race the writer. A failure leaves it + plain, which every reader accepts. + """ + try: + data = path.read_bytes() + if is_compressed(data): + return + packed = compress_bytes(data) + tmp = path.parent / f".{path.name}.{os.getpid()}.{uuid.uuid4().hex}.compress.tmp" + try: + tmp.write_bytes(packed) + os.replace(tmp, path) + finally: + tmp.unlink(missing_ok=True) + except OSError: + pass + def append(self, kind: str, record: dict[str, Any]) -> None: text = f"{to_json_text(record)}\n" path = self._rotate_if_needed(kind, text) diff --git a/raven/trajectory/index.py b/raven/trajectory/index.py index acf9c667e..bc7455b14 100644 --- a/raven/trajectory/index.py +++ b/raven/trajectory/index.py @@ -47,6 +47,7 @@ from pathlib import Path from typing import Any, Callable, Sequence +from raven.tracing import codec as trace_codec from raven.tracing import config as tracing_config from raven.trajectory import entries as _entries from raven.trajectory import store as tstore @@ -260,6 +261,19 @@ def __init__(self, state_dir: Path, *, max_line_bytes: int = MAX_LINE_BYTES) -> self.cursors: dict[FileKey, FileCursor] = {} self.active_key: FileKey | None = None self._active_path = state_dir / "logs" / "audit-spans.log" + # key -> (mtime_ns, raw size, plaintext size), so a catalog pass over + # an unchanged archive never reopens it just to learn its encoding. + self._sizes: dict[FileKey, tuple[int, int, int]] = {} + + def _plaintext_size(self, path: Path, key: FileKey, stat_result: Any) -> int | None: + cached = self._sizes.get(key) + if cached is not None and cached[0] == stat_result.st_mtime_ns and cached[1] == stat_result.st_size: + return cached[2] + size = trace_codec.plaintext_size(path) + if size is None: + return None + self._sizes[key] = (stat_result.st_mtime_ns, stat_result.st_size, size) + return size def _catalog(self) -> list[tuple[Path, FileKey, int, int]]: archive: list[tuple[Path, FileKey, int, int]] = [] @@ -269,7 +283,11 @@ def _catalog(self) -> list[tuple[Path, FileKey, int, int]]: stat_result = path.stat() except OSError: continue - item = (path, _file_key(stat_result), stat_result.st_size, stat_result.st_mtime_ns) + key = _file_key(stat_result) + size = self._plaintext_size(path, key, stat_result) + if size is None: + continue + item = (path, key, size, stat_result.st_mtime_ns) (active if path == self._active_path else archive).append(item) return archive + active @@ -291,6 +309,7 @@ def _generation_changed(self, catalog: Sequence[tuple[Path, FileKey, int, int]]) if cursor.offset < cursor.size_seen or cursor.partial: return True del self.cursors[key] + self._sizes.pop(key, None) continue if present[key] < cursor.offset: return True @@ -334,16 +353,31 @@ def scan(self, budget: ScanBudget) -> ScanBatch: batch.scanned_bytes = sum(c.offset for c in self.cursors.values()) return batch + @staticmethod + def _stream_stat(handle: Any, path: Path) -> tuple[Any, int]: + """The stat describing the opened stream, and its plaintext length. + + A real descriptor is stat-ed through the handle, so identity and size + come from the file actually read. A decompressed buffer has no + descriptor; its length is the frame's plaintext size and the path is + stat-ed for identity, with the fingerprint check beside it catching a + rotation in between. + """ + if hasattr(handle, "getbuffer"): + return path.stat(), len(handle.getbuffer()) + stat_result = os.fstat(handle.fileno()) + return stat_result, stat_result.st_size + def _verify(self, cursor: FileCursor) -> str: """Re-check an unchanged-size file's identity and first bytes; "ok", "retry" or "generation".""" try: - with cursor.path.open("rb") as handle: - stat_result = os.fstat(handle.fileno()) + with trace_codec.open_plaintext(cursor.path) as handle: + stat_result, size = self._stream_stat(handle, cursor.path) if _file_key(stat_result) != cursor.key: return "retry" - if stat_result.st_size < cursor.offset: + if size < cursor.offset: return "generation" - if stat_result.st_size >= FINGERPRINT_BYTES: + if size >= FINGERPRINT_BYTES: fingerprint = self._fingerprint(handle) if fingerprint is not None: if cursor.fingerprint is None: @@ -367,11 +401,10 @@ def _read(self, cursor: FileCursor, budget: ScanBudget, batch: ScanBatch) -> str """ assembler = _LineAssembler(cursor, self.max_line_bytes) try: - with cursor.path.open("rb") as handle: - stat_result = os.fstat(handle.fileno()) + with trace_codec.open_plaintext(cursor.path) as handle: + stat_result, size = self._stream_stat(handle, cursor.path) if _file_key(stat_result) != cursor.key: return "retry" - size = stat_result.st_size if size < cursor.offset: return "generation" cursor.size_seen = size @@ -425,11 +458,10 @@ def rescan(self, task: RecoveryTask, budget: ScanBudget) -> tuple[list[RawRecord task.discarding = False assembler = _LineAssembler(task, self.max_line_bytes) try: - with path.open("rb") as handle: - stat_result = os.fstat(handle.fileno()) + with trace_codec.open_plaintext(path) as handle: + stat_result, size = self._stream_stat(handle, path) if _file_key(stat_result) != key: return records, False - size = stat_result.st_size handle.seek(task.offset) while task.offset < size: want = min(SCAN_CHUNK, size - task.offset, budget.remaining()) diff --git a/raven/trajectory/selection.py b/raven/trajectory/selection.py index 416ba986e..6bd76bef2 100644 --- a/raven/trajectory/selection.py +++ b/raven/trajectory/selection.py @@ -39,6 +39,7 @@ from raven.tracing import artifact_v2 from raven.tracing import config as tracing_config +from raven.tracing.codec import read_plaintext_file from raven.trajectory import entries as _entries from raven.trajectory import store as tstore @@ -133,10 +134,10 @@ def _records( themselves only. """ for path in tstore.span_log_paths(state_dir): - try: - raw_lines = path.read_bytes().splitlines() - except OSError: + data = read_plaintext_file(path) + if data is None: continue + raw_lines = data.splitlines() for raw_line in raw_lines: if needles and not any(needle in raw_line for needle in needles): continue diff --git a/raven/trajectory/store.py b/raven/trajectory/store.py index 250248d8f..08e2706bc 100644 --- a/raven/trajectory/store.py +++ b/raven/trajectory/store.py @@ -52,6 +52,7 @@ from loguru import logger from raven.tracing import config as tracing_config +from raven.tracing.codec import read_plaintext_file from raven.utils.atomic_io import atomic_update T = TypeVar("T") @@ -679,10 +680,10 @@ def iter_spans( if owner is not None: match_ids = set(defs[owner]["traces"]) for path in span_log_paths(state_dir): - try: - raw_lines = path.read_bytes().splitlines() - except OSError: + data = read_plaintext_file(path) + if data is None: continue + raw_lines = data.splitlines() for raw_line in raw_lines: # Decode per line: one invalid-UTF-8 byte must hide one record, # not the whole log file. diff --git a/tests/integration/test_tracing_viewer_e2e.py b/tests/integration/test_tracing_viewer_e2e.py index df3a78c95..c6d5a3c3f 100644 --- a/tests/integration/test_tracing_viewer_e2e.py +++ b/tests/integration/test_tracing_viewer_e2e.py @@ -149,6 +149,19 @@ def test_reads_the_active_log_and_the_archive(tmp_path): assert _session_ids(_get(port, "/api/data")) == {"live-session", "archived-session"} +def test_reads_a_compressed_archive(tmp_path): + from raven.tracing.codec import compress_bytes + + logs = tmp_path / "logs" + archive = logs / "archive" / "2026-07-01" / "audit-spans-2026-07-01-1.log" + _write_spans(archive, [_span("archived-session", "span-arch")]) + archive.write_bytes(compress_bytes(archive.read_bytes())) + _write_spans(logs / "audit-spans.log", [_span("live-session", "span-live")]) + + with _viewer(tmp_path) as port: + assert _session_ids(_get(port, "/api/data")) == {"live-session", "archived-session"} + + def test_ignores_a_log_planted_outside_the_active_and_archive_paths(tmp_path): logs = tmp_path / "logs" _write_spans(logs / "audit-spans.log", [_span("live-session", "span-live")]) diff --git a/tests/test_cli_tracing_commands.py b/tests/test_cli_tracing_commands.py index 51674f713..37c987559 100644 --- a/tests/test_cli_tracing_commands.py +++ b/tests/test_cli_tracing_commands.py @@ -442,3 +442,16 @@ def test_tracing_compress_rewrites_the_blob_and_keeps_the_link(tmp_path, monkeyp assert r.exit_code == 0 assert artifact.read_bytes().startswith(b"\x28\xb5\x2f\xfd") assert artifact.stat().st_ino == blob.stat().st_ino + + +def test_tracing_compress_logs_dry_run_reports_and_writes_nothing(tmp_path, monkeypatch): + monkeypatch.setenv("RAVEN_TRACING_DIR", str(tmp_path)) + archive = tmp_path / "logs" / "archive" / "2026-01-01" / "audit-spans-2026-01-01-1.log" + archive.parent.mkdir(parents=True) + archive.write_text('{"spanId": "s"}\n', encoding="utf-8") + + r = runner.invoke(app, ["tracing", "compress", "--logs", "--dry-run"]) + + assert r.exit_code == 0 + assert "logs 1: compressed 1" in r.output + assert archive.read_text(encoding="utf-8") == '{"spanId": "s"}\n' diff --git a/tests/test_tracing_store_compression.py b/tests/test_tracing_store_compression.py index 73d99901d..f8d952755 100644 --- a/tests/test_tracing_store_compression.py +++ b/tests/test_tracing_store_compression.py @@ -16,12 +16,13 @@ import zstandard from raven.tracing import artifact_v2 -from raven.tracing.codec import plaintext_size +from raven.tracing.codec import is_compressed, plaintext_size from raven.tracing.compact import compact -from raven.tracing.compress import compress +from raven.tracing.compress import compress, compress_logs from raven.tracing.store import TraceStore -from raven.trajectory import details +from raven.trajectory import details, selection from raven.trajectory.conversation import _read_artifact +from raven.trajectory.store import iter_spans MAGIC = b"\x28\xb5\x2f\xfd" PAYLOAD = {"messages": [{"role": "user", "content": "hi " * 4000}]} @@ -213,3 +214,83 @@ def test_compact_records_a_corrupt_frame(tmp_path: Path) -> None: assert result.errors assert bad.exists() + + +def _span_record(trace: str, span_id: str) -> dict: + return {"traceId": trace, "spanId": span_id, "name": "tool.call", "attributes": {}} + + +def _plant_logs(state: Path) -> Path: + """A store with one rotated span log, one active log, and one frame journal.""" + logs = state / "logs" + archive = logs / "archive" / "2026-07-01" / "audit-spans-2026-07-01-1.log" + archive.parent.mkdir(parents=True) + archive.write_text(json.dumps(_span_record("t1", "s1")) + "\n", encoding="utf-8") + (logs / "audit-spans.log").write_text(json.dumps(_span_record("t2", "s2")) + "\n", encoding="utf-8") + frames = logs / "acp-frames" / "2026-07-01" / "a.jsonl" + frames.parent.mkdir(parents=True) + frames.write_text('{"method": "x"}\n', encoding="utf-8") + return archive + + +def test_compress_logs_rewrites_only_rotated_files(tmp_path: Path) -> None: + archive = _plant_logs(tmp_path) + active = tmp_path / "logs" / "audit-spans.log" + frames = tmp_path / "logs" / "acp-frames" / "2026-07-01" / "a.jsonl" + active_before = active.read_bytes() + frames_before = frames.read_bytes() + + result = compress_logs(tmp_path / "logs") + + assert (result.files, result.compressed, result.already) == (1, 1, 0) + assert result.errors == [] + assert is_compressed(archive.read_bytes()) + # A span records a plaintext byte range into a frame journal, so that file + # must keep its bytes; the active log is still being appended to. + assert active.read_bytes() == active_before + assert frames.read_bytes() == frames_before + + +def test_iter_spans_reads_a_compressed_archive(tmp_path: Path) -> None: + _plant_logs(tmp_path) + compress_logs(tmp_path / "logs") + + ids = [span["spanId"] for span in iter_spans(tmp_path)] + + assert ids == ["s1", "s2"] + + +def test_selection_reads_a_compressed_archive(tmp_path: Path) -> None: + _plant_logs(tmp_path) + compress_logs(tmp_path / "logs") + + records = list(selection._records(tmp_path, None)) + + assert [record["spanId"] for record in records] == ["s1", "s2"] + + +def test_compress_logs_is_idempotent_and_dry_run_writes_nothing(tmp_path: Path) -> None: + archive = _plant_logs(tmp_path) + before = archive.read_bytes() + + dry = compress_logs(tmp_path / "logs", dry_run=True) + assert dry.compressed == 1 + assert archive.read_bytes() == before + + once = compress_logs(tmp_path / "logs") + assert once.compressed == 1 + again = compress_logs(tmp_path / "logs") + assert again.compressed == 0 + assert again.already == 1 + + +def test_rotation_compresses_the_rotated_file(tmp_path: Path) -> None: + store = TraceStore(tmp_path, max_bytes=1) + + store.append_span(_span_record("t1", "s1")) + store.append_span(_span_record("t2", "s2")) + + archives = list((tmp_path / "logs" / "archive").rglob("audit-spans-*.log")) + assert len(archives) == 1 + assert is_compressed(archives[0].read_bytes()) + assert [span["spanId"] for span in iter_spans(tmp_path)] == ["s1", "s2"] diff --git a/tests/test_trajectory_index.py b/tests/test_trajectory_index.py index 16319e7e8..c3e4cffa2 100644 --- a/tests/test_trajectory_index.py +++ b/tests/test_trajectory_index.py @@ -10,6 +10,7 @@ import pytest +from raven.tracing.compress import compress_logs from raven.trajectory import conversation as tconv from raven.trajectory import entries as tent from raven.trajectory import index as tidx @@ -185,6 +186,33 @@ def test_broken_record_position_survives_rename(state, clock): assert after == before +def test_compressed_archive_is_indexed_in_plaintext_offsets(state, clock): + archive = state / "logs" / "archive" / "2026-09-01" / "audit-spans-2026-09-01-100000000000.log" + _append(state, [_turn("t", "before", start=0, end=1)], path=archive) + plaintext_bytes = len(archive.read_bytes()) + compress_logs(state / "logs") + _append(state, [_turn("t", "after", start=2, end=3)]) + + index = _index(state, clock) + _refresh_until_ready(index, clock) + + assert {e.span_id for e in index.entries()} == {"before", "after"} + assert index.index_state().total_bytes == plaintext_bytes + _log(state).stat().st_size + + +def test_compressing_a_consumed_archive_keeps_the_cursor_moving(state, clock): + _append(state, [_turn("t", "before", start=0, end=1)]) + index = _index(state, clock) + _refresh_until_ready(index, clock) + + _rotate(state) + compress_logs(state / "logs") + _append(state, [_turn("t", "after", start=2, end=3)]) + _refresh_until_ready(index, clock) + + assert {e.span_id for e in index.entries()} == {"before", "after"} + + # ── budgets ─────────────────────────────────────────────────────────── From b2433f9d43279ef28fe26c7f814299206b42a169 Mon Sep 17 00:00:00 2001 From: Dizhan Xue Date: Thu, 29 Oct 2026 14:31:34 +0000 Subject: [PATCH 03/23] feat(tracing): pack content-addressed messages into archives One small file per message costs a filesystem block each, so a store of tens of thousands of messages is mostly block slack. Packing concatenates message texts into multi-megabyte zstd frames and keeps an address to (pack, offset, length) index; a reader that finds no file reads its slice. `raven tracing pack-messages` writes the packs, commits the index, and only then removes the files, so a crash at any point leaves every address readable. The writer skips an address the index already holds. --- docs-site/docs/commands.md | 1 + docs-site/docs/commands.zh.md | 1 + raven/cli/tracing_commands.py | 37 +- raven/cli/tracing_viewer/server.js | 80 +++- raven/rpc/methods/cli_dispatch.py | 11 +- raven/tracing/artifact_v2.py | 19 +- raven/tracing/compact.py | 3 +- raven/tracing/compress.py | 4 +- raven/tracing/message_pack.py | 428 +++++++++++++++++++ raven/tracing/store.py | 8 +- raven/trajectory/conversation.py | 14 + raven/trajectory/details.py | 4 +- raven/trajectory/selection.py | 26 +- tests/integration/test_tracing_viewer_e2e.py | 32 ++ tests/test_cli_tracing_commands.py | 17 + tests/test_rpc_cli_dispatch.py | 31 +- tests/test_tracing_message_pack.py | 120 ++++++ 17 files changed, 813 insertions(+), 23 deletions(-) create mode 100644 raven/tracing/message_pack.py create mode 100644 tests/test_tracing_message_pack.py diff --git a/docs-site/docs/commands.md b/docs-site/docs/commands.md index 67757668e..e0eb48219 100644 --- a/docs-site/docs/commands.md +++ b/docs-site/docs/commands.md @@ -37,6 +37,7 @@ lists its subcommands and options. | `raven tracing` | Open the local trace dashboard | | `raven tracing compact` | Consolidate duplicate trace artifacts to free disk space | | `raven tracing compress [--logs]` | Rewrite trace payloads as zstd (with `--logs`, the rotated span and event logs too) to free disk space | +| `raven tracing pack-messages` | Pack content-addressed message blobs into a few archives to free disk space | | `raven trajectory` | Save, replay, redact, label, and preserve execution trajectories for debugging, and run the bug report receiver | Run `raven --help` for the command list, or `raven --help` for details diff --git a/docs-site/docs/commands.zh.md b/docs-site/docs/commands.zh.md index 1671bed25..5fbd509b5 100644 --- a/docs-site/docs/commands.zh.md +++ b/docs-site/docs/commands.zh.md @@ -36,6 +36,7 @@ | `raven tracing` | 打开本地追踪面板 | | `raven tracing compact` | 合并重复的追踪产物，回收磁盘空间 | | `raven tracing compress [--logs]` | 以 zstd 重写追踪载荷（加 `--logs` 时还包括轮转后的日志），回收磁盘空间 | +| `raven tracing pack-messages` | 将内容寻址的消息载荷打包成少量归档，回收磁盘空间 | | `raven trajectory` | 保存、回放、脱敏、标注和保留执行轨迹，用于调试；也运行问题报告接收器 | 运行 `raven --help` 查看命令列表，或运行 `raven --help` 查看指定命令的详细说明。 diff --git a/raven/cli/tracing_commands.py b/raven/cli/tracing_commands.py index 73fcea15f..95198c059 100644 --- a/raven/cli/tracing_commands.py +++ b/raven/cli/tracing_commands.py @@ -337,6 +337,28 @@ def _run_compress(*, dry_run: bool, logs: bool = False) -> None: _print_skipped(log_result.errors) +def _run_pack_messages(*, dry_run: bool) -> None: + from raven.tracing.message_pack import pack_messages + from raven.tracing.store import TraceStore + + artifacts_dir = TraceStore(tracing_config.state_dir()).artifacts_dir + if dry_run: + console.print("[dim]dry run - nothing is written[/dim]") + if not artifacts_dir.is_dir(): + console.print(f"[yellow]No artifacts at {escape(str(artifacts_dir))}.[/yellow]") + return + result = pack_messages(artifacts_dir, dry_run=dry_run) + console.print( + f"messages {result.scanned}: packed {result.packed}, already {result.already}, " + f"packs {result.packs}, removed files {result.removed}" + ) + console.print( + f"{result.bytes_before / (1024 * 1024):.1f} MB -> {result.bytes_after / (1024 * 1024):.1f} MB, " + f"saved {result.saved / (1024 * 1024):.1f} MB" + ) + _print_skipped(result.errors) + + def _print_skipped(errors: list[str]) -> None: for message in errors[:10]: console.print(f"[yellow]skipped:[/yellow] {escape(message)}") @@ -360,7 +382,8 @@ def tracing( help=( "Optional action: 'stop' shuts down the background viewer; " "'compact' folds duplicate artifacts; 'compress' rewrites payloads as zstd " - "(--logs adds the rotated span and event logs; both terminal only, not available from the TUI)." + "(--logs adds the rotated span and event logs); 'pack-messages' moves message blobs " + "into packs. All but 'stop' are terminal only, not available from the TUI." ), ), port: int = typer.Option(None, "--port", "-p", help="Port to bind (default: config or 4318)."), @@ -368,7 +391,9 @@ def tracing( False, "--foreground", "-f", help="Run the viewer in the foreground (blocks; Ctrl-C to stop)." ), dry_run: bool = typer.Option( - False, "--dry-run", help="With 'compact' or 'compress': report what would change without writing." + False, + "--dry-run", + help="With 'compact', 'compress' or 'pack-messages': report what would change without writing.", ), logs: bool = typer.Option(False, "--logs", help="With 'compress': also compress rotated span and event logs."), ) -> None: @@ -379,11 +404,17 @@ def tracing( if action == "compress": _run_compress(dry_run=dry_run, logs=logs) return + if action == "pack-messages": + _run_pack_messages(dry_run=dry_run) + return if action == "stop": _stop_viewer() return if action is not None: - console.print(f"[red]Unknown action '{escape(action)}'.[/red] Supported actions: stop, compact, compress") + console.print( + f"[red]Unknown action '{escape(action)}'.[/red] " + "Supported actions: stop, compact, compress, pack-messages" + ) raise typer.Exit(2) bind_port = port if port is not None else tracing_config.port() if foreground: diff --git a/raven/cli/tracing_viewer/server.js b/raven/cli/tracing_viewer/server.js index 2c23886c3..f6043b1b8 100644 --- a/raven/cli/tracing_viewer/server.js +++ b/raven/cli/tracing_viewer/server.js @@ -1061,10 +1061,70 @@ function isSafeArtifactPath(filePath) { // compares the two, so a rule changed there changes here in the same commit. const ARTIFACT_FORMAT_V2 = 'audit.artifact.v2'; const MESSAGES_DIR = path.join(ARTIFACTS_DIR, '_messages'); +const MESSAGE_PACKS_DIR = path.join(ARTIFACTS_DIR, '_message_packs'); +const MESSAGE_PACK_INDEX = path.join(MESSAGE_PACKS_DIR, 'index.json'); +const MESSAGE_PACK_FORMAT = 'raven.message-pack-index.v1'; const MSG_REF_KEY = '$msg'; const TEXT_FIELDS = ['systemPrompt', 'prompt']; const SHA1_RE = /^[0-9a-f]{40}$/; +// Mirror of raven/tracing/message_pack.py: message files can be moved into +// per-chunk zstd packs, addressed by an sha1 -> (pack, plaintext offset, +// length) index. The index is cached until its size or mtime changes, and a +// few decompressed packs are kept because one conversation page rereads the +// same pack for every message on it. +let packIndexCache = { stamp: null, index: null }; +const packCache = new Map(); + +function messagePackIndex() { + let stat; + try { + stat = fs.statSync(MESSAGE_PACK_INDEX); + } catch { + return null; + } + const stamp = `${stat.mtimeMs}:${stat.size}`; + if (packIndexCache.index && packIndexCache.stamp === stamp) return packIndexCache.index; + try { + const parsed = JSON.parse(fs.readFileSync(MESSAGE_PACK_INDEX, 'utf8')); + if (!parsed || parsed.format !== MESSAGE_PACK_FORMAT || typeof parsed.messages !== 'object') return null; + packIndexCache = { stamp, index: parsed }; + return parsed; + } catch { + return null; + } +} + +function readPackedMessage(sha1) { + const index = messagePackIndex(); + const entry = index && index.messages ? index.messages[sha1] : null; + if (!Array.isArray(entry) || entry.length !== 3) return null; + const packPath = path.join(MESSAGE_PACKS_DIR, entry[0]); + let plain = packCache.get(packPath); + if (!plain) { + try { + plain = readStored(packPath); + } catch { + return null; + } + if (plain.length <= 64 * 1024 * 1024) { + if (packCache.size >= 4) packCache.delete(packCache.keys().next().value); + packCache.set(packPath, plain); + } + } + return plain.subarray(entry[1], entry[1] + entry[2]); +} + +function messageSha1FromPath(filePath) { + const name = path.basename(filePath); + if (!name.endsWith('.json')) return null; + const sha1 = name.slice(0, -'.json'.length); + if (!SHA1_RE.test(sha1)) return null; + if (path.basename(path.dirname(filePath)) !== sha1.slice(0, 2)) return null; + if (path.basename(path.dirname(path.dirname(filePath))) !== '_messages') return null; + return sha1; +} + function refSha1(value) { if (!value || typeof value !== 'object' || Array.isArray(value)) return null; const keys = Object.keys(value); @@ -1097,7 +1157,14 @@ function resolvePayload(payload) { try { value = JSON.parse(readStoredText(file)); } catch { - value = { role: 'unknown', content: `[message blob missing: ${sha1}]` }; + const packed = readPackedMessage(sha1); + try { + value = packed + ? JSON.parse(packed.toString('utf8')) + : { role: 'unknown', content: `[message blob missing: ${sha1}]` }; + } catch { + value = { role: 'unknown', content: `[message blob missing: ${sha1}]` }; + } } cache.set(sha1, value); } @@ -1128,8 +1195,15 @@ function resolvePayload(payload) { function readArtifact(filePath) { if (!filePath || !isSafeArtifactPath(filePath)) return null; - if (!fs.existsSync(filePath)) return null; - const content = readStoredText(filePath); + let content = null; + if (fs.existsSync(filePath)) { + content = readStoredText(filePath); + } else { + const sha1 = messageSha1FromPath(filePath); + const packed = sha1 === null ? null : readPackedMessage(sha1); + if (packed) content = packed.toString('utf8'); + } + if (content === null) return null; let parsed = null; try { parsed = resolvePayload(JSON.parse(content)); diff --git a/raven/rpc/methods/cli_dispatch.py b/raven/rpc/methods/cli_dispatch.py index 3f85bc543..9cd2276ed 100644 --- a/raven/rpc/methods/cli_dispatch.py +++ b/raven/rpc/methods/cli_dispatch.py @@ -103,6 +103,7 @@ # every other order. ("tracing", "compact"), # full-store hash walk can outrun the RPC timeout — terminal-only ("tracing", "compress"), # same walk (and, with --logs, a whole archive rewrite) + ("tracing", "pack-messages"), # same walk over the message store # ``agent`` (no -m) — handled specially in _is_dispatch_compatible } @@ -120,20 +121,20 @@ def _is_agent_repl(argv: list[str]) -> bool: def _is_tracing_maintenance(argv: list[str]) -> bool: - """Return True for a ``tracing compact`` or ``tracing compress`` invocation. + """Return True for a ``tracing compact``, ``compress`` or ``pack-messages`` invocation. - Both are positional arguments of the ``tracing`` leaf command, so Click + All three are positional arguments of the ``tracing`` leaf command, so Click accepts them after any option: ``tracing --dry-run compact`` and ``tracing --port 1 compress`` both run. The ``_DISPATCH_BLACKLIST`` prefix match compares fixed positions and therefore only sees the action at ``argv[1]``, so it alone would let those forms reach dispatch. - ``raven tracing compact`` / ``tracing compress`` -> terminal-only - ``raven tracing`` / ``tracing stop`` -> OK + ``raven tracing compact`` / ``tracing compress`` / ``tracing pack-messages`` + -> terminal-only; ``raven tracing`` / ``tracing stop`` -> OK """ if not argv or argv[0] != "tracing": return False - return any(action in argv[1:] for action in ("compact", "compress")) + return any(action in argv[1:] for action in ("compact", "compress", "pack-messages")) # Default timeout if caller did not override. diff --git a/raven/tracing/artifact_v2.py b/raven/tracing/artifact_v2.py index ea01c8d85..2e181fc5a 100644 --- a/raven/tracing/artifact_v2.py +++ b/raven/tracing/artifact_v2.py @@ -7,7 +7,8 @@ validation and the per-field resolution rules all live here and nowhere else. The wire shape itself is stdlib; the default reader decodes the store's zstd -frames through :mod:`raven.tracing.codec`, imported lazily so this module +frames through :mod:`raven.tracing.codec`, and falls back to a packed slice +through :mod:`raven.tracing.message_pack`, both imported lazily so this module stays cheap to import. """ @@ -119,6 +120,13 @@ def _read_stored_text(path: Path) -> str: return plain.decode("utf-8") +def _packed_text(artifacts_dir: Path | str, sha1: str) -> str | None: + from . import message_pack + + packed = message_pack.read_message(artifacts_dir, sha1) + return None if packed is None else packed.decode("utf-8") + + def resolve_payload( payload: Any, artifacts_dir: Path | str, @@ -142,9 +150,18 @@ def resolve_payload( def load(sha1: str) -> Any: if sha1 not in cache: + text: str | None = None try: path = message_path(artifacts_dir, sha1) text = _read_stored_text(path) if read is None else read(path) + except (OSError, ValueError): + text = None + if text is None and read is None: + # A packed address has no file; the other encoding of the same + # store is a pack slice. A caller-supplied read owns its own + # fallback, because it may bound or confine the read. + text = _packed_text(artifacts_dir, sha1) + try: if text is None: raise OSError(f"blob {sha1} could not be read") cache[sha1] = json.loads(text) diff --git a/raven/tracing/compact.py b/raven/tracing/compact.py index ab985a68b..2d6c3307b 100644 --- a/raven/tracing/compact.py +++ b/raven/tracing/compact.py @@ -53,6 +53,7 @@ from raven.tracing import artifact_v2 from raven.tracing.codec import ZSTD_MAGIC +from raven.tracing.message_pack import PACKS_DIR_NAME from raven.tracing.store import BLOBS_DIR_NAME # Directories under ``audit-artifacts/`` that are content-addressed stores, not @@ -69,7 +70,7 @@ _MAX_ERRORS = 500 -_CONTENT_STORES = frozenset({BLOBS_DIR_NAME, artifact_v2.MESSAGES_DIR_NAME}) +_CONTENT_STORES = frozenset({BLOBS_DIR_NAME, artifact_v2.MESSAGES_DIR_NAME, PACKS_DIR_NAME}) @dataclass diff --git a/raven/tracing/compress.py b/raven/tracing/compress.py index 8a840efcd..079a45612 100644 --- a/raven/tracing/compress.py +++ b/raven/tracing/compress.py @@ -26,6 +26,7 @@ from . import artifact_v2 from .codec import compress_bytes, is_compressed +from .message_pack import PACKS_DIR_NAME from .store import BLOBS_DIR_NAME @@ -97,12 +98,13 @@ def compress(artifacts_dir: Path, *, dry_run: bool = False) -> CompressResult: return result blobs_dir = artifacts_dir / BLOBS_DIR_NAME messages_dir = artifact_v2.messages_dir(artifacts_dir) + packs_dir = artifacts_dir / PACKS_DIR_NAME # One walk to learn which artifact paths share each blob inode; the links # must be moved after the blob's own name is swapped. links: dict[int, list[Path]] = {} for path in _files(artifacts_dir): - if path.is_relative_to(blobs_dir) or path.is_relative_to(messages_dir): + if path.is_relative_to(blobs_dir) or path.is_relative_to(messages_dir) or path.is_relative_to(packs_dir): continue try: inode = path.stat().st_ino diff --git a/raven/tracing/message_pack.py b/raven/tracing/message_pack.py new file mode 100644 index 000000000..4ebc42c1d --- /dev/null +++ b/raven/tracing/message_pack.py @@ -0,0 +1,428 @@ +"""Message blobs packed into a few large zstd frames. + +The content-addressed message store writes one small file per distinct message. +Most messages are a few kilobytes, so a file system spends a block on each and +the store is mostly block slack: on a developer machine 40 thousand messages +held 224 MB while their payloads were a fraction of that. Packing concatenates +message texts into multi-megabyte frames and keeps an ``sha1 -> (pack, offset, +length)`` index beside them. A reader that finds no file for an address reads +its slice instead. + +Packing is a migration, not a writer change: :func:`pack_messages` appends the +files to packs, writes the index naming their slices, and only then deletes the +files. A store that has never been packed reads exactly as before, a +half-packed one resolves each address from whichever of the two holds it, and a +crash at any point leaves every address readable. + +An index entry stores plaintext offsets, so a reader can slice a pack without +knowing how the frame compressed it. The index and a few decompressed packs are +cached per process: a conversation page rereads the same pack for every message +it shows. +""" + +from __future__ import annotations + +import json +import os +import re +import threading +import uuid +from collections import OrderedDict +from dataclasses import dataclass, field +from pathlib import Path +from typing import Iterator + +import zstandard + +from .codec import compress_bytes, read_plaintext + +PACKS_DIR_NAME = "_message_packs" +MESSAGES_DIR_NAME = "_messages" +#: The message directory's name is a literal rather than an import of +#: ``artifact_v2``: that module resolves packed addresses back through this +#: one, so importing it here would close an import cycle. The pack test pins +#: the two constants equal. +INDEX_NAME = "index.json" +PACK_FORMAT = "raven.message-pack.v1" +INDEX_FORMAT = "raven.message-pack-index.v1" +PACK_TARGET_BYTES = 8 * 1024 * 1024 + +#: Decompressed packs kept by one process. Sized for a few conversation pages; +#: a pack larger than the whole budget is streamed on every read instead. +CACHE_BYTES = 32 * 1024 * 1024 + +_SHA1_RE = re.compile(r"^[0-9a-f]{40}$") + + +def packs_dir(artifacts_dir: Path | str) -> Path: + return Path(artifacts_dir) / PACKS_DIR_NAME + + +def index_path(artifacts_dir: Path | str) -> Path: + return packs_dir(artifacts_dir) / INDEX_NAME + + +@dataclass(frozen=True) +class _Index: + messages: dict[str, tuple[str, int, int]] + packs: dict[str, int] + + +_INDEX_CACHE: dict[Path, tuple[tuple[int, int], _Index]] = {} +_INDEX_LOCK = threading.Lock() +_PACK_CACHE: "OrderedDict[Path, bytes]" = OrderedDict() +_PACK_CACHE_BYTES = 0 +_PACK_LOCK = threading.Lock() + + +def _evict_index(path: Path) -> None: + with _INDEX_LOCK: + _INDEX_CACHE.pop(path, None) + + +def load_index(artifacts_dir: Path | str) -> _Index | None: + """The artifact store's pack index, cached until the file changes.""" + target = index_path(artifacts_dir) + try: + stat_result = target.stat() + except OSError: + return None + stamp = (stat_result.st_mtime_ns, stat_result.st_size) + with _INDEX_LOCK: + cached = _INDEX_CACHE.get(target) + if cached is not None and cached[0] == stamp: + return cached[1] + try: + raw = json.loads(target.read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + if not isinstance(raw, dict) or raw.get("format") != INDEX_FORMAT: + return None + messages_raw = raw.get("messages") + packs_raw = raw.get("packs") + if not isinstance(messages_raw, dict) or not isinstance(packs_raw, dict): + return None + messages: dict[str, tuple[str, int, int]] = {} + for sha1, entry in messages_raw.items(): + if not isinstance(sha1, str) or not _SHA1_RE.match(sha1): + continue + if ( + isinstance(entry, list) + and len(entry) == 3 + and isinstance(entry[0], str) + and all(isinstance(value, int) and not isinstance(value, bool) for value in entry[1:]) + ): + messages[sha1] = (entry[0], entry[1], entry[2]) + packs: dict[str, int] = {} + for name, info in packs_raw.items(): + if not isinstance(name, str) or not isinstance(info, dict): + continue + size = info.get("bytes", 0) + if isinstance(size, int) and not isinstance(size, bool): + packs[name] = size + index = _Index(messages=messages, packs=packs) + with _INDEX_LOCK: + _INDEX_CACHE[target] = (stamp, index) + return index + + +def _cached_pack(path: Path) -> bytes | None: + global _PACK_CACHE_BYTES + with _PACK_LOCK: + cached = _PACK_CACHE.get(path) + if cached is not None: + _PACK_CACHE.move_to_end(path) + return cached + try: + data = path.read_bytes() + plain = read_plaintext(data) + except OSError: + return None + if plain is None: + return None + with _PACK_LOCK: + if len(plain) <= CACHE_BYTES: + _PACK_CACHE[path] = plain + _PACK_CACHE_BYTES += len(plain) + while _PACK_CACHE_BYTES > CACHE_BYTES and len(_PACK_CACHE) > 1: + _, dropped = _PACK_CACHE.popitem(last=False) + _PACK_CACHE_BYTES -= len(dropped) + return plain + + +def _stream_slice(path: Path, offset: int, length: int) -> bytes | None: + try: + with ( + path.open("rb") as handle, + zstandard.ZstdDecompressor().stream_reader(handle, read_across_frames=False) as reader, + ): + remaining = offset + while remaining > 0: + chunk = reader.read(min(remaining, 1 << 20)) + if not chunk: + return None + remaining -= len(chunk) + return reader.read(length) + except (OSError, zstandard.ZstdError): + return None + + +def _slice(artifacts_dir: Path, sha1: str, limit: int | None) -> bytes | None: + index = load_index(artifacts_dir) + if index is None: + return None + located = index.messages.get(sha1) + if located is None: + return None + name, offset, length = located + want = length if limit is None else min(length, limit) + pack = packs_dir(artifacts_dir) / name + plain = _cached_pack(pack) + if plain is None: + data = _stream_slice(pack, offset, want) + if data is None or len(data) != want: + return None + return data + if offset < 0 or offset + length > len(plain): + return None + return plain[offset : offset + want] + + +def read_message(artifacts_dir: Path | str, sha1: str, limit: int | None = None) -> bytes | None: + """One packed message's plaintext, or None when the index does not hold it.""" + if not _SHA1_RE.match(sha1): + return None + return _slice(Path(artifacts_dir), sha1, limit) + + +def message_size(artifacts_dir: Path | str, sha1: str) -> int | None: + index = load_index(artifacts_dir) + if index is None: + return None + located = index.messages.get(sha1) + return located[2] if located is not None else None + + +def sha1_from_path(path: Path | str) -> str | None: + """The address a message-blob path names, or None when it is not one.""" + path = Path(path) + if not path.name.endswith(".json"): + return None + sha1 = path.name[: -len(".json")] + if not _SHA1_RE.match(sha1): + return None + if path.parent.name != sha1[:2] or path.parent.parent.name != MESSAGES_DIR_NAME: + return None + return sha1 + + +def _artifacts_from_path(path: Path) -> Path | None: + if sha1_from_path(path) is None: + return None + return path.parent.parent.parent + + +def read_path(path: Path | str, limit: int | None = None) -> bytes | None: + """The packed slice a message-file path would have held, or None.""" + target = Path(path) + if not target.is_absolute(): + target = target.resolve() + artifacts_dir = _artifacts_from_path(target) + sha1 = sha1_from_path(target) + if artifacts_dir is None or sha1 is None: + return None + return _slice(artifacts_dir, sha1, limit) + + +def size_path(path: Path | str) -> int | None: + target = Path(path) + if not target.is_absolute(): + target = target.resolve() + artifacts_dir = _artifacts_from_path(target) + sha1 = sha1_from_path(target) + if artifacts_dir is None or sha1 is None: + return None + return message_size(artifacts_dir, sha1) + + +def path_mtime(path: Path | str) -> float | None: + """When the slice's pack was written, for callers that need an mtime.""" + target = Path(path) + if not target.is_absolute(): + target = target.resolve() + artifacts_dir = _artifacts_from_path(target) + if artifacts_dir is None: + return None + index = load_index(artifacts_dir) + sha1 = sha1_from_path(target) + if index is None or sha1 is None: + return None + located = index.messages.get(sha1) + if located is None: + return None + try: + return (packs_dir(artifacts_dir) / located[0]).stat().st_mtime + except OSError: + return None + + +def _walk(root: Path) -> Iterator[Path]: + for directory, _, names in os.walk(root): + for name in names: + if name.startswith("."): + continue + path = Path(directory) / name + if path.is_file(): + yield path + + +def _atomic_write_index(artifacts_dir: Path, index: _Index) -> None: + payload = { + "format": INDEX_FORMAT, + "packs": {name: {"bytes": size} for name, size in index.packs.items()}, + "messages": {sha1: list(entry) for sha1, entry in index.messages.items()}, + } + target = index_path(artifacts_dir) + target.parent.mkdir(parents=True, exist_ok=True) + tmp = target.parent / f".{target.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp" + try: + tmp.write_text(json.dumps(payload, separators=(",", ":")), encoding="utf-8") + os.replace(tmp, target) + finally: + tmp.unlink(missing_ok=True) + _evict_index(target) + + +@dataclass +class PackResult: + """What one packing pass saw, did, and saved.""" + + scanned: int = 0 + packed: int = 0 + already: int = 0 + packs: int = 0 + removed: int = 0 + bytes_before: int = 0 + bytes_after: int = 0 + errors: list[str] = field(default_factory=list) + + @property + def saved(self) -> int: + return max(self.bytes_before - self.bytes_after, 0) + + +def pack_messages( + artifacts_dir: Path | str, *, dry_run: bool = False, target_bytes: int = PACK_TARGET_BYTES +) -> PackResult: + """Move unpacked message files into packs; see the module doc. + + Idempotent: an address already in the index is counted and left alone (its + file is removed if a previous run left one behind). A dry run compresses in + memory to report the real saving and writes or deletes nothing. + """ + artifacts_dir = Path(artifacts_dir) + messages_dir = artifacts_dir / MESSAGES_DIR_NAME + result = PackResult() + if not messages_dir.is_dir(): + return result + existing = load_index(artifacts_dir) + known = dict(existing.messages) if existing is not None else {} + packs = dict(existing.packs) if existing is not None else {} + added: dict[str, tuple[str, int, int]] = {} + pending: list[tuple[str, bytes]] = [] + pending_bytes = 0 + candidates: list[tuple[Path, str]] = [] + + def flush() -> None: + nonlocal pending, pending_bytes + if not pending: + return + payload = b"".join(data for _, data in pending) + packed = compress_bytes(payload) + name = f"{uuid.uuid4().hex}.pack.zst" + offset = 0 + for sha1, data in pending: + added[sha1] = (name, offset, len(data)) + offset += len(data) + result.bytes_after += len(packed) + result.packs += 1 + if not dry_run: + target = packs_dir(artifacts_dir) / name + target.parent.mkdir(parents=True, exist_ok=True) + tmp = target.parent / f".{name}.{os.getpid()}.{uuid.uuid4().hex}.tmp" + try: + tmp.write_bytes(packed) + os.replace(tmp, target) + finally: + tmp.unlink(missing_ok=True) + packs[name] = len(payload) + pending = [] + pending_bytes = 0 + + for path in sorted(_walk(messages_dir)): + sha1 = path.name[: -len(".json")] if path.name.endswith(".json") else "" + if not _SHA1_RE.match(sha1) or path.name != f"{sha1}.json": + continue + result.scanned += 1 + candidates.append((path, sha1)) + if sha1 in known: + result.already += 1 + continue + try: + data = read_plaintext(path.read_bytes()) + except OSError as exc: + result.errors.append(f"{path}: {exc}") + continue + if data is None: + result.errors.append(f"{path}: corrupt compressed message") + continue + pending.append((sha1, data)) + pending_bytes += len(data) + result.packed += 1 + result.bytes_before += len(data) + if pending_bytes >= target_bytes: + flush() + flush() + + if added and not dry_run: + try: + _atomic_write_index(artifacts_dir, _Index(messages={**known, **added}, packs=packs)) + except OSError as exc: + result.errors.append(f"{index_path(artifacts_dir)}: {exc}") + return result + for path, sha1 in candidates: + if sha1 not in known and sha1 not in added: + continue + try: + path.unlink() + result.removed += 1 + except OSError as exc: + result.errors.append(f"{path}: {exc}") + for orphan in packs_dir(artifacts_dir).glob("*.pack.zst"): + if orphan.name not in packs: + try: + orphan.unlink() + except OSError: + pass + return result + + +__all__ = [ + "CACHE_BYTES", + "INDEX_FORMAT", + "MESSAGES_DIR_NAME", + "PACKS_DIR_NAME", + "PACK_FORMAT", + "PACK_TARGET_BYTES", + "PackResult", + "index_path", + "load_index", + "message_size", + "pack_messages", + "packs_dir", + "path_mtime", + "read_message", + "read_path", + "sha1_from_path", + "size_path", +] diff --git a/raven/tracing/store.py b/raven/tracing/store.py index 407d61529..91a00f6c6 100644 --- a/raven/tracing/store.py +++ b/raven/tracing/store.py @@ -335,7 +335,7 @@ def address_items(self, items: list[Any]) -> list[Any]: sha1 = artifact_v2.message_sha1(item) blob = artifact_v2.message_path(self.artifacts_dir, sha1) payload = compress_bytes(text.encode("utf-8")) - if not blob.exists(): + if not blob.exists() and not self._is_packed(sha1): self._publish_blob(blob, payload, sha1) if blob.exists() and not self._blob_is_intact(blob, sha1): self._repair_blob(blob, payload, sha1) @@ -344,6 +344,12 @@ def address_items(self, items: list[Any]) -> list[Any]: refs.append(item) return refs + def _is_packed(self, sha1: str) -> bool: + """Whether the message store's pack index already holds this address.""" + from . import message_pack + + return message_pack.message_size(self.artifacts_dir, sha1) is not None + @staticmethod def artifact_attributes(prefix: str, artifact: dict[str, Any] | None) -> dict[str, Any]: if not artifact: diff --git a/raven/trajectory/conversation.py b/raven/trajectory/conversation.py index 1c38b9c1a..af7008524 100644 --- a/raven/trajectory/conversation.py +++ b/raven/trajectory/conversation.py @@ -134,6 +134,18 @@ def _kind(name: str) -> str: return _KIND_BY_DOMAIN.get(name.split(".", 1)[0], "other") +def _read_packed_message(target: Path, limit: int) -> tuple[str | None, str | None]: + """A message whose file was moved into a pack; the cap matches a file read.""" + from raven.tracing import message_pack + + data = message_pack.read_path(target, limit + 1) + if data is None: + return None, "artifact missing" + if len(data) > limit: + return data[:limit].decode("utf-8", errors="replace"), f"content over {limit // 1024} KiB — truncated" + return data.decode("utf-8", errors="replace"), None + + def _read_artifact(state: Path, path: str, limit: int = _ARTIFACT_LIMIT) -> tuple[str | None, str | None]: """(text, degraded reason); (None, reason) means unreadable. @@ -151,6 +163,8 @@ def _read_artifact(state: Path, path: str, limit: int = _ARTIFACT_LIMIT) -> tupl return None, "artifact path outside the trace store" if target.exists() and not target.is_file(): return None, "artifact is not a regular file" + if not target.exists(): + return _read_packed_message(target, limit) with target.open("rb") as handle: head = handle.read(4) handle.seek(0) diff --git a/raven/trajectory/details.py b/raven/trajectory/details.py index 85336efbb..e7da18816 100644 --- a/raven/trajectory/details.py +++ b/raven/trajectory/details.py @@ -39,7 +39,7 @@ from pathlib import Path from typing import Any, Callable, Sequence -from raven.tracing import artifact_v2 +from raven.tracing import artifact_v2, message_pack from raven.tracing import codec as trace_codec from raven.trajectory import conversation as _conv from raven.trajectory import entries as _entries @@ -493,6 +493,8 @@ def blob_prefix(self, sha1: str, limit: int) -> tuple[str | None, int | None] | self.exhausted = True return None size: int | None = trace_codec.plaintext_size(path) + if size is None: + size = message_pack.size_path(path) text, _ = _conv._read_artifact(self.state, str(path), limit) if size is not None else (None, None) self.blobs += 1 self.bytes += len(text.encode("utf-8", errors="replace")) if text else 0 diff --git a/raven/trajectory/selection.py b/raven/trajectory/selection.py index 6bd76bef2..c6eed3f65 100644 --- a/raven/trajectory/selection.py +++ b/raven/trajectory/selection.py @@ -349,11 +349,14 @@ def read_store_file(path: Path, root: Path) -> tuple[bytes, float] | None: """ try: target = Path(path).resolve() - if not target.is_relative_to(root): - return None - fd = os.open(target, os.O_RDONLY | getattr(os, "O_NONBLOCK", 0) | getattr(os, "O_BINARY", 0)) except (OSError, ValueError): return None + if not target.is_relative_to(root): + return None + try: + fd = os.open(target, os.O_RDONLY | getattr(os, "O_NONBLOCK", 0) | getattr(os, "O_BINARY", 0)) + except OSError: + return _read_packed_store_file(target, root) try: info = os.fstat(fd) if not stat.S_ISREG(info.st_mode): @@ -372,6 +375,23 @@ def read_store_file(path: Path, root: Path) -> tuple[bytes, float] | None: os.close(fd) +def _read_packed_store_file(target: Path, root: Path) -> tuple[bytes, float] | None: + """A message whose file moved into a pack, confined to the same root.""" + from raven.tracing import message_pack + + artifacts_dir = target.parent.parent.parent + try: + if not message_pack.packs_dir(artifacts_dir).resolve().is_relative_to(Path(root).resolve()): + return None + except (OSError, ValueError): + return None + data = message_pack.read_path(target) + mtime = message_pack.path_mtime(target) + if data is None or mtime is None: + return None + return data, mtime + + def _read_session(path: Path) -> tuple[dict[str, Any] | None, list[dict[str, Any]]] | None: try: lines = path.read_text(encoding="utf-8").splitlines() diff --git a/tests/integration/test_tracing_viewer_e2e.py b/tests/integration/test_tracing_viewer_e2e.py index c6d5a3c3f..10c6808aa 100644 --- a/tests/integration/test_tracing_viewer_e2e.py +++ b/tests/integration/test_tracing_viewer_e2e.py @@ -162,6 +162,38 @@ def test_reads_a_compressed_archive(tmp_path): assert _session_ids(_get(port, "/api/data")) == {"live-session", "archived-session"} +def test_artifact_endpoint_resolves_a_packed_message(tmp_path): + from raven.tracing import artifact_v2 + from raven.tracing.message_pack import pack_messages + + logs = tmp_path / "logs" + artifacts = logs / "audit-artifacts" + message = {"role": "user", "content": "packed"} + sha1 = artifact_v2.message_sha1(message) + path = artifact_v2.message_path(artifacts, sha1) + path.parent.mkdir(parents=True) + path.write_text(artifact_v2.message_text(message), encoding="utf-8") + shell = artifacts / "llm.input" / "2026-08-01" / "a.json" + shell.parent.mkdir(parents=True) + shell.write_text( + json.dumps( + { + "artifactFormat": artifact_v2.ARTIFACT_FORMAT, + "messages": [artifact_v2.make_ref(sha1)], + "prompt": artifact_v2.make_ref(sha1), + } + ), + encoding="utf-8", + ) + assert pack_messages(artifacts).packed == 1 + + with _viewer(tmp_path) as port: + found = _get(port, "/api/artifact?path=" + urllib.parse.quote(str(shell), safe="")) + + assert found["parsed"]["messages"] == [message] + assert found["parsed"]["prompt"] == "packed" + + def test_ignores_a_log_planted_outside_the_active_and_archive_paths(tmp_path): logs = tmp_path / "logs" _write_spans(logs / "audit-spans.log", [_span("live-session", "span-live")]) diff --git a/tests/test_cli_tracing_commands.py b/tests/test_cli_tracing_commands.py index 37c987559..02e080f86 100644 --- a/tests/test_cli_tracing_commands.py +++ b/tests/test_cli_tracing_commands.py @@ -455,3 +455,20 @@ def test_tracing_compress_logs_dry_run_reports_and_writes_nothing(tmp_path, monk assert r.exit_code == 0 assert "logs 1: compressed 1" in r.output assert archive.read_text(encoding="utf-8") == '{"spanId": "s"}\n' + + +def test_tracing_pack_messages_packs_the_message_store(tmp_path, monkeypatch): + from raven.tracing import artifact_v2 + + monkeypatch.setenv("RAVEN_TRACING_DIR", str(tmp_path)) + message = {"role": "user", "content": "hi"} + sha1 = artifact_v2.message_sha1(message) + path = artifact_v2.message_path(tmp_path / "logs" / "audit-artifacts", sha1) + path.parent.mkdir(parents=True) + path.write_text(artifact_v2.message_text(message), encoding="utf-8") + + r = runner.invoke(app, ["tracing", "pack-messages"]) + + assert r.exit_code == 0 + assert "packed 1" in r.output + assert not path.exists() diff --git a/tests/test_rpc_cli_dispatch.py b/tests/test_rpc_cli_dispatch.py index bc7c316e4..3601930ef 100644 --- a/tests/test_rpc_cli_dispatch.py +++ b/tests/test_rpc_cli_dispatch.py @@ -258,12 +258,12 @@ def test_channels_login_normal_channels_dispatch(): def test_blacklist_full_coverage(): - """10-entry blacklist (5 original + tui + onboard + upgrade + 2 tracing maintenance). + """11-entry blacklist (5 original + tui + onboard + upgrade + 3 tracing maintenance). `tui` blocks recursive Ink+Node spawn; `onboard` blocks prompt_toolkit wizard stdin hijack; `upgrade` blocks replacing the active Raven process; - the `tracing` maintenance actions block a full-store walk that can outrun - the RPC timeout with no cancellation of the backing thread. All are + the three `tracing` maintenance actions block a full-store walk that can + outrun the RPC timeout with no cancellation of the backing thread. All are necessary because reflection will otherwise let them through. """ from raven.rpc.methods.cli_dispatch import _DISPATCH_BLACKLIST @@ -279,9 +279,10 @@ def test_blacklist_full_coverage(): ("upgrade",), ("tracing", "compact"), ("tracing", "compress"), + ("tracing", "pack-messages"), } assert _DISPATCH_BLACKLIST == expected_entries, ( - f"blacklist drift; expected 10 prefix tuples (+ agent REPL special-case), got {_DISPATCH_BLACKLIST}" + f"blacklist drift; expected 11 prefix tuples (+ agent REPL special-case), got {_DISPATCH_BLACKLIST}" ) # Hard-reject probe (prefix match) assert _is_dispatch_compatible(["tui"]) is False @@ -293,6 +294,7 @@ def test_blacklist_full_coverage(): assert _is_dispatch_compatible(["tracing", "compact"]) is False assert _is_dispatch_compatible(["tracing", "compact", "--dry-run"]) is False # prefix assert _is_dispatch_compatible(["tracing", "compress"]) is False + assert _is_dispatch_compatible(["tracing", "pack-messages"]) is False # Bare `tracing` and its other action stay dispatchable -- only the # maintenance actions are terminal-only. assert _is_dispatch_compatible(["tracing"]) is True @@ -340,6 +342,27 @@ def test_tracing_compress_is_rejected_whatever_the_option_order(): assert _is_dispatch_compatible(argv) is False, argv +def test_tracing_pack_messages_is_rejected_whatever_the_option_order(): + """``pack-messages`` walks the whole message store and deletes files. + + Like the other maintenance actions it is positional, so Click accepts it + after any option and the fixed-position blacklist prefix would miss it. + """ + for argv in ( + ["tracing", "pack-messages"], + ["tracing", "pack-messages", "--dry-run"], + ["tracing", "--dry-run", "pack-messages"], + ["tracing", "--port", "1", "pack-messages"], + ["tracing", "-p", "1", "pack-messages"], + ): + assert _is_dispatch_compatible(argv) is False, argv + + +# --------------------------------------------------------------------------- +# Reflection-based dispatch compat (_DISPATCH_WHITELIST deleted) +# --------------------------------------------------------------------------- + + def test_dispatch_compatible_via_reflection_newly_active(): """Commands previously absent from _DISPATCH_WHITELIST now dispatch-OK purely because reflection sees them registered on ec.cli.commands.app. diff --git a/tests/test_tracing_message_pack.py b/tests/test_tracing_message_pack.py new file mode 100644 index 000000000..59a6e0e61 --- /dev/null +++ b/tests/test_tracing_message_pack.py @@ -0,0 +1,120 @@ +"""Packed message blobs: a few large frames plus an address index. + +A packed store keeps the artifact contract: an address resolves whether its +bytes still live in a file or in a pack slice, every reader that resolved a +file before resolves a packed address, and a migration never deletes a file +before the index naming its slice is durable. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from raven.tracing import artifact_v2, message_pack +from raven.tracing.store import TraceStore +from raven.trajectory import conversation, details, selection + +MESSAGES = [ + {"role": "user", "content": "first " * 100}, + {"role": "assistant", "content": "second " * 100}, +] + + +def _publish(state: Path) -> list[dict]: + return TraceStore(state).address_items(MESSAGES) + + +def test_the_pack_module_names_the_artifact_store_directory() -> None: + """The literal exists to keep artifact_v2 out of this module's imports.""" + assert message_pack.MESSAGES_DIR_NAME == artifact_v2.MESSAGES_DIR_NAME + + +def test_pack_moves_files_into_a_pack_and_every_reader_resolves(tmp_path: Path) -> None: + refs = _publish(tmp_path) + sha1s = [ref["$msg"] for ref in refs] + artifacts = tmp_path / "logs" / "audit-artifacts" + texts = {sha1: artifact_v2.message_text(message).encode() for sha1, message in zip(sha1s, MESSAGES)} + assert all(artifact_v2.message_path(artifacts, sha1).is_file() for sha1 in sha1s) + + result = message_pack.pack_messages(artifacts) + + assert (result.packed, result.packs, result.removed) == (2, 1, 2) + assert result.errors == [] + assert not any(artifact_v2.message_path(artifacts, sha1).exists() for sha1 in sha1s) + for sha1 in sha1s: + assert message_pack.read_message(artifacts, sha1) == texts[sha1] + + shell = {"artifactFormat": artifact_v2.ARTIFACT_FORMAT, "messages": refs, "prompt": refs[0]} + resolved = artifact_v2.resolve_payload(shell, artifacts) + assert resolved["messages"] == MESSAGES + assert resolved["prompt"] == MESSAGES[0]["content"] + + path = str(artifact_v2.message_path(artifacts, sha1s[0])) + text, reason = conversation._read_artifact(tmp_path, path) + assert reason is None + assert json.loads(text) == MESSAGES[0] + limited, note = conversation._read_artifact(tmp_path, path, 10) + assert limited == texts[sha1s[0]][:10].decode() + assert note is not None and "truncated" in note + + reader = details._Reader(tmp_path, max_artifacts=None, max_blobs=None, max_bytes=None) + prefix, size = reader.blob_prefix(sha1s[0], 12) + assert prefix == texts[sha1s[0]][:12].decode() + assert size == len(texts[sha1s[0]]) + + found = selection.read_store_file(artifact_v2.message_path(artifacts, sha1s[0]), artifacts.resolve()) + assert found is not None + assert found[0] == texts[sha1s[0]] + + +def test_a_small_target_splits_messages_across_packs(tmp_path: Path) -> None: + refs = _publish(tmp_path) + artifacts = tmp_path / "logs" / "audit-artifacts" + + result = message_pack.pack_messages(artifacts, target_bytes=1) + + assert result.packs == 2 + for ref, message in zip(refs, MESSAGES): + assert json.loads(message_pack.read_message(artifacts, ref["$msg"])) == message + + +def test_pack_is_idempotent_and_dry_run_writes_nothing(tmp_path: Path) -> None: + refs = _publish(tmp_path) + artifacts = tmp_path / "logs" / "audit-artifacts" + files = [artifact_v2.message_path(artifacts, ref["$msg"]) for ref in refs] + + dry = message_pack.pack_messages(artifacts, dry_run=True) + assert (dry.packed, dry.packs) == (2, 1) + assert all(path.is_file() for path in files) + assert not message_pack.index_path(artifacts).exists() + + once = message_pack.pack_messages(artifacts) + assert once.packed == 2 + again = message_pack.pack_messages(artifacts) + assert again.scanned == 0 + assert again.packed == 0 + + +def test_the_writer_does_not_republish_a_packed_message(tmp_path: Path) -> None: + store = TraceStore(tmp_path) + refs = store.address_items(MESSAGES) + message_pack.pack_messages(store.artifacts_dir) + + assert store.address_items(MESSAGES) == refs + assert not any(artifact_v2.message_path(store.artifacts_dir, ref["$msg"]).exists() for ref in refs) + + +def test_a_corrupt_message_file_is_reported_and_not_removed(tmp_path: Path) -> None: + refs = _publish(tmp_path) + artifacts = tmp_path / "logs" / "audit-artifacts" + corrupt = artifact_v2.message_path(artifacts, refs[1]["$msg"]) + corrupt.write_bytes(b"\x28\xb5\x2f\xfd not a frame") + + result = message_pack.pack_messages(artifacts) + + assert result.packed == 1 + assert result.errors + assert corrupt.exists() + assert not artifact_v2.message_path(artifacts, refs[0]["$msg"]).exists() + assert message_pack.read_message(artifacts, refs[0]["$msg"]) is not None From babad89e05146a30324f331c085bfabacaa6204d Mon Sep 17 00:00:00 2001 From: Dizhan Xue Date: Thu, 29 Oct 2026 14:38:37 +0000 Subject: [PATCH 04/23] fix(tracing): bound how much a stored frame may decode to A trace store can be imported from another machine, so a compressed payload is untrusted input. Reading one now stops at MAX_PLAINTEXT_BYTES (256 MiB, far above the largest captured payload) and treats an oversized or corrupt frame as unreadable instead of allocating it. The same bound applies to the viewer's zstd decode through maxOutputLength. resolve_payload also gets a per-shell total budget, so a shell naming many addresses cannot pull in more than one bounded amount. --- raven/cli/tracing_viewer/log-store.js | 11 ++++-- raven/tracing/artifact_v2.py | 45 ++++++++++++++++--------- raven/tracing/codec.py | 37 +++++++++++++++----- raven/tracing/message_pack.py | 5 ++- tests/test_tracing_message_pack.py | 12 +++++++ tests/test_tracing_store_compression.py | 14 ++++++++ 6 files changed, 96 insertions(+), 28 deletions(-) diff --git a/raven/cli/tracing_viewer/log-store.js b/raven/cli/tracing_viewer/log-store.js index 7a4988178..4dfb60eeb 100644 --- a/raven/cli/tracing_viewer/log-store.js +++ b/raven/cli/tracing_viewer/log-store.js @@ -4,6 +4,7 @@ const path = require('path'); const zlib = require('zlib'); const DEFAULT_MAX_BYTES = 50 * 1024 * 1024; +const MAX_STORED_BYTES = 256 * 1024 * 1024; const ZSTD_MAGIC = Buffer.from([0x28, 0xb5, 0x2f, 0xfd]); function readStored(filePath) { @@ -17,8 +18,14 @@ function readStored(filePath) { return Buffer.alloc(0); } try { - return zlib.zstdDecompressSync(data); - } catch { + return zlib.zstdDecompressSync(data, { maxOutputLength: MAX_STORED_BYTES }); + } catch (err) { + if (err && err.code === 'ERR_BUFFER_TOO_LARGE') { + // A compressed file can be imported with a store; a frame that expands + // past the read limit is treated as unreadable rather than allocated. + console.warn('tracing viewer: a compressed file expands past the read limit; treating it as empty'); + return Buffer.alloc(0); + } return data; } } diff --git a/raven/tracing/artifact_v2.py b/raven/tracing/artifact_v2.py index 2e181fc5a..29bfd89db 100644 --- a/raven/tracing/artifact_v2.py +++ b/raven/tracing/artifact_v2.py @@ -20,7 +20,7 @@ from pathlib import Path from typing import Any, Callable -from .codec import read_plaintext +from .codec import MAX_PLAINTEXT_BYTES, read_plaintext ARTIFACT_FORMAT = "audit.artifact.v2" MESSAGES_DIR_NAME = "_messages" @@ -112,18 +112,18 @@ def placeholder(sha1: str) -> dict[str, str]: return {"role": "unknown", "content": f"[message blob missing: {sha1}]"} -def _read_stored_text(path: Path) -> str: +def _read_stored_text(path: Path, limit: int = MAX_PLAINTEXT_BYTES) -> str: """A blob's plaintext, whichever encoding it was stored under.""" - plain = read_plaintext(path.read_bytes()) + plain = read_plaintext(path.read_bytes(), limit) if plain is None: - raise ValueError("corrupt compressed message blob") + raise ValueError("corrupt or oversized compressed message blob") return plain.decode("utf-8") -def _packed_text(artifacts_dir: Path | str, sha1: str) -> str | None: +def _packed_text(artifacts_dir: Path | str, sha1: str, limit: int = MAX_PLAINTEXT_BYTES) -> str | None: from . import message_pack - packed = message_pack.read_message(artifacts_dir, sha1) + packed = message_pack.read_message(artifacts_dir, sha1, limit) return None if packed is None else packed.decode("utf-8") @@ -133,6 +133,7 @@ def resolve_payload( *, on_missing: Callable[[str], Any] | None = None, read: Callable[[Path], str | None] | None = None, + max_total_bytes: int = MAX_PLAINTEXT_BYTES, ) -> Any: """A v2 shell as its v1 equivalent; any other payload returned unchanged. @@ -142,29 +143,41 @@ def resolve_payload( position, so omitting one shifts every later message and turns a locatable gap into a spurious full divergence. ``read`` replaces how a blob's text is read (None means it cannot be), for a caller that must bound what it reads. + + ``max_total_bytes`` bounds the plaintext one shell may pull in. A shell can + name any number of addresses, and an imported one is untrusted input, so + the addresses past the budget resolve to the missing-blob stand-in instead + of being read. """ if not is_v2(payload): return payload missing = placeholder if on_missing is None else on_missing cache: dict[str, Any] = {} + used = 0 def load(sha1: str) -> Any: + nonlocal used if sha1 not in cache: text: str | None = None - try: - path = message_path(artifacts_dir, sha1) - text = _read_stored_text(path) if read is None else read(path) - except (OSError, ValueError): - text = None - if text is None and read is None: - # A packed address has no file; the other encoding of the same - # store is a pack slice. A caller-supplied read owns its own - # fallback, because it may bound or confine the read. - text = _packed_text(artifacts_dir, sha1) + remaining = max_total_bytes - used + if remaining > 0: + try: + path = message_path(artifacts_dir, sha1) + text = _read_stored_text(path, remaining) if read is None else read(path) + except (OSError, ValueError): + text = None + if text is not None and len(text.encode("utf-8")) > remaining: + text = None + if text is None and read is None: + # A packed address has no file; the other encoding of the + # same store is a pack slice. A caller-supplied read owns + # its own fallback, because it may bound or confine it. + text = _packed_text(artifacts_dir, sha1, remaining) try: if text is None: raise OSError(f"blob {sha1} could not be read") cache[sha1] = json.loads(text) + used += len(text.encode("utf-8")) except (OSError, ValueError): cache[sha1] = missing(sha1) return cache[sha1] diff --git a/raven/tracing/codec.py b/raven/tracing/codec.py index 32cd40901..c979cae19 100644 --- a/raven/tracing/codec.py +++ b/raven/tracing/codec.py @@ -24,6 +24,13 @@ ZSTD_MAGIC = b"\x28\xb5\x2f\xfd" COMPRESSION_LEVEL = 3 +#: Ceiling on what one stored frame may decode to. The store's writer caps a +#: rotated log at 50 MiB and the largest captured payload observed is 32.9 MB, +#: so this is headroom rather than a routine limit. It exists because a trace +#: store can be imported from another machine: a small frame that expands to +#: gigabytes must be refused, not allocated. +MAX_PLAINTEXT_BYTES = 256 * 1024 * 1024 + def is_compressed(data: bytes) -> bool: return data[:4] == ZSTD_MAGIC @@ -47,12 +54,20 @@ def decompress_bytes(data: bytes, limit: int | None = None) -> bytes: return reader.read() if limit is None else reader.read(limit) -def read_plaintext(data: bytes) -> bytes | None: - """The payload's plaintext, or None when a compressed frame is unreadable.""" +def read_plaintext(data: bytes, limit: int = MAX_PLAINTEXT_BYTES) -> bytes | None: + """The payload's plaintext, or None when unreadable or larger than ``limit``. + + A compressed frame is decoded only up to ``limit + 1`` bytes, so a crafted + frame is refused instead of allocated. Plain bytes were already read by the + caller and are returned as they are. + """ + if not is_compressed(data): + return data try: - return decompress_bytes(data) if is_compressed(data) else data + plain = decompress_bytes(data, limit + 1) except zstandard.ZstdError: return None + return None if len(plain) > limit else plain def plaintext_size(path: Path) -> int | None: @@ -78,21 +93,21 @@ def plaintext_size(path: Path) -> int | None: return None if plain is None else len(plain) -def read_plaintext_file(path: Path) -> bytes | None: +def read_plaintext_file(path: Path, limit: int = MAX_PLAINTEXT_BYTES) -> bytes | None: """A stored file's plaintext, or None when it cannot be read or decoded.""" try: - return read_plaintext(path.read_bytes()) + return read_plaintext(path.read_bytes(), limit) except OSError: return None -def open_plaintext(path: Path) -> BinaryIO: +def open_plaintext(path: Path, limit: int = MAX_PLAINTEXT_BYTES) -> BinaryIO: """Open a stored file as a plaintext byte stream. A caller that seeks or sizes the stream works in plaintext offsets, which is the only coordinate the store records (a span log's cursor, an - artifact's range). A compressed payload is decompressed whole: these files - are bounded by the writer's own rotation cap. + artifact's range). A compressed payload is decompressed whole, up to + ``limit`` bytes; a frame that expands past it is refused as unreadable. """ handle = path.open("rb") try: @@ -106,13 +121,17 @@ def open_plaintext(path: Path) -> BinaryIO: raise handle.close() try: - return io.BytesIO(decompress_bytes(data)) + plain = decompress_bytes(data, limit + 1) except zstandard.ZstdError as exc: raise OSError(f"corrupt compressed payload: {path}") from exc + if len(plain) > limit: + raise OSError(f"compressed payload over {limit} bytes: {path}") + return io.BytesIO(plain) __all__ = [ "COMPRESSION_LEVEL", + "MAX_PLAINTEXT_BYTES", "ZSTD_MAGIC", "compress_bytes", "decompress_bytes", diff --git a/raven/tracing/message_pack.py b/raven/tracing/message_pack.py index 4ebc42c1d..5646f9179 100644 --- a/raven/tracing/message_pack.py +++ b/raven/tracing/message_pack.py @@ -34,7 +34,7 @@ import zstandard -from .codec import compress_bytes, read_plaintext +from .codec import MAX_PLAINTEXT_BYTES, compress_bytes, read_plaintext PACKS_DIR_NAME = "_message_packs" MESSAGES_DIR_NAME = "_messages" @@ -176,6 +176,9 @@ def _slice(artifacts_dir: Path, sha1: str, limit: int | None) -> bytes | None: return None name, offset, length = located want = length if limit is None else min(length, limit) + # The index can be imported with the store; an offset and length pair that + # asks for gigabytes is refused rather than allocated. + want = min(want, MAX_PLAINTEXT_BYTES) pack = packs_dir(artifacts_dir) / name plain = _cached_pack(pack) if plain is None: diff --git a/tests/test_tracing_message_pack.py b/tests/test_tracing_message_pack.py index 59a6e0e61..c17e8b2fd 100644 --- a/tests/test_tracing_message_pack.py +++ b/tests/test_tracing_message_pack.py @@ -68,6 +68,18 @@ def test_pack_moves_files_into_a_pack_and_every_reader_resolves(tmp_path: Path) assert found[0] == texts[sha1s[0]] +def test_a_shell_stops_resolving_after_its_total_budget(tmp_path: Path) -> None: + refs = _publish(tmp_path) + artifacts = tmp_path / "logs" / "audit-artifacts" + first = len(artifact_v2.message_text(MESSAGES[0]).encode()) + shell = {"artifactFormat": artifact_v2.ARTIFACT_FORMAT, "messages": refs} + + resolved = artifact_v2.resolve_payload(shell, artifacts, max_total_bytes=first + 1) + + assert resolved["messages"][0] == MESSAGES[0] + assert "missing" in resolved["messages"][1]["content"] + + def test_a_small_target_splits_messages_across_packs(tmp_path: Path) -> None: refs = _publish(tmp_path) artifacts = tmp_path / "logs" / "audit-artifacts" diff --git a/tests/test_tracing_store_compression.py b/tests/test_tracing_store_compression.py index f8d952755..10362e100 100644 --- a/tests/test_tracing_store_compression.py +++ b/tests/test_tracing_store_compression.py @@ -13,9 +13,11 @@ import os from pathlib import Path +import pytest import zstandard from raven.tracing import artifact_v2 +from raven.tracing import codec as trace_codec from raven.tracing.codec import is_compressed, plaintext_size from raven.tracing.compact import compact from raven.tracing.compress import compress, compress_logs @@ -204,6 +206,18 @@ def test_a_corrupt_frame_is_reported_not_raised(tmp_path: Path) -> None: assert reason is not None and "corrupt" in reason +def test_decompression_is_bounded(tmp_path: Path) -> None: + frame = zstandard.ZstdCompressor().compress(b"x" * 4096) + assert trace_codec.read_plaintext(frame) == b"x" * 4096 + assert trace_codec.read_plaintext(frame, 1024) is None + + path = tmp_path / "bomb" + path.write_bytes(frame) + assert trace_codec.read_plaintext_file(path, 1024) is None + with pytest.raises(OSError): + trace_codec.open_plaintext(path, 1024) + + def test_compact_records_a_corrupt_frame(tmp_path: Path) -> None: bad = tmp_path / "logs" / "audit-artifacts" / "llm.input" / "2026-01-01" / "bad.json" bad.parent.mkdir(parents=True) From bf63a1b39086019cee226b22ba2fa7c1b54ddf15 Mon Sep 17 00:00:00 2001 From: Dizhan Xue Date: Thu, 29 Oct 2026 16:50:00 +0000 Subject: [PATCH 05/23] fix(tracing): confine pack names before reading them A pack index ships with an imported store and is untrusted. A name that is not a generated pack basename is refused before it is joined under the pack directory, and the range fields must be non-negative integers. The viewer applies the same validation before it opens a pack. --- raven/tracing/message_pack.py | 8 +++++++- tests/test_tracing_message_pack.py | 17 +++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/raven/tracing/message_pack.py b/raven/tracing/message_pack.py index 5646f9179..39c924090 100644 --- a/raven/tracing/message_pack.py +++ b/raven/tracing/message_pack.py @@ -52,6 +52,11 @@ CACHE_BYTES = 32 * 1024 * 1024 _SHA1_RE = re.compile(r"^[0-9a-f]{40}$") +#: A pack name is generated by this module, never taken from a payload. The +#: index lives in the store and can be imported with it, so a name with a path +#: separator or ``..`` must be refused before it is joined under the pack +#: directory. +_PACK_NAME_RE = re.compile(r"^[0-9a-f]{32}\.pack\.zst$") def packs_dir(artifacts_dir: Path | str) -> Path: @@ -110,7 +115,8 @@ def load_index(artifacts_dir: Path | str) -> _Index | None: isinstance(entry, list) and len(entry) == 3 and isinstance(entry[0], str) - and all(isinstance(value, int) and not isinstance(value, bool) for value in entry[1:]) + and _PACK_NAME_RE.match(entry[0]) + and all(isinstance(value, int) and not isinstance(value, bool) and value >= 0 for value in entry[1:]) ): messages[sha1] = (entry[0], entry[1], entry[2]) packs: dict[str, int] = {} diff --git a/tests/test_tracing_message_pack.py b/tests/test_tracing_message_pack.py index c17e8b2fd..ed0c94d83 100644 --- a/tests/test_tracing_message_pack.py +++ b/tests/test_tracing_message_pack.py @@ -80,6 +80,23 @@ def test_a_shell_stops_resolving_after_its_total_budget(tmp_path: Path) -> None: assert "missing" in resolved["messages"][1]["content"] +def test_a_pack_name_that_escapes_the_store_is_refused(tmp_path: Path) -> None: + refs = _publish(tmp_path) + artifacts = tmp_path / "logs" / "audit-artifacts" + message_pack.pack_messages(artifacts) + secret = tmp_path / "secret.json" + secret.write_text('{"secret": true}', encoding="utf-8") + index_file = message_pack.index_path(artifacts) + data = json.loads(index_file.read_text(encoding="utf-8")) + data["messages"][refs[0]["$msg"]] = ["../../../../secret.json", 0, 16] + index_file.write_text(json.dumps(data), encoding="utf-8") + + assert message_pack.read_message(artifacts, refs[0]["$msg"]) is None + shell = {"artifactFormat": artifact_v2.ARTIFACT_FORMAT, "messages": [refs[0]]} + resolved = artifact_v2.resolve_payload(shell, artifacts) + assert "missing" in resolved["messages"][0]["content"] + + def test_a_small_target_splits_messages_across_packs(tmp_path: Path) -> None: refs = _publish(tmp_path) artifacts = tmp_path / "logs" / "audit-artifacts" From 28a2d20a6629bb7c915856bdd2a66b874d585476 Mon Sep 17 00:00:00 2001 From: Dizhan Xue Date: Thu, 29 Oct 2026 16:50:00 +0000 Subject: [PATCH 06/23] fix(trajectory): decode compressed turn inputs before matching A turn input written after compression is a zstd frame, but the legacy turn matcher handed its raw bytes to json.loads and found nothing. It now uses the bounded plaintext decoder, so pre-Turn-Trace-Mark sessions still join their conversation messages. --- raven/trajectory/selection.py | 24 ++++++++++++++++++------ tests/test_trajectory_selection.py | 21 +++++++++++++++++++++ 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/raven/trajectory/selection.py b/raven/trajectory/selection.py index c6eed3f65..0d2cfddc5 100644 --- a/raven/trajectory/selection.py +++ b/raven/trajectory/selection.py @@ -39,7 +39,7 @@ from raven.tracing import artifact_v2 from raven.tracing import config as tracing_config -from raven.tracing.codec import read_plaintext_file +from raven.tracing.codec import ZSTD_MAGIC, read_plaintext, read_plaintext_file from raven.trajectory import entries as _entries from raven.trajectory import store as tstore @@ -461,11 +461,23 @@ def _turn_input(turn: TurnSummary, state_dir: Path) -> str | None: return None try: with target.open("rb") as handle: - data = handle.read(_INPUT_READ_LIMIT + 1) - if len(data) > _INPUT_READ_LIMIT: - return None - payload = json.loads(data.decode("utf-8")) - except (OSError, UnicodeDecodeError, ValueError): + head = handle.read(4) + handle.seek(0) + if head == ZSTD_MAGIC: + # A turn input written after compression is a zstd frame; the + # bounded decoder is the same one every other artifact reader + # uses. A plain file keeps the bounded read it always had. + plain = read_plaintext(handle.read(), _INPUT_READ_LIMIT) + else: + data = handle.read(_INPUT_READ_LIMIT + 1) + plain = data if len(data) <= _INPUT_READ_LIMIT else None + except OSError: + return None + if plain is None: + return None + try: + payload = json.loads(plain.decode("utf-8")) + except (UnicodeDecodeError, ValueError): return None payload = artifact_v2.resolve_payload(payload, state_dir / "logs" / "audit-artifacts") return _text_of(payload.get("content")) if isinstance(payload, dict) else None diff --git a/tests/test_trajectory_selection.py b/tests/test_trajectory_selection.py index a0084de75..858eb7101 100644 --- a/tests/test_trajectory_selection.py +++ b/tests/test_trajectory_selection.py @@ -120,6 +120,27 @@ def _plan(*trace_ids: str, answering: bool = False) -> sel.SelectionPlan: # -- main-line turns --------------------------------------------------------- +def test_turn_input_reads_a_compressed_artifact(state): + from raven.tracing.codec import compress_bytes + + artifact = state / "logs" / "audit-artifacts" / "turn.input" / "2026-10-01" / "a.json" + artifact.parent.mkdir(parents=True) + artifact.write_bytes(compress_bytes(json.dumps({"content": "hello"}).encode("utf-8"))) + turn = sel.TurnSummary( + trace_id="t", + span_id="s", + number=1, + start=_ts(0), + end=None, + input_preview="", + in_progress=False, + input_artifact=str(artifact), + main_line=True, + ) + + assert sel._turn_input(turn, state) == "hello" + + def test_main_line_turns_leave_out_dispatched_and_foreign_turns(state): _write( state, From ddc6f9d16e2c56258ccabe69a7d47393fd91d1d4 Mon Sep 17 00:00:00 2001 From: Dizhan Xue Date: Thu, 29 Oct 2026 16:50:00 +0000 Subject: [PATCH 07/23] fix(cli): require Node 22.15 for the tracing viewer The viewer reads zstd-compressed payloads by default, and zstd landed in Node's zlib in 22.15. The launcher accepted any Node 22 and then showed empty payloads on 22.0 through 22.14; it now refuses those versions and names the real floor. --- raven/cli/tracing_commands.py | 12 ++++++++---- tests/test_cli_tracing_commands.py | 7 +++++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/raven/cli/tracing_commands.py b/raven/cli/tracing_commands.py index 95198c059..1ba635414 100644 --- a/raven/cli/tracing_commands.py +++ b/raven/cli/tracing_commands.py @@ -42,6 +42,10 @@ console = Console() +#: zstd decoding landed in Node's zlib in 22.15, and the viewer reads payloads +#: compressed by default. Its floor is therefore higher than the TUI's. +_MIN_VIEWER_NODE_VERSION = (22, 15, 0) + def _viewer_dir() -> Path: return Path(__file__).resolve().parent / "tracing_viewer" @@ -190,15 +194,15 @@ def _stop_viewer() -> None: def _resolve_node() -> str: - from raven.cli.tui_commands import _MIN_NODE_VERSION, find_node + from raven.cli.tui_commands import find_node node, version = find_node() # find_node returns the best node it saw even below the minimum, so the # version check is the caller's -- the TUI's own launch does the same. - if not node or version is None or version < _MIN_NODE_VERSION: + if not node or version is None or version < _MIN_VIEWER_NODE_VERSION: console.print( - "[red]Node (>= 22) not found.[/red] The tracing dashboard needs the " - "same Node runtime as the TUI.\n" + "[red]Node (>= 22.15) not found.[/red] The tracing dashboard reads zstd-compressed " + "payloads and needs a Node with zlib zstd support.\n" "Install: https://nodejs.org/ or brew install node@22 or nvm install 22" ) raise typer.Exit(1) diff --git a/tests/test_cli_tracing_commands.py b/tests/test_cli_tracing_commands.py index 02e080f86..365117896 100644 --- a/tests/test_cli_tracing_commands.py +++ b/tests/test_cli_tracing_commands.py @@ -394,15 +394,18 @@ def test_tracing_rejects_an_unknown_action_with_bracketed_markup(tmp_path, monke def test_a_below_minimum_node_is_refused(monkeypatch): """find_node deliberately returns the best node it saw even below the - minimum; the viewer used to discard the version and run on it anyway.""" + minimum; the viewer reads zstd-compressed payloads, whose decoder landed + in Node's zlib in 22.15.""" import pytest import typer import raven.cli.tracing_commands as tc - monkeypatch.setattr("raven.cli.tui_commands.find_node", lambda: ("/usr/bin/node", (20, 0, 0))) + monkeypatch.setattr("raven.cli.tui_commands.find_node", lambda: ("/usr/bin/node", (22, 14, 0))) with pytest.raises(typer.Exit): tc._resolve_node() + monkeypatch.setattr("raven.cli.tui_commands.find_node", lambda: ("/usr/bin/node", (22, 15, 0))) + assert tc._resolve_node() == "/usr/bin/node" def _plain_blob_with_link(state_dir: Path, text: str) -> tuple[Path, Path]: From e550c32af6176ecb5dbca003bde856cf4baf30dd Mon Sep 17 00:00:00 2001 From: Dizhan Xue Date: Thu, 29 Oct 2026 16:50:00 +0000 Subject: [PATCH 08/23] fix(cli): decode stored payloads in spawn synthesis sessions_spawn synthesis read tool artifacts with a raw text read, so a compressed input or output artifact always parsed to null and the dispatch was silently omitted. It now uses the stored-payload decoder, which picks the encoding from the frame. --- raven/cli/tracing_viewer/server.js | 9 ++++- tests/integration/test_tracing_viewer_e2e.py | 36 ++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/raven/cli/tracing_viewer/server.js b/raven/cli/tracing_viewer/server.js index f6043b1b8..fbe37b8a0 100644 --- a/raven/cli/tracing_viewer/server.js +++ b/raven/cli/tracing_viewer/server.js @@ -111,7 +111,9 @@ function readJsonFileMaybe(filePath) { if (!filePath || typeof filePath !== 'string') return null; if (!fs.existsSync(filePath)) return null; try { - return JSON.parse(fs.readFileSync(filePath, 'utf8')); + // Persisted artifacts are stored encoded; readStoredText picks the + // encoding from the frame, so a compressed artifact still parses here. + return JSON.parse(readStoredText(filePath)); } catch { return null; } @@ -1099,7 +1101,12 @@ function readPackedMessage(sha1) { const index = messagePackIndex(); const entry = index && index.messages ? index.messages[sha1] : null; if (!Array.isArray(entry) || entry.length !== 3) return null; + // The index travels with an imported store; a name is a generated basename, + // never a path, and the range fields are non-negative integers. + if (typeof entry[0] !== 'string' || !/^[0-9a-f]{32}\.pack\.zst$/.test(entry[0])) return null; + if (!Number.isInteger(entry[1]) || !Number.isInteger(entry[2]) || entry[1] < 0 || entry[2] < 0) return null; const packPath = path.join(MESSAGE_PACKS_DIR, entry[0]); + if (!path.resolve(packPath).startsWith(path.resolve(MESSAGE_PACKS_DIR) + path.sep)) return null; let plain = packCache.get(packPath); if (!plain) { try { diff --git a/tests/integration/test_tracing_viewer_e2e.py b/tests/integration/test_tracing_viewer_e2e.py index 10c6808aa..210f0fae1 100644 --- a/tests/integration/test_tracing_viewer_e2e.py +++ b/tests/integration/test_tracing_viewer_e2e.py @@ -194,6 +194,42 @@ def test_artifact_endpoint_resolves_a_packed_message(tmp_path): assert found["parsed"]["prompt"] == "packed" +def test_sessions_spawn_synthesis_reads_a_compressed_artifact(tmp_path): + from raven.tracing.codec import compress_bytes + + logs = tmp_path / "logs" + artifacts = logs / "audit-artifacts" / "tool" / "2026-08-01" + artifacts.mkdir(parents=True) + input_artifact = artifacts / "in.json" + output_artifact = artifacts / "out.json" + input_artifact.write_bytes( + compress_bytes( + json.dumps({"params": {"runtime": "subagent", "agentId": "raven", "task": "do"}}).encode("utf-8") + ) + ) + output_artifact.write_bytes( + compress_bytes(json.dumps({"result": {"status": "accepted", "childSessionKey": "child"}}).encode("utf-8")) + ) + span = _span("live-session", "span-spawn", name="tool.call") + span["attributes"].update( + { + "tool.name": "sessions_spawn", + "tool.call_id": "call-1", + "tool.args_preview": "[truncated]", + "tool.result_preview": "[truncated]", + "tool.input.artifact_path": str(input_artifact), + "tool.output.artifact_path": str(output_artifact), + } + ) + _write_spans(logs / "audit-spans.log", [span]) + + with _viewer(tmp_path) as port: + payload = _get(port, "/api/data") + + names = {s["name"] for session in payload["sessions"] for trace in session["traces"] for s in trace["spans"]} + assert "subagent.call" in names + + def test_ignores_a_log_planted_outside_the_active_and_archive_paths(tmp_path): logs = tmp_path / "logs" _write_spans(logs / "audit-spans.log", [_span("live-session", "span-live")]) From 5a2715f558bb691497092207181c0cf8d1788d65 Mon Sep 17 00:00:00 2001 From: Dizhan Xue Date: Thu, 29 Oct 2026 17:00:54 +0000 Subject: [PATCH 09/23] test(tracing): cover the compressed store's failure branches Exercise the index write failure, an unremovable message file, an orphan pack sweep, an already-compressed rotated log, a corrupt blob, and the readers that must degrade to missing inside a confined root. --- tests/test_tracing_compression_edges.py | 177 ++++++++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 tests/test_tracing_compression_edges.py diff --git a/tests/test_tracing_compression_edges.py b/tests/test_tracing_compression_edges.py new file mode 100644 index 000000000..b1b8bc449 --- /dev/null +++ b/tests/test_tracing_compression_edges.py @@ -0,0 +1,177 @@ +"""Edge paths of the compressed trace store, exercised directly. + +The happy paths live with their modules; these pin the branches a migration or +a damaged store reaches: an index write that fails, a file that cannot be +removed, a compressed log that is already compressed, and readers that must +degrade to "missing" instead of raising. +""" + +from __future__ import annotations + +from pathlib import Path + +from raven.tracing import artifact_v2, message_pack +from raven.tracing import codec as trace_codec +from raven.tracing.store import TraceStore +from raven.trajectory import bundle, selection +from raven.trajectory import index as tidx + +MAGIC = b"\x28\xb5\x2f\xfd" + + +def _publish(store: TraceStore, content: str = "hi") -> str: + return store.address_items([{"role": "user", "content": content}])[0]["$msg"] + + +def test_path_mtime_and_a_missing_pack(tmp_path: Path) -> None: + store = TraceStore(tmp_path) + sha1 = _publish(store) + message_pack.pack_messages(store.artifacts_dir) + path = artifact_v2.message_path(store.artifacts_dir, sha1) + assert message_pack.path_mtime(path) is not None + + next(message_pack.packs_dir(store.artifacts_dir).glob("*.pack.zst")).unlink() + assert message_pack.path_mtime(path) is None + + +def test_pack_without_a_message_directory(tmp_path: Path) -> None: + result = message_pack.pack_messages(tmp_path / "logs" / "audit-artifacts") + assert result.scanned == 0 + + +def test_pack_skips_a_file_that_is_not_a_message(tmp_path: Path) -> None: + store = TraceStore(tmp_path) + _publish(store) + (store.messages_dir / "notes.txt").write_text("x", encoding="utf-8") + + result = message_pack.pack_messages(store.artifacts_dir) + + assert result.packed == 1 + + +def test_pack_index_write_failure_keeps_the_files(tmp_path: Path, monkeypatch) -> None: + store = TraceStore(tmp_path) + sha1 = _publish(store) + path = artifact_v2.message_path(store.artifacts_dir, sha1) + + def boom(*args, **kwargs): + raise OSError("disk full") + + monkeypatch.setattr(message_pack, "_atomic_write_index", boom) + result = message_pack.pack_messages(store.artifacts_dir) + + assert result.errors + assert path.exists() + + +def test_pack_removes_an_orphan_pack(tmp_path: Path) -> None: + store = TraceStore(tmp_path) + _publish(store) + message_pack.pack_messages(store.artifacts_dir) + orphan = message_pack.packs_dir(store.artifacts_dir) / ("0" * 32 + ".pack.zst") + orphan.write_bytes(b"orphan") + _publish(store, "second") + + message_pack.pack_messages(store.artifacts_dir) + + assert not orphan.exists() + + +def test_pack_reports_a_file_it_cannot_remove(tmp_path: Path, monkeypatch) -> None: + store = TraceStore(tmp_path) + sha1 = _publish(store) + path = artifact_v2.message_path(store.artifacts_dir, sha1) + original = Path.unlink + + def failing(self, *args, **kwargs): + if self == path: + raise OSError("busy") + return original(self, *args, **kwargs) + + monkeypatch.setattr(Path, "unlink", failing) + result = message_pack.pack_messages(store.artifacts_dir) + + assert any("busy" in error for error in result.errors) + + +def test_blob_intact_rejects_a_corrupt_frame(tmp_path: Path) -> None: + store = TraceStore(tmp_path) + blob = store._blob_path("a" * 40, "json") + blob.parent.mkdir(parents=True) + blob.write_bytes(MAGIC + b" broken") + + assert store._blob_is_intact(blob, "a" * 40) is False + + +def test_compress_rotated_skips_a_compressed_log(tmp_path: Path) -> None: + store = TraceStore(tmp_path) + path = tmp_path / "rotated.log" + path.write_bytes(trace_codec.compress_bytes(b"line\n")) + before = path.read_bytes() + + store._compress_rotated(path) + + assert path.read_bytes() == before + + +def test_compress_rotated_swallows_a_missing_file(tmp_path: Path) -> None: + store = TraceStore(tmp_path) + store._compress_rotated(tmp_path / "missing.log") + + +def test_stored_bytes_of_a_missing_file_is_none(tmp_path: Path) -> None: + assert bundle._stored_bytes(tmp_path / "gone.json") is None + + +def test_copy_resolved_rejects_invalid_json(tmp_path: Path) -> None: + source = tmp_path / "src.json" + source.write_text("{not json", encoding="utf-8") + + assert bundle._copy_resolved(source, tmp_path / "out.json", tmp_path / "artifacts", []) is False + + +def test_plaintext_size_cache_hit_and_unreadable_file(tmp_path: Path) -> None: + logs = tmp_path / "logs" + logs.mkdir() + (logs / "audit-spans.log").write_text('{"spanId": "s"}\n', encoding="utf-8") + # A name the chain lists but whose plaintext size cannot be read is skipped. + (logs / "archive" / "2026-01-01" / "audit-spans-1.log").mkdir(parents=True) + + scanner = tidx.SpanLogScanner(tmp_path) + first = scanner._catalog() + second = scanner._catalog() + + assert first == second + assert [path.name for path, *_ in first] == ["audit-spans.log"] + + +def test_a_packed_read_outside_the_root_is_refused(tmp_path: Path) -> None: + store = TraceStore(tmp_path / "store") + sha1 = _publish(store) + message_pack.pack_messages(store.artifacts_dir) + path = artifact_v2.message_path(store.artifacts_dir, sha1) + outside = tmp_path / "elsewhere" + outside.mkdir() + + assert selection.read_store_file(path, outside) is None + + +def test_turn_input_degrades_on_bad_json(tmp_path: Path) -> None: + state = tmp_path / "state" + (state / "logs").mkdir(parents=True) + artifact = state / "logs" / "audit-artifacts" / "turn.input" / "2026-10-01" / "bad.json" + artifact.parent.mkdir(parents=True) + artifact.write_text("{not json", encoding="utf-8") + turn = selection.TurnSummary( + trace_id="t", + span_id="s", + number=1, + start="2026-10-01T10:00:00+00:00", + end=None, + input_preview="", + in_progress=False, + input_artifact=str(artifact), + main_line=True, + ) + + assert selection._turn_input(turn, state) is None From 0f696157040eeed8323a12173935a5ce1f4cb6c0 Mon Sep 17 00:00:00 2001 From: Dizhan Xue Date: Thu, 29 Oct 2026 17:09:41 +0000 Subject: [PATCH 10/23] test(tracing): cover the pack reader and turn-input edges Exercise the streaming slice, an out-of-range index entry, relative message paths, an address that was never packed, hidden files, an unremovable orphan, the bundle import fallback, and the turn-input OSError and oversize branches. --- tests/test_tracing_compression_edges.py | 134 ++++++++++++++++++++++++ 1 file changed, 134 insertions(+) diff --git a/tests/test_tracing_compression_edges.py b/tests/test_tracing_compression_edges.py index b1b8bc449..9270a0c3e 100644 --- a/tests/test_tracing_compression_edges.py +++ b/tests/test_tracing_compression_edges.py @@ -8,6 +8,7 @@ from __future__ import annotations +import json from pathlib import Path from raven.tracing import artifact_v2, message_pack @@ -175,3 +176,136 @@ def test_turn_input_degrades_on_bad_json(tmp_path: Path) -> None: ) assert selection._turn_input(turn, state) is None + + +def test_stream_slice_and_slice_fallbacks(tmp_path: Path, monkeypatch) -> None: + store = TraceStore(tmp_path / "store") + sha1 = _publish(store) + message_pack.pack_messages(store.artifacts_dir) + pack = next(message_pack.packs_dir(store.artifacts_dir).glob("*.pack.zst")) + assert message_pack._stream_slice(pack, 0, 5) is not None + assert message_pack._stream_slice(pack, 10**9, 1) is None + assert message_pack._stream_slice(tmp_path / "missing", 0, 1) is None + + monkeypatch.setattr(message_pack, "_cached_pack", lambda path: None) + assert message_pack.read_message(store.artifacts_dir, sha1) is not None + + +def test_slice_refuses_an_out_of_range_entry(tmp_path: Path) -> None: + store = TraceStore(tmp_path) + sha1 = _publish(store) + message_pack.pack_messages(store.artifacts_dir) + index_file = message_pack.index_path(store.artifacts_dir) + data = json.loads(index_file.read_text(encoding="utf-8")) + data["messages"][sha1] = [data["messages"][sha1][0], 10**9, 10] + index_file.write_text(json.dumps(data), encoding="utf-8") + + assert message_pack.read_message(store.artifacts_dir, sha1) is None + + +def test_address_and_path_validation() -> None: + assert message_pack.read_message("/tmp/none", "not-a-sha") is None + assert message_pack.sha1_from_path("x.txt") is None + assert message_pack.sha1_from_path("_messages/zz/" + "a" * 40 + ".json") is None + + +def test_relative_message_paths_are_resolved(tmp_path: Path, monkeypatch) -> None: + store = TraceStore(tmp_path / "store") + sha1 = _publish(store) + message_pack.pack_messages(store.artifacts_dir) + path = artifact_v2.message_path(store.artifacts_dir, sha1) + monkeypatch.chdir(tmp_path) + relative = path.relative_to(tmp_path) + + assert message_pack.read_path(relative) is not None + assert message_pack.size_path(relative) is not None + assert message_pack.path_mtime(relative) is not None + + +def test_path_mtime_of_an_address_that_was_not_packed(tmp_path: Path) -> None: + store = TraceStore(tmp_path) + sha1 = "b" * 40 + assert message_pack.path_mtime(artifact_v2.message_path(store.artifacts_dir, sha1)) is None + + +def test_walk_skips_hidden_files(tmp_path: Path) -> None: + store = TraceStore(tmp_path) + _publish(store) + (store.messages_dir / ".hidden").write_text("x", encoding="utf-8") + + result = message_pack.pack_messages(store.artifacts_dir) + + assert result.packed == 1 + + +def test_orphan_sweep_ignores_an_entry_it_cannot_remove(tmp_path: Path) -> None: + store = TraceStore(tmp_path) + _publish(store) + message_pack.pack_messages(store.artifacts_dir) + orphan = message_pack.packs_dir(store.artifacts_dir) / ("0" * 32 + ".pack.zst") + orphan.mkdir() + _publish(store, "second") + + result = message_pack.pack_messages(store.artifacts_dir) + + assert result.packed == 1 + assert orphan.is_dir() + + +def test_import_artifact_reports_a_file_it_cannot_read(tmp_path: Path, monkeypatch) -> None: + source = tmp_path / "src.bin" + source.write_bytes(b"raw") + monkeypatch.setattr(bundle, "_stored_bytes", lambda path: None) + missing: list[str] = [] + + result = bundle._import_artifact(str(source), tmp_path / "out", {}, missing, {}, tmp_path / "artifacts", []) + + assert result is None + assert missing == [str(source)] + + +def test_copy_resolved_rejects_invalid_json_again(tmp_path: Path) -> None: + source = tmp_path / "src.json" + source.write_text("[1, 2", encoding="utf-8") + + assert bundle._copy_resolved(source, tmp_path / "out.json", tmp_path / "artifacts", []) is False + + +def _turn(tmp_path: Path, artifact: Path) -> selection.TurnSummary: + return selection.TurnSummary( + trace_id="t", + span_id="s", + number=1, + start="2026-10-01T10:00:00+00:00", + end=None, + input_preview="", + in_progress=False, + input_artifact=str(artifact), + main_line=True, + ) + + +def test_turn_input_degrades_when_the_file_vanishes(tmp_path: Path, monkeypatch) -> None: + state = tmp_path / "state" + artifact = state / "logs" / "audit-artifacts" / "turn.input" / "2026-10-01" / "a.json" + artifact.parent.mkdir(parents=True) + artifact.write_text('{"content": "hi"}', encoding="utf-8") + original = Path.open + + def failing(self, *args, **kwargs): + if self == artifact: + raise OSError("gone") + return original(self, *args, **kwargs) + + monkeypatch.setattr(Path, "open", failing) + + assert selection._turn_input(_turn(tmp_path, artifact), state) is None + + +def test_turn_input_degrades_on_an_oversized_plain_artifact(tmp_path: Path) -> None: + state = tmp_path / "state" + artifact = state / "logs" / "audit-artifacts" / "turn.input" / "2026-10-01" / "big.json" + artifact.parent.mkdir(parents=True) + artifact.write_text("x" * (selection._INPUT_READ_LIMIT + 1), encoding="utf-8") + + assert selection._turn_input(_turn(tmp_path, artifact), state) is None From f91a8aaff7eb6cc545fd1678577d69297bb8ab20 Mon Sep 17 00:00:00 2001 From: Dizhan Xue Date: Fri, 30 Oct 2026 04:48:50 +0000 Subject: [PATCH 11/23] fix(tracing): measure a frame without a content size from its start plaintext_size() read the first 18 bytes to ask the frame header for the content size and, when the field was absent, decoded what was left on the handle: a slice starting 18 bytes into the frame. read_plaintext() then saw no zstd magic, treated those bytes as plaintext, and returned the remaining file size as the uncompressed length. A span log written without a content size was catalogued with that wrong size, so SpanLogScanner reported generation_changed on every other pass and SessionIndex reset its entries. Rewind before the fallback decode. Test: tests/test_tracing_compression_edges.py adds a direct length check (12000-byte payload measured as 11 before the fix) and a scanner pass pair (second scan reported generation_changed before the fix). --- raven/tracing/codec.py | 2 ++ tests/test_tracing_compression_edges.py | 27 +++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/raven/tracing/codec.py b/raven/tracing/codec.py index c979cae19..5c7efc083 100644 --- a/raven/tracing/codec.py +++ b/raven/tracing/codec.py @@ -86,6 +86,8 @@ def plaintext_size(path: Path) -> int | None: return int(size) # A frame written without a content size is only measurable by # decoding it; the store's own writer always records the size. + # `read(18)` above consumed the header, so rewind before decoding. + handle.seek(0) data = handle.read() except OSError: return None diff --git a/tests/test_tracing_compression_edges.py b/tests/test_tracing_compression_edges.py index 9270a0c3e..a43aafad7 100644 --- a/tests/test_tracing_compression_edges.py +++ b/tests/test_tracing_compression_edges.py @@ -11,6 +11,8 @@ import json from pathlib import Path +import zstandard + from raven.tracing import artifact_v2, message_pack from raven.tracing import codec as trace_codec from raven.tracing.store import TraceStore @@ -309,3 +311,28 @@ def test_turn_input_degrades_on_an_oversized_plain_artifact(tmp_path: Path) -> N artifact.write_text("x" * (selection._INPUT_READ_LIMIT + 1), encoding="utf-8") assert selection._turn_input(_turn(tmp_path, artifact), state) is None + + +def test_plaintext_size_decodes_a_frame_without_a_content_size(tmp_path: Path) -> None: + payload = b"x" * 4096 + path = tmp_path / "audit-spans-2026-01-01-1.log" + path.write_bytes(zstandard.ZstdCompressor(write_content_size=False).compress(payload)) + + assert trace_codec.plaintext_size(path) == len(payload) + + +def test_a_frame_without_a_content_size_does_not_reset_the_scanner(tmp_path: Path) -> None: + archive = tmp_path / "logs" / "archive" / "2026-01-01" / "audit-spans-2026-01-01-1.log" + archive.parent.mkdir(parents=True) + record = {"traceId": "t", "spanId": "s", "name": "tool.call", "attributes": {}} + archive.write_bytes( + zstandard.ZstdCompressor(write_content_size=False).compress(json.dumps(record).encode() + b"\n") + ) + + scanner = tidx.SpanLogScanner(tmp_path) + first = scanner.scan(tidx.ScanBudget(max_bytes=10_000, deadline=None)) + second = scanner.scan(tidx.ScanBudget(max_bytes=10_000, deadline=None)) + + assert [raw.span["spanId"] for raw in first.records] == ["s"] + assert second.generation_changed is False + assert second.done is True From 28626cbb58b9186d5b13ee9d976ab649ec07cf54 Mon Sep 17 00:00:00 2001 From: Dizhan Xue Date: Fri, 30 Oct 2026 04:48:54 +0000 Subject: [PATCH 12/23] fix(tracing): degrade a character-split pack slice instead of raising resolve_payload() guarantees that a message whose blob cannot be read resolves to the placeholder, never to an exception. Its packed fallback called _packed_text() outside the try that enforces that: read_message() returns at most `limit` bytes, and a budget that ended inside a multi-byte character made decode("utf-8") raise UnicodeDecodeError, which escaped resolve_payload. _packed_text() now returns None for that decode failure, so the address degrades like a missing one. Test: tests/test_tracing_message_pack.py adds a packed non-ASCII message under a budget cut inside a character; it raised before the fix and resolves to the placeholder now. --- raven/tracing/artifact_v2.py | 12 +++++++++++- tests/test_tracing_message_pack.py | 20 ++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/raven/tracing/artifact_v2.py b/raven/tracing/artifact_v2.py index 29bfd89db..85aab9eba 100644 --- a/raven/tracing/artifact_v2.py +++ b/raven/tracing/artifact_v2.py @@ -121,10 +121,20 @@ def _read_stored_text(path: Path, limit: int = MAX_PLAINTEXT_BYTES) -> str: def _packed_text(artifacts_dir: Path | str, sha1: str, limit: int = MAX_PLAINTEXT_BYTES) -> str | None: + """A packed message's text, or None when it is missing or cut mid-character. + + ``limit`` can end the slice inside a multi-byte character; that is not a + message this reader can hand back, so it degrades like a missing one. + """ from . import message_pack packed = message_pack.read_message(artifacts_dir, sha1, limit) - return None if packed is None else packed.decode("utf-8") + if packed is None: + return None + try: + return packed.decode("utf-8") + except UnicodeDecodeError: + return None def resolve_payload( diff --git a/tests/test_tracing_message_pack.py b/tests/test_tracing_message_pack.py index ed0c94d83..da433d967 100644 --- a/tests/test_tracing_message_pack.py +++ b/tests/test_tracing_message_pack.py @@ -147,3 +147,23 @@ def test_a_corrupt_message_file_is_reported_and_not_removed(tmp_path: Path) -> N assert corrupt.exists() assert not artifact_v2.message_path(artifacts, refs[0]["$msg"]).exists() assert message_pack.read_message(artifacts, refs[0]["$msg"]) is not None + + +def test_a_budget_that_splits_a_character_resolves_to_the_placeholder(tmp_path: Path) -> None: + messages = [ + {"role": "user", "content": "a" * 200}, + {"role": "user", "content": "\u00e9" * 100}, + ] + refs = TraceStore(tmp_path).address_items(messages) + artifacts = tmp_path / "logs" / "audit-artifacts" + first = len(artifact_v2.message_text(messages[0]).encode("utf-8")) + second = artifact_v2.message_text(messages[1]).encode("utf-8") + cut = next(i for i in range(1, len(second)) if 0x80 <= second[i] <= 0xBF) + message_pack.pack_messages(artifacts) + + shell = {"artifactFormat": artifact_v2.ARTIFACT_FORMAT, "messages": refs} + resolved = artifact_v2.resolve_payload(shell, artifacts, max_total_bytes=first + cut) + + assert resolved["messages"][0] == messages[0] + assert "missing" in resolved["messages"][1]["content"] + From e85a7e4ab2e38943a7eb68d2948f72a9e8c5e0ae Mon Sep 17 00:00:00 2001 From: Dizhan Xue Date: Fri, 30 Oct 2026 04:48:56 +0000 Subject: [PATCH 13/23] fix(cli): require Node 23.8 for the tracing viewer's zstd decoder zstd landed in Node's zlib in 22.15 and 23.8. The viewer's floor compared the version tuple against (22, 15, 0), which accepts 23.0 through 23.7. The launcher let those run, zlib.zstdDecompressSync is absent there, and every compressed artifact was shown as empty. The floor is now explicit per major: 22.15+, 23.8+, or 24 and later. Test: tests/test_cli_tracing_commands.py rejects 23.0 and accepts 23.8. --- raven/cli/tracing_commands.py | 18 +++++++++++++----- tests/test_cli_tracing_commands.py | 14 ++++++++++++++ 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/raven/cli/tracing_commands.py b/raven/cli/tracing_commands.py index 1ba635414..1d1152fc0 100644 --- a/raven/cli/tracing_commands.py +++ b/raven/cli/tracing_commands.py @@ -42,9 +42,17 @@ console = Console() -#: zstd decoding landed in Node's zlib in 22.15, and the viewer reads payloads -#: compressed by default. Its floor is therefore higher than the TUI's. +#: zstd decoding landed in Node's zlib in 22.15 and 23.8, and the viewer reads +#: payloads compressed by default. Its floor is therefore higher than the TUI's. _MIN_VIEWER_NODE_VERSION = (22, 15, 0) +_FIRST_VIEWER_NODE_23_VERSION = (23, 8, 0) + + +def _node_supports_zstd(version: tuple[int, ...]) -> bool: + """Whether this Node has zlib zstd: 22.15+, 23.8+, or any later major.""" + if version[0] == 23: + return version >= _FIRST_VIEWER_NODE_23_VERSION + return version >= _MIN_VIEWER_NODE_VERSION def _viewer_dir() -> Path: @@ -199,10 +207,10 @@ def _resolve_node() -> str: node, version = find_node() # find_node returns the best node it saw even below the minimum, so the # version check is the caller's -- the TUI's own launch does the same. - if not node or version is None or version < _MIN_VIEWER_NODE_VERSION: + if not node or version is None or not _node_supports_zstd(version): console.print( - "[red]Node (>= 22.15) not found.[/red] The tracing dashboard reads zstd-compressed " - "payloads and needs a Node with zlib zstd support.\n" + "[red]Node with zstd support not found (22.15+, or 23.8+).[/red] The tracing dashboard reads " + "zstd-compressed payloads and needs a Node with zlib zstd support.\n" "Install: https://nodejs.org/ or brew install node@22 or nvm install 22" ) raise typer.Exit(1) diff --git a/tests/test_cli_tracing_commands.py b/tests/test_cli_tracing_commands.py index 365117896..6f60b5109 100644 --- a/tests/test_cli_tracing_commands.py +++ b/tests/test_cli_tracing_commands.py @@ -408,6 +408,20 @@ def test_a_below_minimum_node_is_refused(monkeypatch): assert tc._resolve_node() == "/usr/bin/node" +def test_node_23_before_23_8_is_refused(monkeypatch): + """zstd landed in Node 22.15 and 23.8, so a 23.x below 23.8 cannot decode.""" + import pytest + import typer + + import raven.cli.tracing_commands as tc + + monkeypatch.setattr("raven.cli.tui_commands.find_node", lambda: ("/usr/bin/node", (23, 0, 0))) + with pytest.raises(typer.Exit): + tc._resolve_node() + monkeypatch.setattr("raven.cli.tui_commands.find_node", lambda: ("/usr/bin/node", (23, 8, 0))) + assert tc._resolve_node() == "/usr/bin/node" + + def _plain_blob_with_link(state_dir: Path, text: str) -> tuple[Path, Path]: sha1 = hashlib.sha1(text.encode("utf-8")).hexdigest() blob = state_dir / "logs" / "audit-artifacts" / "_blobs" / sha1[:2] / f"{sha1}.json" From 74bb3291b6a83b5d95a4566de7c34c41e10ddf97 Mon Sep 17 00:00:00 2001 From: Dizhan Xue Date: Fri, 30 Oct 2026 04:49:00 +0000 Subject: [PATCH 14/23] fix(tracing): bound the pack reader's skip and stream packs over the budget Two mismatches against the stated decode bound: - _stream_slice() decompressed up to the index's `offset` with no bound; only the slice length was capped. An imported index could name a huge offset and make one read decompress the whole pack. An entry whose offset plus wanted length exceeds MAX_PLAINTEXT_BYTES is now refused, so the skip is bounded too. - _cached_pack() returned the plaintext of every pack it could decode, even one larger than CACHE_BYTES; the module docstring and the PR description say such a pack is streamed per read. It now returns None above the budget and the caller streams the slice it needs. The header is read first, so an oversized pack is not decoded whole just to be refused. Tests: a crafted offset is refused without calling _stream_slice(), and a pack above a shrunken cache budget is streamed and still resolves. --- raven/tracing/message_pack.py | 31 ++++++++++++++++++++---------- tests/test_tracing_message_pack.py | 30 +++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 10 deletions(-) diff --git a/raven/tracing/message_pack.py b/raven/tracing/message_pack.py index 39c924090..d271e1b8c 100644 --- a/raven/tracing/message_pack.py +++ b/raven/tracing/message_pack.py @@ -34,7 +34,7 @@ import zstandard -from .codec import MAX_PLAINTEXT_BYTES, compress_bytes, read_plaintext +from .codec import MAX_PLAINTEXT_BYTES, compress_bytes, plaintext_size, read_plaintext PACKS_DIR_NAME = "_message_packs" MESSAGES_DIR_NAME = "_messages" @@ -133,26 +133,34 @@ def load_index(artifacts_dir: Path | str) -> _Index | None: def _cached_pack(path: Path) -> bytes | None: + """A pack's plaintext when it fits the cache budget, else None to stream it. + + A pack larger than the whole budget is never decoded here: the caller + streams to the slice it needs instead, which is the memory the budget + exists to save. + """ global _PACK_CACHE_BYTES with _PACK_LOCK: cached = _PACK_CACHE.get(path) if cached is not None: _PACK_CACHE.move_to_end(path) return cached + size = plaintext_size(path) + if size is None or size > CACHE_BYTES: + return None try: data = path.read_bytes() - plain = read_plaintext(data) + plain = read_plaintext(data, CACHE_BYTES + 1) except OSError: return None - if plain is None: + if plain is None or len(plain) > CACHE_BYTES: return None with _PACK_LOCK: - if len(plain) <= CACHE_BYTES: - _PACK_CACHE[path] = plain - _PACK_CACHE_BYTES += len(plain) - while _PACK_CACHE_BYTES > CACHE_BYTES and len(_PACK_CACHE) > 1: - _, dropped = _PACK_CACHE.popitem(last=False) - _PACK_CACHE_BYTES -= len(dropped) + _PACK_CACHE[path] = plain + _PACK_CACHE_BYTES += len(plain) + while _PACK_CACHE_BYTES > CACHE_BYTES and len(_PACK_CACHE) > 1: + _, dropped = _PACK_CACHE.popitem(last=False) + _PACK_CACHE_BYTES -= len(dropped) return plain @@ -183,8 +191,11 @@ def _slice(artifacts_dir: Path, sha1: str, limit: int | None) -> bytes | None: name, offset, length = located want = length if limit is None else min(length, limit) # The index can be imported with the store; an offset and length pair that - # asks for gigabytes is refused rather than allocated. + # asks for gigabytes is refused rather than allocated. The bound covers + # the skip too: the streaming reader decompresses up to offset + want. want = min(want, MAX_PLAINTEXT_BYTES) + if offset < 0 or length < 0 or offset + want > MAX_PLAINTEXT_BYTES: + return None pack = packs_dir(artifacts_dir) / name plain = _cached_pack(pack) if plain is None: diff --git a/tests/test_tracing_message_pack.py b/tests/test_tracing_message_pack.py index da433d967..0e1cb7374 100644 --- a/tests/test_tracing_message_pack.py +++ b/tests/test_tracing_message_pack.py @@ -167,3 +167,33 @@ def test_a_budget_that_splits_a_character_resolves_to_the_placeholder(tmp_path: assert resolved["messages"][0] == messages[0] assert "missing" in resolved["messages"][1]["content"] + +def test_a_streamed_slice_refuses_an_offset_past_the_decode_bound(tmp_path: Path, monkeypatch) -> None: + refs = _publish(tmp_path) + artifacts = tmp_path / "logs" / "audit-artifacts" + message_pack.pack_messages(artifacts) + index_file = message_pack.index_path(artifacts) + index = json.loads(index_file.read_text(encoding="utf-8")) + entry = index["messages"][refs[0]["$msg"]] + index["messages"][refs[0]["$msg"]] = [entry[0], message_pack.MAX_PLAINTEXT_BYTES, 1] + index_file.write_text(json.dumps(index), encoding="utf-8") + + monkeypatch.setattr(message_pack, "_cached_pack", lambda path: None) + calls: list[tuple] = [] + monkeypatch.setattr(message_pack, "_stream_slice", lambda *args: calls.append(args) or None) + + assert message_pack.read_message(artifacts, refs[0]["$msg"]) is None + assert calls == [] + + +def test_a_pack_over_the_cache_budget_is_streamed(tmp_path: Path, monkeypatch) -> None: + refs = _publish(tmp_path) + artifacts = tmp_path / "logs" / "audit-artifacts" + message_pack.pack_messages(artifacts) + pack = next(message_pack.packs_dir(artifacts).glob("*.pack.zst")) + monkeypatch.setattr(message_pack, "CACHE_BYTES", 1) + + assert message_pack._cached_pack(pack) is None + for ref, message in zip(refs, MESSAGES): + assert json.loads(message_pack.read_message(artifacts, ref["$msg"])) == message + From dae1062690a61389940e9c9e315ecf15b520cd4c Mon Sep 17 00:00:00 2001 From: Dizhan Xue Date: Fri, 30 Oct 2026 04:49:03 +0000 Subject: [PATCH 15/23] fix(tracing): keep a message file whose pack no longer decodes pack_messages() deleted every message file whose sha1 the index held without checking that the named pack still provided the slice. A store copied incompletely, or with a pack deleted or truncated outside this module, keeps a readable message file as the only copy; a run that also packed a new message deleted it, and the address became unreadable. The deletion pass now keeps a file unless the pack decodes whole and its decoded length covers the entry's offset and range. The decoded length, not the frame header's recorded size, because a truncated frame still carries the original size. One read and one decompression per distinct pack, cached for the run however many entries name it. Tests: a missing pack and a truncated pack each leave the restored file in place and still resolving; the healthy case still deletes every packed file. --- raven/tracing/message_pack.py | 33 ++++++++++++++++++++++++ tests/test_tracing_message_pack.py | 41 ++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/raven/tracing/message_pack.py b/raven/tracing/message_pack.py index d271e1b8c..f1f1d45e4 100644 --- a/raven/tracing/message_pack.py +++ b/raven/tracing/message_pack.py @@ -208,6 +208,35 @@ def _slice(artifacts_dir: Path, sha1: str, limit: int | None) -> bytes | None: return plain[offset : offset + want] +def _pack_plaintext_length(artifacts_dir: Path, name: str, lengths: dict[str, int | None]) -> int | None: + """A pack's decoded length, or None when it does not decode whole. + + Cached per run: one read and one decompression per distinct pack, however + many index entries name it. A store copied incompletely can leave a pack + truncated, and the frame header still records the original size then, so + the decoded length is what decides whether an entry still resolves. + """ + if name not in lengths: + try: + plain = read_plaintext((packs_dir(artifacts_dir) / name).read_bytes()) + except OSError: + plain = None + lengths[name] = None if plain is None else len(plain) + return lengths[name] + + +def _slice_is_readable(artifacts_dir: Path, entry: tuple[str, int, int], lengths: dict[str, int | None]) -> bool: + """Whether the pack an index entry names decodes and covers its slice. + + A store can be copied or cleaned outside this module, so an address the + index lists may have no readable slice left; the message file is then the + only copy and must be kept. + """ + name, offset, length = entry + size = _pack_plaintext_length(artifacts_dir, name, lengths) + return size is not None and offset + length <= size + + def read_message(artifacts_dir: Path | str, sha1: str, limit: int | None = None) -> bytes | None: """One packed message's plaintext, or None when the index does not hold it.""" if not _SHA1_RE.match(sha1): @@ -348,6 +377,7 @@ def pack_messages( existing = load_index(artifacts_dir) known = dict(existing.messages) if existing is not None else {} packs = dict(existing.packs) if existing is not None else {} + lengths: dict[str, int | None] = {} added: dict[str, tuple[str, int, int]] = {} pending: list[tuple[str, bytes]] = [] pending_bytes = 0 @@ -413,6 +443,9 @@ def flush() -> None: for path, sha1 in candidates: if sha1 not in known and sha1 not in added: continue + entry = known.get(sha1) or added.get(sha1) + if entry is not None and not _slice_is_readable(artifacts_dir, entry, lengths): + continue try: path.unlink() result.removed += 1 diff --git a/tests/test_tracing_message_pack.py b/tests/test_tracing_message_pack.py index 0e1cb7374..9a0360a41 100644 --- a/tests/test_tracing_message_pack.py +++ b/tests/test_tracing_message_pack.py @@ -197,3 +197,44 @@ def test_a_pack_over_the_cache_budget_is_streamed(tmp_path: Path, monkeypatch) - for ref, message in zip(refs, MESSAGES): assert json.loads(message_pack.read_message(artifacts, ref["$msg"])) == message + +def test_pack_keeps_a_file_whose_pack_is_gone(tmp_path: Path) -> None: + refs = _publish(tmp_path) + artifacts = tmp_path / "logs" / "audit-artifacts" + message_pack.pack_messages(artifacts) + kept = artifact_v2.message_path(artifacts, refs[0]["$msg"]) + for pack in message_pack.packs_dir(artifacts).glob("*.pack.zst"): + pack.unlink() + kept.parent.mkdir(parents=True, exist_ok=True) + kept.write_text(artifact_v2.message_text(MESSAGES[0]), encoding="utf-8") + newcomer = {"role": "user", "content": "a message written after the pack was lost"} + new_path = artifact_v2.message_path(artifacts, artifact_v2.message_sha1(newcomer)) + new_path.parent.mkdir(parents=True, exist_ok=True) + new_path.write_text(artifact_v2.message_text(newcomer), encoding="utf-8") + + message_pack.pack_messages(artifacts) + + assert kept.is_file() + shell = {"artifactFormat": artifact_v2.ARTIFACT_FORMAT, "messages": [refs[0]]} + assert artifact_v2.resolve_payload(shell, artifacts)["messages"][0] == MESSAGES[0] + + +def test_pack_keeps_a_file_whose_pack_is_truncated(tmp_path: Path) -> None: + refs = _publish(tmp_path) + artifacts = tmp_path / "logs" / "audit-artifacts" + message_pack.pack_messages(artifacts) + kept = artifact_v2.message_path(artifacts, refs[0]["$msg"]) + pack = next(message_pack.packs_dir(artifacts).glob("*.pack.zst")) + pack.write_bytes(pack.read_bytes()[:16]) + kept.parent.mkdir(parents=True, exist_ok=True) + kept.write_text(artifact_v2.message_text(MESSAGES[0]), encoding="utf-8") + newcomer = {"role": "user", "content": "a message written after the pack was cut"} + new_path = artifact_v2.message_path(artifacts, artifact_v2.message_sha1(newcomer)) + new_path.parent.mkdir(parents=True, exist_ok=True) + new_path.write_text(artifact_v2.message_text(newcomer), encoding="utf-8") + + message_pack.pack_messages(artifacts) + + assert kept.is_file() + shell = {"artifactFormat": artifact_v2.ARTIFACT_FORMAT, "messages": [refs[0]]} + assert artifact_v2.resolve_payload(shell, artifacts)["messages"][0] == MESSAGES[0] From 00e65f224998a7de10b225e2e9ea4bba49094353 Mon Sep 17 00:00:00 2001 From: Dizhan Xue Date: Fri, 30 Oct 2026 04:49:06 +0000 Subject: [PATCH 16/23] docs(tracing): define Message Pack and update the Message Blob entry CONTEXT.md said the message store had no reclaim path and that compact names two content stores. This branch adds `raven tracing pack-messages` and a third store, so the Message Blob entry now points at the Message Pack and names all three. The new term carries its format, migration order, untrusted-index validation and read cache. --- CONTEXT.md | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/CONTEXT.md b/CONTEXT.md index e4b26bc08..34bc39756 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -2548,13 +2548,30 @@ sha1 of `json.dumps(message, ensure_ascii=False, default=str)` — key order preserved, since sorting would make resolution hand back a reordered copy. Beside `_blobs/` and never inside it: `raven.tracing.compact` sweeps a blob whose link count is 1, and a Message Blob's is permanently 1 because a shell references it from its -JSON text rather than by hard link. `compact` names both stores in -`_CONTENT_STORES` and walks neither as an artifact tree -- their layout is +JSON text rather than by hard link. `compact` names the three content stores +(`_blobs/`, `_messages/`, `_message_packs/`) in `_CONTENT_STORES` and walks none +of them as an artifact tree -- their layout is indistinguishable from `//`, so a store left in the walk is rehashed -whole on every run and hard-linked into `_blobs/`, breaking this boundary. No reclaim path yet; growth is bounded only by use. +whole on every run and hard-linked into `_blobs/`, breaking this boundary. Its +reclaim path is Message Pack: `raven tracing pack-messages` moves the files into +packs; until that runs, growth is bounded only by use. _Avoid_: storing one under `_blobs/` — that directory holds whole-artifact payloads, and `compact` sweeps any member of it whose link count is 1. +**Message Pack** (`raven/tracing/message_pack.py`): +Message Blobs concatenated into multi-megabyte zstd frames under +`_message_packs/.pack.zst`, with `index.json` mapping each sha1 to +`(pack, plaintext offset, length)`. A migration, not a writer change: +`raven tracing pack-messages` writes the packs, replaces the index atomically, and +only then deletes the message files, so a never-packed store, a half-packed one, +and a crash at any step all leave every address readable (a reader that finds no +file reads its slice). The index travels with an imported store and is untrusted: +a pack name is a generated basename and the range fields are non-negative +integers, validated before any path is built. A reader keeps at most 32 MB of +decompressed packs; a larger pack is streamed per read. +_Avoid_: "archive" — that names the rotated-log directory; a pack is not a +whole-store backup. + **Artifact Shell** (`raven/observability/semconv.py`): An `llm.input` artifact in `audit.artifact.v2` form: the call's identity plus one `{"$msg": ""}` reference per message, with `systemPrompt` and `prompt` aliasing From 72fc2298d97333cb1715a4e86de18a44b1c59abe Mon Sep 17 00:00:00 2001 From: Dizhan Xue Date: Fri, 30 Oct 2026 08:39:29 +0000 Subject: [PATCH 17/23] fix(trajectory): carry a span log cursor over to its compressed file The session index identifies a span log by (st_dev, st_ino), which is what lets a rotation rename keep the cursor. Compressing a closed log writes a new file and renames it over the old name, so the same plaintext came back under a new inode. The scanner dropped the old cursor and read the new file from offset 0: every record without trace and span ids was listed a second time under a new position key, a rotation in the middle of a budgeted scan reset the whole index (new epoch), and a spilled trace whose position named the old inode was counted as lost. On ext4 the freed inode is handed to the next file created, usually the new active log, so the old key did not even vanish: it came back holding a shorter file, which read as a truncation and reset the index. Before treating an unknown file as new, the scanner now matches it against the cursors whose key is gone or now holds less plaintext than they consumed: the same sha1 of the first 64 plaintext bytes, and at least as much plaintext as was consumed. A match carries the cursor over at its offset, the key's new holder is read as a new file, and a recovery task holding the old key follows it to the new one. codec.read_head() reads only those first bytes, so a match does not decode the whole archive. Tests: tests/test_trajectory_index.py adds four cases that failed before the change: a log the store compresses after rotation is not read again (a broken record was listed twice), compressing a log mid-scan keeps the epoch (it moved from -2 to -3), a compressed log whose inode is reused keeps the epoch (a hard link keeps the old inode alive so the reuse is the same on every file system), and a spilled trace whose first file was compressed loses nothing (unresolved_dropped was 1). --- raven/tracing/codec.py | 21 ++++++++ raven/trajectory/index.py | 56 ++++++++++++++++++- tests/test_trajectory_index.py | 99 ++++++++++++++++++++++++++++++++++ 3 files changed, 175 insertions(+), 1 deletion(-) diff --git a/raven/tracing/codec.py b/raven/tracing/codec.py index 5c7efc083..7b832555e 100644 --- a/raven/tracing/codec.py +++ b/raven/tracing/codec.py @@ -95,6 +95,26 @@ def plaintext_size(path: Path) -> int | None: return None if plain is None else len(plain) +def read_head(path: Path, size: int) -> bytes | None: + """The first ``size`` plaintext bytes of a stored file, decoding no further.""" + try: + with path.open("rb") as handle: + if not is_compressed(handle.read(4)): + handle.seek(0) + return handle.read(size) + handle.seek(0) + head = b"" + with zstandard.ZstdDecompressor().stream_reader(handle, read_across_frames=False) as reader: + while len(head) < size: + chunk = reader.read(size - len(head)) + if not chunk: + break + head += chunk + return head + except (OSError, zstandard.ZstdError): + return None + + def read_plaintext_file(path: Path, limit: int = MAX_PLAINTEXT_BYTES) -> bytes | None: """A stored file's plaintext, or None when it cannot be read or decoded.""" try: @@ -140,6 +160,7 @@ def open_plaintext(path: Path, limit: int = MAX_PLAINTEXT_BYTES) -> BinaryIO: "is_compressed", "open_plaintext", "plaintext_size", + "read_head", "read_plaintext", "read_plaintext_file", ] diff --git a/raven/trajectory/index.py b/raven/trajectory/index.py index bc7455b14..39d4276bc 100644 --- a/raven/trajectory/index.py +++ b/raven/trajectory/index.py @@ -13,7 +13,9 @@ - Files are identified by ``(st_dev, st_ino)``, so the rotation rename keeps the same cursor; whether a file has more to read is decided by its current - size against the consumed offset, never by a past EOF. + size against the consumed offset, never by a past EOF. Compressing a closed + log replaces its inode; the scanner recognises it by its plaintext head and + carries the cursor over instead of reading it again. - Trace membership is three-valued. A span whose ``session.key`` matches makes its trace a member; a root span's ``trace.dispatched_in_trace_id`` links a sub-agent trace to its parent, and the decision propagates down @@ -264,6 +266,9 @@ def __init__(self, state_dir: Path, *, max_line_bytes: int = MAX_LINE_BYTES) -> # key -> (mtime_ns, raw size, plaintext size), so a catalog pass over # an unchanged archive never reopens it just to learn its encoding. self._sizes: dict[FileKey, tuple[int, int, int]] = {} + # old key -> new key of a log re-encoded under a new inode, so a + # recovery task holding the old key still finds where it was. + self._moved: dict[FileKey, FileKey] = {} def _plaintext_size(self, path: Path, key: FileKey, stat_result: Any) -> int | None: cached = self._sizes.get(key) @@ -302,6 +307,50 @@ def _fingerprint(handle: Any) -> str | None: return None return hashlib.sha1(head).hexdigest() + @staticmethod + def _head_fingerprint(path: Path) -> str | None: + """sha1 of a file's first FINGERPRINT_BYTES of plaintext, or None when it is shorter.""" + head = trace_codec.read_head(path, FINGERPRINT_BYTES) + if head is None or len(head) < FINGERPRINT_BYTES: + return None + return hashlib.sha1(head).hexdigest() + + def _adopt_reencoded(self, catalog: Sequence[tuple[Path, FileKey, int, int]]) -> None: + """Carry a cursor over to its log re-encoded under a new inode. + + Compressing a closed log writes a new file and renames it over the old + name, so the same plaintext comes back under a new key. Read as a new + file it would be replayed from offset 0: every position-keyed record + listed twice, and an index reset when the old cursor was mid-read. + + The old key is either gone or, since a file system hands a freed inode + to the next file created, held by a file shorter than what the cursor + consumed. Such a cursor whose plaintext head matches a file this + scanner has no cursor for, holding at least what was consumed, moves + to that file; the key's new holder is then read as a new file. + """ + sizes = {key: size for _, key, size, _ in catalog} + unknown = [(path, key, size) for path, key, size, _ in catalog if key not in self.cursors] + if not unknown: + return + vanished = [c for key, c in self.cursors.items() if c.fingerprint is not None and sizes.get(key, -1) < c.offset] + for path, key, size in unknown: + if not vanished: + return + fingerprint = self._head_fingerprint(path) + cursor = next((c for c in vanished if c.fingerprint == fingerprint and c.offset <= size), None) + if cursor is None: + continue + vanished.remove(cursor) + del self.cursors[cursor.key] + self._sizes.pop(cursor.key, None) + self._moved.pop(key, None) + self._moved[cursor.key] = key + if self.active_key == cursor.key: + self.active_key = key + cursor.key, cursor.path, cursor.mtime_seen = key, path, -1 + self.cursors[key] = cursor + def _generation_changed(self, catalog: Sequence[tuple[Path, FileKey, int, int]]) -> bool: present = {key: size for _, key, size, _ in catalog} for key, cursor in list(self.cursors.items()): @@ -323,6 +372,7 @@ def scan(self, budget: ScanBudget) -> ScanBatch: batch = ScanBatch() catalog = self._catalog() batch.total_bytes = sum(size for _, _, size, _ in catalog) + self._adopt_reencoded(catalog) if self._generation_changed(catalog): batch.generation_changed = True return batch @@ -443,6 +493,10 @@ def rescan(self, task: RecoveryTask, budget: ScanBudget) -> tuple[list[RawRecord keys = [key for _, key, _, _ in catalog] if not keys: return [], True + for _ in range(len(self._moved)): + if task.key not in self._moved: + break + task.key = self._moved[task.key] if task.key not in keys: # Removed out of band: what that file held of the trace is lost, # and the rest is read again from the oldest file still there. diff --git a/tests/test_trajectory_index.py b/tests/test_trajectory_index.py index c3e4cffa2..cdad53a15 100644 --- a/tests/test_trajectory_index.py +++ b/tests/test_trajectory_index.py @@ -10,7 +10,9 @@ import pytest +from raven.tracing.codec import is_compressed from raven.tracing.compress import compress_logs +from raven.tracing.store import TraceStore from raven.trajectory import conversation as tconv from raven.trajectory import entries as tent from raven.trajectory import index as tidx @@ -68,6 +70,11 @@ def _rotate(state: Path, name: str = "2026-09-01-100000000000") -> Path: return target +def _compress(state: Path, archived: Path) -> None: + """Re-encode one rotated log under its own name, as the store does a rotation later.""" + TraceStore(state)._compress_rotated(archived) + + def _artifact(state: Path, payload, name: str) -> str: directory = state / "logs" / "audit-artifacts" directory.mkdir(parents=True, exist_ok=True) @@ -213,6 +220,72 @@ def test_compressing_a_consumed_archive_keeps_the_cursor_moving(state, clock): assert {e.span_id for e in index.entries()} == {"before", "after"} +def test_a_log_the_store_compresses_after_rotation_is_not_read_again(state, clock): + """The store re-encodes a rotated log under a new inode. A record without + ids is keyed by its log position, so reading that log again as a new file + would list it twice.""" + store = TraceStore(state, max_bytes=4096) + broken = {"name": "memory.enqueue", "startTime": _ts(0), "endTime": _ts(1), "attributes": {"session.key": SESSION}} + store.append_span(broken) + store.append_span(_turn("t", "a", start=1, end=2)) + index = _index(state, clock) + _refresh_until_ready(index, clock) + before = [i for i in _ids(index) if i.startswith("broken:")] + assert len(before) == 1 + + archive = state / "logs" / "archive" + turns = 0 + while len(list(archive.rglob("audit-spans-*.log"))) < 2: + store.append_span(_turn("t", f"b{turns}", start=3 + turns, end=3 + turns)) + turns += 1 + oldest = sorted(archive.rglob("audit-spans-*.log"))[0] + assert is_compressed(oldest.read_bytes()) + _refresh_until_ready(index, clock) + + assert [i for i in _ids(index) if i.startswith("broken:")] == before + assert len(_ids(index)) == 1 + 2 * (1 + turns) + + +def test_compressing_a_log_mid_scan_keeps_the_epoch(state, clock): + _append(state, [_turn("t", f"s{i}", start=i, end=i) for i in range(300)]) + index = _index(state, clock, budget_bytes=16 * 1024) + index.refresh_sync(None) + assert index.index_state().phase == tidx.PHASE_SCANNING + epoch = index.epoch + + _compress(state, _rotate(state)) + _append(state, [_turn("t", "after", start=400, end=401)]) + _refresh_until_ready(index, clock) + + assert index.epoch == epoch + assert len(_ids(index)) == 2 * 301 + + +def test_a_compressed_log_whose_inode_is_reused_keeps_the_epoch(state, clock): + """ext4 hands a freed inode to the next file created, so the active log + written right after a compression can carry the old log's key. The key is + then present but smaller than the cursor's offset; that is the old log + moved, not truncated, when its plaintext turns up under a new key.""" + _append(state, [_turn("t", f"s{i}", start=i, end=i) for i in range(300)]) + index = _index(state, clock, budget_bytes=16 * 1024) + index.refresh_sync(None) + assert index.index_state().phase == tidx.PHASE_SCANNING + epoch = index.epoch + + archived = _rotate(state) + held = state / "logs" / "held-inode" + os.link(archived, held) + _compress(state, archived) + os.replace(held, _log(state)) + with _log(state).open("r+b") as fh: + fh.truncate(0) + _append(state, [_turn("t", "after", start=400, end=401)]) + _refresh_until_ready(index, clock) + + assert index.epoch == epoch + assert len(_ids(index)) == 2 * 301 + + # ── budgets ─────────────────────────────────────────────────────────── @@ -480,6 +553,32 @@ def test_a_spilled_trace_whose_first_file_was_removed_keeps_what_later_files_hol assert index.index_state().recovering_traces == 0 and index.spilled == {} +def test_a_spilled_trace_whose_first_file_was_compressed_loses_nothing(state, clock): + """A spill records a position in a file; compressing that file moves it to + a new inode, and the rescan must follow it rather than count a gap.""" + _append(state, [_tool("S", f"early-{i}", "s-root", start=1, end=2, session="sub") for i in range(6)]) + index = _index(state, clock, pending_spans=5) + index.refresh_sync(None) + assert "S" in index.spilled + archived = _rotate(state) + _append(state, [_tool("S", f"late-{i}", "s-root", start=1, end=2, session="sub") for i in range(6)]) + index.refresh_sync(None) + _compress(state, archived) + attrs = {"trace.dispatched_in_trace_id": "t"} + _append( + state, + [ + _turn("t", "a", start=0, end=9), + _span("S", "s-root", "session.turn", start=0, end=3, session="sub", attrs=attrs), + ], + ) + _refresh_until_ready(index, clock) + recovered = {e.span_id for e in index.entries() if e.trace_id == "S"} + assert recovered == {f"early-{i}" for i in range(6)} | {f"late-{i}" for i in range(6)} | {"s-root"} + assert index.index_state().unresolved_dropped == 0 + assert index.index_state().recovering_traces == 0 and index.spilled == {} + + def test_spilled_foreign_trace_is_forgotten(state, clock): children = [_tool("F", f"f-{i}", "f-root", start=1, end=2, session="other") for i in range(12)] _append(state, children) From 92f74862282232b3b2ffe749adf4812940844439 Mon Sep 17 00:00:00 2001 From: Dizhan Xue Date: Fri, 30 Oct 2026 08:39:29 +0000 Subject: [PATCH 18/23] fix(tracing): compress the previous rotated log, not the one just rotated Every append opens the active log, writes a line and closes it. A writer that opened the file just before another writer's rotation renamed it still writes into that inode a moment later. Compressing the just-rotated log at rotation replaced the inode, so that line went into the unlinked file and was lost. Rotation now compresses the newest older archive of the same kind instead: nothing holds a log one rotation old. One file per rotation keeps the cost where it was (about 9 ms to find it among 2191 archives); an older plain backlog is left to `raven tracing compress --logs`. The re-encoded file keeps its mtime, because the viewer orders logs by mtime and keeps the last write of a span, and an archive compressed a rotation later must not sort after the logs written since. Tests: tests/test_tracing_store_compression.py replaces the test that pinned compression of the just-rotated file with one that pins the previous archive compressed, its mtime unchanged and the newest archive plain, and adds a writer holding the active log across a rotation, whose span was lost before. --- raven/tracing/store.py | 37 +++++++++++++++++++------ tests/test_tracing_store_compression.py | 33 ++++++++++++++++++---- 2 files changed, 57 insertions(+), 13 deletions(-) diff --git a/raven/tracing/store.py b/raven/tracing/store.py index 91a00f6c6..a50112a09 100644 --- a/raven/tracing/store.py +++ b/raven/tracing/store.py @@ -120,24 +120,45 @@ def _rotate_if_needed(self, kind: str, next_text: str) -> Path: base = _KIND_FILES[kind].replace(".log", "") rotated = day_dir / f"{base}-{current_day}-{suffix}.log" path.rename(rotated) - self._compress_rotated(rotated) + self._compress_previous_archive(base, rotated) return path + def _compress_previous_archive(self, base: str, rotated: Path) -> None: + """Compress the log this kind's previous rotation archived. + + Not ``rotated`` itself: another writer may have opened the active log + just before the rename and still write into that inode, and a rewrite + would drop the line. Every append opens and closes the file, so nothing + holds a log one rotation older. One file per rotation bounds the cost + here; an older plain backlog is left to ``raven tracing compress --logs``. + """ + try: + archived = [p for p in self.archive_dir.glob(f"*/{base}-*.log") if p != rotated] + previous = max(archived, key=lambda p: p.stat().st_mtime_ns, default=None) + except OSError: + return + if previous is not None: + self._compress_rotated(previous) + def _compress_rotated(self, path: Path) -> None: - """Store a just-rotated log compressed under the name it already has. + """Store a rotated log compressed under the name it already has. - Rotation is the safe moment: the file is closed and never appended to - again, so rewriting it cannot race the writer. A failure leaves it - plain, which every reader accepts. + The mtime is kept: the viewer orders logs by it, and a re-encoded + archive must not sort after the logs written later. A failure leaves + the log plain, which every reader accepts. """ try: - data = path.read_bytes() - if is_compressed(data): - return + with path.open("rb") as handle: + if is_compressed(handle.read(4)): + return + handle.seek(0) + data = handle.read() + stat = os.fstat(handle.fileno()) packed = compress_bytes(data) tmp = path.parent / f".{path.name}.{os.getpid()}.{uuid.uuid4().hex}.compress.tmp" try: tmp.write_bytes(packed) + os.utime(tmp, ns=(stat.st_atime_ns, stat.st_mtime_ns)) os.replace(tmp, path) finally: tmp.unlink(missing_ok=True) diff --git a/tests/test_tracing_store_compression.py b/tests/test_tracing_store_compression.py index 10362e100..da20b5fda 100644 --- a/tests/test_tracing_store_compression.py +++ b/tests/test_tracing_store_compression.py @@ -298,13 +298,36 @@ def test_compress_logs_is_idempotent_and_dry_run_writes_nothing(tmp_path: Path) assert again.already == 1 -def test_rotation_compresses_the_rotated_file(tmp_path: Path) -> None: +def test_rotation_compresses_the_previous_archive_and_keeps_its_mtime(tmp_path: Path) -> None: + """The viewer orders logs by mtime, so a re-encoded archive keeps the one + it had; the log just rotated stays plain until the next rotation.""" store = TraceStore(tmp_path, max_bytes=1) + store.append_span(_span_record("t1", "s1")) + store.append_span(_span_record("t2", "s2")) + archive = tmp_path / "logs" / "archive" + first = next(archive.rglob("audit-spans-*.log")) + assert not is_compressed(first.read_bytes()) + mtime = first.stat().st_mtime_ns + + store.append_span(_span_record("t3", "s3")) + + second = next(path for path in archive.rglob("audit-spans-*.log") if path != first) + assert is_compressed(first.read_bytes()) + assert first.stat().st_mtime_ns == mtime + assert not is_compressed(second.read_bytes()) + assert [span["spanId"] for span in iter_spans(tmp_path)] == ["s1", "s2", "s3"] + +@pytest.mark.skipif(os.name == "nt", reason="Windows refuses to rename a log another handle holds open") +def test_a_writer_holding_the_active_log_across_rotation_keeps_its_span(tmp_path: Path) -> None: + """Another writer can open the active log just before a rotation renames + it and write a moment later; that line lands in the rotated inode.""" + store = TraceStore(tmp_path, max_bytes=1) store.append_span(_span_record("t1", "s1")) + held = (tmp_path / "logs" / "audit-spans.log").open("a", encoding="utf-8") + store.append_span(_span_record("t2", "s2")) + held.write(json.dumps(_span_record("t3", "late")) + "\n") + held.close() - archives = list((tmp_path / "logs" / "archive").rglob("audit-spans-*.log")) - assert len(archives) == 1 - assert is_compressed(archives[0].read_bytes()) - assert [span["spanId"] for span in iter_spans(tmp_path)] == ["s1", "s2"] + assert {span["spanId"] for span in iter_spans(tmp_path)} == {"s1", "s2", "late"} From c2d5b17de026ee0ab200aa939ed9bc7a519710bf Mon Sep 17 00:00:00 2001 From: Dizhan Xue Date: Fri, 30 Oct 2026 08:39:29 +0000 Subject: [PATCH 19/23] fix(tracing): leave the newest rotated log to the next rotation in compress --logs `raven tracing compress --logs` rewrote every plain rotated log, including the one just rotated, which a writer that opened the active log a moment before the rename may still append its last line to. The store stopped compressing that log at rotation for this reason, and the manual migration now follows the same rule: the newest rotated log of each kind is counted and left for the store's next rotation, and every older one is compressed. A compressed log also keeps its mtime now. The viewer orders logs by mtime and keeps a span's last write, and choosing the newest log of each kind reads the same mtimes: with the migration's own time on every rewritten file, a second run took an already compressed log for the newest and compressed the real newest one. The command prints how many logs it left for the next rotation. Tests: tests/test_tracing_store_compression.py adds a store with two rotated logs of each kind, where the newest of each stays plain and the older ones keep their mtimes; it failed before the change. The tests that planted one rotated log and expected it compressed now plant two and assert that the older one is compressed, so none of them passes with nothing compressed. The two index tests that used the migration to compress the one log they need now re-encode it with the store's per-file compression. --- raven/cli/tracing_commands.py | 5 ++- raven/tracing/compress.py | 45 ++++++++++++++++++---- tests/test_cli_tracing_commands.py | 4 +- tests/test_tracing_store_compression.py | 50 +++++++++++++++++++++---- tests/test_trajectory_index.py | 7 ++-- 5 files changed, 91 insertions(+), 20 deletions(-) diff --git a/raven/cli/tracing_commands.py b/raven/cli/tracing_commands.py index 1d1152fc0..28a70da7a 100644 --- a/raven/cli/tracing_commands.py +++ b/raven/cli/tracing_commands.py @@ -341,7 +341,10 @@ def _run_compress(*, dry_run: bool, logs: bool = False) -> None: console.print(f"[yellow]No artifacts at {escape(str(artifacts_dir))}.[/yellow]") if logs: log_result = compress_logs(store.logs_dir, dry_run=dry_run) - console.print(f"logs {log_result.files}: compressed {log_result.compressed}, already {log_result.already}") + console.print( + f"logs {log_result.files}: compressed {log_result.compressed}, already {log_result.already}, " + f"left for the next rotation {log_result.newest}" + ) console.print( f"{log_result.bytes_before / (1024 * 1024):.1f} MB -> {log_result.bytes_after / (1024 * 1024):.1f} MB, " f"saved {log_result.saved / (1024 * 1024):.1f} MB" diff --git a/raven/tracing/compress.py b/raven/tracing/compress.py index 079a45612..85f8be1b6 100644 --- a/raven/tracing/compress.py +++ b/raven/tracing/compress.py @@ -59,11 +59,17 @@ def _files(root: Path) -> Iterator[Path]: yield path -def _replace(path: Path, payload: bytes) -> None: - """Swap ``path``'s bytes without ever leaving the name pointing at nothing.""" +def _replace(path: Path, payload: bytes, *, times: tuple[int, int] | None = None) -> None: + """Swap ``path``'s bytes without ever leaving the name pointing at nothing. + + ``times`` (atime, mtime in ns) goes on the new file before the swap, for a + file whose readers order by mtime. + """ tmp = path.parent / f".{path.name}.{os.getpid()}.{uuid.uuid4().hex}.compress.tmp" try: tmp.write_bytes(payload) + if times is not None: + os.utime(tmp, ns=times) os.replace(tmp, path) finally: tmp.unlink(missing_ok=True) @@ -171,6 +177,20 @@ def _is_rotated_log(name: str) -> bool: return (name.startswith("audit-spans-") or name.startswith("audit-events-")) and name.endswith(".log") +def _newest_of_each_kind(paths: list[Path]) -> set[Path]: + """The last rotated log of each kind (spans, events), by mtime.""" + newest: dict[str, tuple[int, Path]] = {} + for path in paths: + try: + mtime = path.stat().st_mtime_ns + except OSError: + continue + kind = path.name.split("-", 2)[1] + if kind not in newest or mtime > newest[kind][0]: + newest[kind] = (mtime, path) + return {path for _, path in newest.values()} + + @dataclass class LogCompressResult: """What one pass over the rotated logs saw, did, and saved.""" @@ -178,6 +198,7 @@ class LogCompressResult: files: int = 0 compressed: int = 0 already: int = 0 + newest: int = 0 bytes_before: int = 0 bytes_after: int = 0 errors: list[str] = field(default_factory=list) @@ -194,18 +215,28 @@ def compress_logs(logs_dir: Path, *, dry_run: bool = False) -> LogCompressResult ACP frame journals are left alone -- a span records a plaintext byte range into one, so re-encoding it would invalidate that audit record. - A rotated log is closed and never appended to again, which is why it can be - rewritten here and why a running writer needs no coordination. Idempotent: - a compressed file is counted and left alone; a dry run writes nothing. + The newest rotated log of each kind is left for the store to compress at + its next rotation: a writer that opened the active log just before it was + renamed may still append its last line to that inode, and a rewrite would + drop the line. Every older one is closed for good, which is why it can be + rewritten here and why a running writer needs no coordination. A compressed + log keeps its mtime, which is how the viewer orders logs. Idempotent: a + compressed file is counted and left alone; a dry run writes nothing. """ result = LogCompressResult() archive = Path(logs_dir) / "archive" if not archive.is_dir(): return result - for path in sorted(p for p in _files(archive) if _is_rotated_log(p.name)): + logs = sorted(p for p in _files(archive) if _is_rotated_log(p.name)) + newest = _newest_of_each_kind(logs) + for path in logs: result.files += 1 + if path in newest: + result.newest += 1 + continue try: data = path.read_bytes() + stat = path.stat() except OSError as exc: result.errors.append(f"{path}: {exc}") continue @@ -219,7 +250,7 @@ def compress_logs(logs_dir: Path, *, dry_run: bool = False) -> LogCompressResult if dry_run: continue try: - _replace(path, packed) + _replace(path, packed, times=(stat.st_atime_ns, stat.st_mtime_ns)) except OSError as exc: result.errors.append(f"{path}: {exc}") return result diff --git a/tests/test_cli_tracing_commands.py b/tests/test_cli_tracing_commands.py index 6f60b5109..5b97dce8b 100644 --- a/tests/test_cli_tracing_commands.py +++ b/tests/test_cli_tracing_commands.py @@ -466,11 +466,13 @@ def test_tracing_compress_logs_dry_run_reports_and_writes_nothing(tmp_path, monk archive = tmp_path / "logs" / "archive" / "2026-01-01" / "audit-spans-2026-01-01-1.log" archive.parent.mkdir(parents=True) archive.write_text('{"spanId": "s"}\n', encoding="utf-8") + os.utime(archive, ns=(1_000_000_000_000_000_000, 1_000_000_000_000_000_000)) + archive.with_name("audit-spans-2026-01-01-2.log").write_text('{"spanId": "t"}\n', encoding="utf-8") r = runner.invoke(app, ["tracing", "compress", "--logs", "--dry-run"]) assert r.exit_code == 0 - assert "logs 1: compressed 1" in r.output + assert "logs 2: compressed 1, already 0, left for the next rotation 1" in r.output assert archive.read_text(encoding="utf-8") == '{"spanId": "s"}\n' diff --git a/tests/test_tracing_store_compression.py b/tests/test_tracing_store_compression.py index da20b5fda..3eb6c367e 100644 --- a/tests/test_tracing_store_compression.py +++ b/tests/test_tracing_store_compression.py @@ -235,11 +235,18 @@ def _span_record(trace: str, span_id: str) -> dict: def _plant_logs(state: Path) -> Path: - """A store with one rotated span log, one active log, and one frame journal.""" + """A store with two rotated span logs, one active log, and one frame journal. + + Returns the older rotated log: the newest is left for the next rotation. + """ logs = state / "logs" archive = logs / "archive" / "2026-07-01" / "audit-spans-2026-07-01-1.log" archive.parent.mkdir(parents=True) archive.write_text(json.dumps(_span_record("t1", "s1")) + "\n", encoding="utf-8") + os.utime(archive, ns=(1_000_000_000_000_000_000, 1_000_000_000_000_000_000)) + newest = archive.with_name("audit-spans-2026-07-01-2.log") + newest.write_text(json.dumps(_span_record("t1", "s1b")) + "\n", encoding="utf-8") + os.utime(newest, ns=(1_000_000_100_000_000_000, 1_000_000_100_000_000_000)) (logs / "audit-spans.log").write_text(json.dumps(_span_record("t2", "s2")) + "\n", encoding="utf-8") frames = logs / "acp-frames" / "2026-07-01" / "a.jsonl" frames.parent.mkdir(parents=True) @@ -247,6 +254,33 @@ def _plant_logs(state: Path) -> Path: return archive +def _rotated(state: Path, kind: str, name: str, mtime_ns: int) -> Path: + path = state / "logs" / "archive" / "2026-07-01" / f"audit-{kind}-2026-07-01-{name}.log" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(_span_record("t", f"{kind}-{name}")) + "\n", encoding="utf-8") + os.utime(path, ns=(mtime_ns, mtime_ns)) + return path + + +def test_compress_logs_leaves_the_newest_log_of_each_kind_and_keeps_mtimes(tmp_path: Path) -> None: + """A writer that opened the active log just before it rotated may still + append to the newest archive, so that one waits for the next rotation; the + viewer orders logs by mtime, so a compressed one keeps its own.""" + old_spans = _rotated(tmp_path, "spans", "1", 1_000_000_000_000_000_000) + new_spans = _rotated(tmp_path, "spans", "2", 1_000_000_100_000_000_000) + old_events = _rotated(tmp_path, "events", "1", 1_000_000_000_000_000_000) + new_events = _rotated(tmp_path, "events", "2", 1_000_000_100_000_000_000) + + result = compress_logs(tmp_path / "logs") + + for newest in (new_spans, new_events): + assert not is_compressed(newest.read_bytes()) + for older in (old_spans, old_events): + assert is_compressed(older.read_bytes()) + assert older.stat().st_mtime_ns == 1_000_000_000_000_000_000 + assert (result.files, result.compressed, result.newest) == (4, 2, 2) + + def test_compress_logs_rewrites_only_rotated_files(tmp_path: Path) -> None: archive = _plant_logs(tmp_path) active = tmp_path / "logs" / "audit-spans.log" @@ -256,7 +290,7 @@ def test_compress_logs_rewrites_only_rotated_files(tmp_path: Path) -> None: result = compress_logs(tmp_path / "logs") - assert (result.files, result.compressed, result.already) == (1, 1, 0) + assert (result.files, result.compressed, result.already, result.newest) == (2, 1, 0, 1) assert result.errors == [] assert is_compressed(archive.read_bytes()) # A span records a plaintext byte range into a frame journal, so that file @@ -266,21 +300,23 @@ def test_compress_logs_rewrites_only_rotated_files(tmp_path: Path) -> None: def test_iter_spans_reads_a_compressed_archive(tmp_path: Path) -> None: - _plant_logs(tmp_path) + archive = _plant_logs(tmp_path) compress_logs(tmp_path / "logs") + assert is_compressed(archive.read_bytes()) ids = [span["spanId"] for span in iter_spans(tmp_path)] - assert ids == ["s1", "s2"] + assert ids == ["s1", "s1b", "s2"] def test_selection_reads_a_compressed_archive(tmp_path: Path) -> None: - _plant_logs(tmp_path) + archive = _plant_logs(tmp_path) compress_logs(tmp_path / "logs") + assert is_compressed(archive.read_bytes()) records = list(selection._records(tmp_path, None)) - assert [record["spanId"] for record in records] == ["s1", "s2"] + assert [record["spanId"] for record in records] == ["s1", "s1b", "s2"] def test_compress_logs_is_idempotent_and_dry_run_writes_nothing(tmp_path: Path) -> None: @@ -295,7 +331,7 @@ def test_compress_logs_is_idempotent_and_dry_run_writes_nothing(tmp_path: Path) assert once.compressed == 1 again = compress_logs(tmp_path / "logs") assert again.compressed == 0 - assert again.already == 1 + assert (again.already, again.newest) == (1, 1) def test_rotation_compresses_the_previous_archive_and_keeps_its_mtime(tmp_path: Path) -> None: diff --git a/tests/test_trajectory_index.py b/tests/test_trajectory_index.py index cdad53a15..e3d635334 100644 --- a/tests/test_trajectory_index.py +++ b/tests/test_trajectory_index.py @@ -11,7 +11,6 @@ import pytest from raven.tracing.codec import is_compressed -from raven.tracing.compress import compress_logs from raven.tracing.store import TraceStore from raven.trajectory import conversation as tconv from raven.trajectory import entries as tent @@ -197,7 +196,8 @@ def test_compressed_archive_is_indexed_in_plaintext_offsets(state, clock): archive = state / "logs" / "archive" / "2026-09-01" / "audit-spans-2026-09-01-100000000000.log" _append(state, [_turn("t", "before", start=0, end=1)], path=archive) plaintext_bytes = len(archive.read_bytes()) - compress_logs(state / "logs") + _compress(state, archive) + assert is_compressed(archive.read_bytes()) _append(state, [_turn("t", "after", start=2, end=3)]) index = _index(state, clock) @@ -212,8 +212,7 @@ def test_compressing_a_consumed_archive_keeps_the_cursor_moving(state, clock): index = _index(state, clock) _refresh_until_ready(index, clock) - _rotate(state) - compress_logs(state / "logs") + _compress(state, _rotate(state)) _append(state, [_turn("t", "after", start=2, end=3)]) _refresh_until_ready(index, clock) From 8ae67a06a126338ebd654a2fe58f117e47940313 Mon Sep 17 00:00:00 2001 From: Dizhan Xue Date: Fri, 30 Oct 2026 08:39:29 +0000 Subject: [PATCH 20/23] fix(tracing): write a packed message again when its pack is gone The store heals a missing message blob: the next turn that sends the message writes it again. address_items() skipped that write for any address the pack index held, without checking that the pack still existed, so a store copied without its packs kept those addresses unreadable however often the message was sent again. _is_packed() now also requires the pack file the index names to exist. That is one stat per packed address. A truncated pack is not detected here: proving it would mean decoding packs on every write, and a conversation's messages are spread over all packs, so that is left to pack_messages(), which decodes before it deletes anything. Test: tests/test_tracing_message_pack.py deletes the packs and sends the same messages again; the files were not written before the fix and resolve now. --- raven/tracing/store.py | 12 ++++++++++-- tests/test_tracing_message_pack.py | 16 ++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/raven/tracing/store.py b/raven/tracing/store.py index a50112a09..89cff5122 100644 --- a/raven/tracing/store.py +++ b/raven/tracing/store.py @@ -366,10 +366,18 @@ def address_items(self, items: list[Any]) -> list[Any]: return refs def _is_packed(self, sha1: str) -> bool: - """Whether the message store's pack index already holds this address.""" + """Whether a pack still holds this address, so its file is not written again. + + The index alone does not say so: a store copied without its packs keeps + an index naming slices that are gone, and writing the file is then how + the address heals. One stat per address; a truncated pack is left to + ``pack_messages``, which decodes before it deletes anything. + """ from . import message_pack - return message_pack.message_size(self.artifacts_dir, sha1) is not None + index = message_pack.load_index(self.artifacts_dir) + located = index.messages.get(sha1) if index is not None else None + return located is not None and (message_pack.packs_dir(self.artifacts_dir) / located[0]).is_file() @staticmethod def artifact_attributes(prefix: str, artifact: dict[str, Any] | None) -> dict[str, Any]: diff --git a/tests/test_tracing_message_pack.py b/tests/test_tracing_message_pack.py index 9a0360a41..d7845cda4 100644 --- a/tests/test_tracing_message_pack.py +++ b/tests/test_tracing_message_pack.py @@ -134,6 +134,22 @@ def test_the_writer_does_not_republish_a_packed_message(tmp_path: Path) -> None: assert not any(artifact_v2.message_path(store.artifacts_dir, ref["$msg"]).exists() for ref in refs) +def test_the_writer_republishes_a_packed_message_whose_pack_is_gone(tmp_path: Path) -> None: + """A missing blob is written again by the next turn that sends it; a + packed address whose pack was lost must heal the same way.""" + store = TraceStore(tmp_path) + refs = store.address_items(MESSAGES) + message_pack.pack_messages(store.artifacts_dir) + for pack in message_pack.packs_dir(store.artifacts_dir).glob("*.pack.zst"): + pack.unlink() + + assert store.address_items(MESSAGES) == refs + + assert all(artifact_v2.message_path(store.artifacts_dir, ref["$msg"]).is_file() for ref in refs) + shell = {"artifactFormat": artifact_v2.ARTIFACT_FORMAT, "messages": refs} + assert artifact_v2.resolve_payload(shell, store.artifacts_dir)["messages"] == MESSAGES + + def test_a_corrupt_message_file_is_reported_and_not_removed(tmp_path: Path) -> None: refs = _publish(tmp_path) artifacts = tmp_path / "logs" / "audit-artifacts" From d764a32a65092ef11571f639a40ba88e1eef250b Mon Sep 17 00:00:00 2001 From: Dizhan Xue Date: Fri, 30 Oct 2026 08:39:29 +0000 Subject: [PATCH 21/23] fix(cli): let the viewer's zstd floor steer the Node search find_node() returns the first candidate at or above the TUI's floor (22.0), in the order RAVEN_NODE, venv, PATH, Raven's private runtime. The viewer then refused that node when it had no zstd, so a PATH node in 22.0-22.14 or 23.0-23.7 hid a zstd-capable node later in the order, such as the runtime the installer puts under ~/.raven/runtime, and the launcher reported that no usable Node was found. find_node() takes an optional acceptance predicate, defaulting to the TUI's floor, and the viewer passes its zstd check, so the search continues past a node the viewer cannot use. It still returns the best node it saw when none qualifies, for the error message. Test: tests/test_cli_tracing_commands.py puts a 22.12 node on PATH and a 22.20 runtime under RAVEN_HOME; the launcher refused before the fix and picks the runtime now. The two viewer floor tests stub find_node with the new argument. --- raven/cli/tracing_commands.py | 8 ++++--- raven/cli/tui_commands.py | 17 ++++++++++---- tests/test_cli_tracing_commands.py | 36 ++++++++++++++++++++++++++---- 3 files changed, 50 insertions(+), 11 deletions(-) diff --git a/raven/cli/tracing_commands.py b/raven/cli/tracing_commands.py index 28a70da7a..ffb63d22f 100644 --- a/raven/cli/tracing_commands.py +++ b/raven/cli/tracing_commands.py @@ -204,9 +204,11 @@ def _stop_viewer() -> None: def _resolve_node() -> str: from raven.cli.tui_commands import find_node - node, version = find_node() - # find_node returns the best node it saw even below the minimum, so the - # version check is the caller's -- the TUI's own launch does the same. + # The viewer's floor is stricter than the TUI's, so the search has to use + # it: a PATH node the TUI accepts would otherwise hide a zstd-capable one + # later in the order. find_node still returns the best node it saw when + # none qualifies, so the version check stays the caller's. + node, version = find_node(_node_supports_zstd) if not node or version is None or not _node_supports_zstd(version): console.print( "[red]Node with zstd support not found (22.15+, or 23.8+).[/red] The tracing dashboard reads " diff --git a/raven/cli/tui_commands.py b/raven/cli/tui_commands.py index 81c0cb2ce..a28d61fc5 100644 --- a/raven/cli/tui_commands.py +++ b/raven/cli/tui_commands.py @@ -23,7 +23,7 @@ import sys import threading from pathlib import Path -from typing import Optional, Tuple +from typing import Callable, Optional, Tuple import typer @@ -128,8 +128,17 @@ def _is_windows() -> bool: return os.name == "nt" -def find_node() -> Tuple[Optional[str], Optional[Tuple[int, int, int]]]: - """Find a usable node executable (>= 22). +def _meets_tui_floor(version: Tuple[int, int, int]) -> bool: + return version >= _MIN_NODE_VERSION + + +def find_node( + accept: Callable[[Tuple[int, int, int]], bool] = _meets_tui_floor, +) -> Tuple[Optional[str], Optional[Tuple[int, int, int]]]: + """Find a usable node executable (>= 22, or whatever ``accept`` says). + + ``accept`` lets a caller with a stricter floor keep searching past a node + the TUI would take, instead of being handed that one and refusing it. Returns (path, version_tuple) or (None, None) if not found. """ @@ -208,7 +217,7 @@ def find_node() -> Tuple[Optional[str], Optional[Tuple[int, int, int]]]: version = (int(match.group(1)), int(match.group(2)), int(match.group(3))) except (subprocess.SubprocessError, FileNotFoundError, OSError): continue - if version >= _MIN_NODE_VERSION: + if accept(version): return (node_path, version) if best_below_min is None or version > best_below_min[1]: best_below_min = (node_path, version) diff --git a/tests/test_cli_tracing_commands.py b/tests/test_cli_tracing_commands.py index 5b97dce8b..d7093dd50 100644 --- a/tests/test_cli_tracing_commands.py +++ b/tests/test_cli_tracing_commands.py @@ -401,10 +401,10 @@ def test_a_below_minimum_node_is_refused(monkeypatch): import raven.cli.tracing_commands as tc - monkeypatch.setattr("raven.cli.tui_commands.find_node", lambda: ("/usr/bin/node", (22, 14, 0))) + monkeypatch.setattr("raven.cli.tui_commands.find_node", lambda *_accept: ("/usr/bin/node", (22, 14, 0))) with pytest.raises(typer.Exit): tc._resolve_node() - monkeypatch.setattr("raven.cli.tui_commands.find_node", lambda: ("/usr/bin/node", (22, 15, 0))) + monkeypatch.setattr("raven.cli.tui_commands.find_node", lambda *_accept: ("/usr/bin/node", (22, 15, 0))) assert tc._resolve_node() == "/usr/bin/node" @@ -415,13 +415,41 @@ def test_node_23_before_23_8_is_refused(monkeypatch): import raven.cli.tracing_commands as tc - monkeypatch.setattr("raven.cli.tui_commands.find_node", lambda: ("/usr/bin/node", (23, 0, 0))) + monkeypatch.setattr("raven.cli.tui_commands.find_node", lambda *_accept: ("/usr/bin/node", (23, 0, 0))) with pytest.raises(typer.Exit): tc._resolve_node() - monkeypatch.setattr("raven.cli.tui_commands.find_node", lambda: ("/usr/bin/node", (23, 8, 0))) + monkeypatch.setattr("raven.cli.tui_commands.find_node", lambda *_accept: ("/usr/bin/node", (23, 8, 0))) assert tc._resolve_node() == "/usr/bin/node" +def test_a_zstd_node_behind_an_older_node_on_path_is_chosen(tmp_path, monkeypatch): + """find_node stops at the first node the TUI accepts (22.0); a PATH node + without zstd must not hide the installer's newer private runtime.""" + import subprocess + + from raven.cli import tui_commands + + exe = "node.exe" if tui_commands._is_windows() else "node" + on_path = tmp_path / "bin" / exe + runtime_dir = tmp_path / "home" / "runtime" / "node-v22.20.0" + managed = runtime_dir / exe if tui_commands._is_windows() else runtime_dir / "bin" / exe + for binary in (on_path, managed): + binary.parent.mkdir(parents=True, exist_ok=True) + binary.write_text("") + versions = {str(on_path): "v22.12.0", str(managed): "v22.20.0"} + + def fake_run(argv, **_kwargs): + return subprocess.CompletedProcess(argv, 0, stdout=f"{versions[argv[0]]}\n", stderr="") + + monkeypatch.setattr(tui_commands.subprocess, "run", fake_run) + monkeypatch.delenv("RAVEN_NODE", raising=False) + monkeypatch.delenv("VIRTUAL_ENV", raising=False) + monkeypatch.setenv("PATH", str(on_path.parent)) + monkeypatch.setenv("RAVEN_HOME", str(tmp_path / "home")) + + assert tc._resolve_node() == str(managed) + + def _plain_blob_with_link(state_dir: Path, text: str) -> tuple[Path, Path]: sha1 = hashlib.sha1(text.encode("utf-8")).hexdigest() blob = state_dir / "logs" / "audit-artifacts" / "_blobs" / sha1[:2] / f"{sha1}.json" From 93cfa8f940863be27afdc2ff5d6fc10da891b981 Mon Sep 17 00:00:00 2001 From: Dizhan Xue Date: Fri, 30 Oct 2026 08:39:29 +0000 Subject: [PATCH 22/23] docs(tracing): name message packs by their term in the command table The command table described `raven tracing pack-messages` as packing message blobs into "archives". CONTEXT.md lists "archive" under the Message Pack entry's Avoid list, because it names the rotated-log directory that `raven tracing compress --logs` works on. Both language versions now say Message Pack. --- docs-site/docs/commands.md | 2 +- docs-site/docs/commands.zh.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-site/docs/commands.md b/docs-site/docs/commands.md index e0eb48219..7dcd58c76 100644 --- a/docs-site/docs/commands.md +++ b/docs-site/docs/commands.md @@ -37,7 +37,7 @@ lists its subcommands and options. | `raven tracing` | Open the local trace dashboard | | `raven tracing compact` | Consolidate duplicate trace artifacts to free disk space | | `raven tracing compress [--logs]` | Rewrite trace payloads as zstd (with `--logs`, the rotated span and event logs too) to free disk space | -| `raven tracing pack-messages` | Pack content-addressed message blobs into a few archives to free disk space | +| `raven tracing pack-messages` | Pack content-addressed message blobs into a few Message Packs to free disk space | | `raven trajectory` | Save, replay, redact, label, and preserve execution trajectories for debugging, and run the bug report receiver | Run `raven --help` for the command list, or `raven --help` for details diff --git a/docs-site/docs/commands.zh.md b/docs-site/docs/commands.zh.md index 5fbd509b5..617490cf5 100644 --- a/docs-site/docs/commands.zh.md +++ b/docs-site/docs/commands.zh.md @@ -36,7 +36,7 @@ | `raven tracing` | 打开本地追踪面板 | | `raven tracing compact` | 合并重复的追踪产物，回收磁盘空间 | | `raven tracing compress [--logs]` | 以 zstd 重写追踪载荷（加 `--logs` 时还包括轮转后的日志），回收磁盘空间 | -| `raven tracing pack-messages` | 将内容寻址的消息载荷打包成少量归档，回收磁盘空间 | +| `raven tracing pack-messages` | 将内容寻址的消息载荷打包成少量消息包（Message Pack），回收磁盘空间 | | `raven trajectory` | 保存、回放、脱敏、标注和保留执行轨迹，用于调试；也运行问题报告接收器 | 运行 `raven --help` 查看命令列表，或运行 `raven --help` 查看指定命令的详细说明。 From 7c21f3f927f341cae6bd8b7f1bf557f294d113bb Mon Sep 17 00:00:00 2001 From: Dizhan Xue Date: Fri, 30 Oct 2026 09:21:05 +0000 Subject: [PATCH 23/23] fix(tracing): heal a packed message whose pack was cut short A pack can be left short by an interrupted copy while the index still names its slices. The writer only checked that the pack file existed, so an address whose slice no longer decoded kept resolving to the missing-blob stand-in however often its message was sent again; running `raven tracing pack-messages` again could not help either, because the message files were deleted when the pack was written. The writer now asks whether the pack still decodes and covers the slice. The decoded length is streamed and cached per pack, so the first check of a pack costs one decompression and every later address is a stat plus a lookup. A frame cut short decodes to a shorter length rather than raising, so an entry before the cut stays packed and only the entries past it are written again. Test: tests/test_tracing_message_pack.py truncates a pack to its magic and re-sends the same messages. The files were not written before the change and resolve now. --- raven/tracing/message_pack.py | 70 ++++++++++++++++++++++++++---- raven/tracing/store.py | 13 +++--- tests/test_tracing_message_pack.py | 17 ++++++++ 3 files changed, 84 insertions(+), 16 deletions(-) diff --git a/raven/tracing/message_pack.py b/raven/tracing/message_pack.py index f1f1d45e4..7952b4518 100644 --- a/raven/tracing/message_pack.py +++ b/raven/tracing/message_pack.py @@ -208,20 +208,56 @@ def _slice(artifacts_dir: Path, sha1: str, limit: int | None) -> bytes | None: return plain[offset : offset + want] +_LENGTH_CACHE: dict[Path, tuple[tuple[int, int], int | None]] = {} +_LENGTH_LOCK = threading.Lock() + + +def _decoded_length(path: Path) -> int | None: + """A pack's decoded plaintext length, or None when it does not decode. + + Streamed, so the plaintext is never held, and cached per process because a + pack is written once under a generated name; the stamp changes only when + the file is replaced. A frame cut short decodes to a shorter length + instead of raising, which is what tells an entry past the cut from one + before it. + """ + try: + stat_result = path.stat() + except OSError: + return None + stamp = (stat_result.st_mtime_ns, stat_result.st_size) + with _LENGTH_LOCK: + cached = _LENGTH_CACHE.get(path) + if cached is not None and cached[0] == stamp: + return cached[1] + length: int | None = None + try: + with ( + path.open("rb") as handle, + zstandard.ZstdDecompressor().stream_reader(handle, read_across_frames=False) as reader, + ): + total = 0 + for chunk in iter(lambda: reader.read(1 << 20), b""): + total += len(chunk) + length = total + except (OSError, zstandard.ZstdError): + length = None + with _LENGTH_LOCK: + _LENGTH_CACHE[path] = (stamp, length) + return length + + def _pack_plaintext_length(artifacts_dir: Path, name: str, lengths: dict[str, int | None]) -> int | None: """A pack's decoded length, or None when it does not decode whole. - Cached per run: one read and one decompression per distinct pack, however - many index entries name it. A store copied incompletely can leave a pack - truncated, and the frame header still records the original size then, so - the decoded length is what decides whether an entry still resolves. + ``lengths`` memoizes one run, so several index entries naming a pack share + a check; ``_decoded_length`` keeps it across runs. A store copied + incompletely can leave a pack truncated, and the frame header still records + the original size then, so the decoded length is what decides whether an + entry still resolves. """ if name not in lengths: - try: - plain = read_plaintext((packs_dir(artifacts_dir) / name).read_bytes()) - except OSError: - plain = None - lengths[name] = None if plain is None else len(plain) + lengths[name] = _decoded_length(packs_dir(artifacts_dir) / name) return lengths[name] @@ -237,6 +273,21 @@ def _slice_is_readable(artifacts_dir: Path, entry: tuple[str, int, int], lengths return size is not None and offset + length <= size +def slice_readable(artifacts_dir: Path | str, sha1: str) -> bool: + """Whether the pack a stored index names still decodes and covers this address. + + A store copied incompletely can leave the index naming a pack that is gone + or cut short; the address then has no readable copy in the packs. + """ + index = load_index(artifacts_dir) + located = index.messages.get(sha1) if index is not None else None + if located is None: + return False + name, offset, length = located + size = _decoded_length(packs_dir(artifacts_dir) / name) + return size is not None and offset + length <= size + + def read_message(artifacts_dir: Path | str, sha1: str, limit: int | None = None) -> bytes | None: """One packed message's plaintext, or None when the index does not hold it.""" if not _SHA1_RE.match(sha1): @@ -478,4 +529,5 @@ def flush() -> None: "read_path", "sha1_from_path", "size_path", + "slice_readable", ] diff --git a/raven/tracing/store.py b/raven/tracing/store.py index 89cff5122..35098ac6a 100644 --- a/raven/tracing/store.py +++ b/raven/tracing/store.py @@ -368,16 +368,15 @@ def address_items(self, items: list[Any]) -> list[Any]: def _is_packed(self, sha1: str) -> bool: """Whether a pack still holds this address, so its file is not written again. - The index alone does not say so: a store copied without its packs keeps - an index naming slices that are gone, and writing the file is then how - the address heals. One stat per address; a truncated pack is left to - ``pack_messages``, which decodes before it deletes anything. + The index alone does not say so: a store copied incompletely keeps an + index naming slices whose pack is gone or cut short, and writing the + file is then how the address heals. The pack's decoded length is cached + per process, so this is one stat per address after each pack's first + check. """ from . import message_pack - index = message_pack.load_index(self.artifacts_dir) - located = index.messages.get(sha1) if index is not None else None - return located is not None and (message_pack.packs_dir(self.artifacts_dir) / located[0]).is_file() + return message_pack.slice_readable(self.artifacts_dir, sha1) @staticmethod def artifact_attributes(prefix: str, artifact: dict[str, Any] | None) -> dict[str, Any]: diff --git a/tests/test_tracing_message_pack.py b/tests/test_tracing_message_pack.py index d7845cda4..e3ac77e4c 100644 --- a/tests/test_tracing_message_pack.py +++ b/tests/test_tracing_message_pack.py @@ -150,6 +150,23 @@ def test_the_writer_republishes_a_packed_message_whose_pack_is_gone(tmp_path: Pa assert artifact_v2.resolve_payload(shell, store.artifacts_dir)["messages"] == MESSAGES +def test_the_writer_republishes_a_packed_message_whose_pack_is_truncated(tmp_path: Path) -> None: + """An interrupted copy can leave a pack short of its frame. The addresses + whose slice no longer decodes must heal like a missing pack, since the + message files were deleted when the pack was written.""" + store = TraceStore(tmp_path) + refs = store.address_items(MESSAGES) + message_pack.pack_messages(store.artifacts_dir) + pack = next(message_pack.packs_dir(store.artifacts_dir).glob("*.pack.zst")) + pack.write_bytes(pack.read_bytes()[:8]) + + assert store.address_items(MESSAGES) == refs + + assert all(artifact_v2.message_path(store.artifacts_dir, ref["$msg"]).is_file() for ref in refs) + shell = {"artifactFormat": artifact_v2.ARTIFACT_FORMAT, "messages": refs} + assert artifact_v2.resolve_payload(shell, store.artifacts_dir)["messages"] == MESSAGES + + def test_a_corrupt_message_file_is_reported_and_not_removed(tmp_path: Path) -> None: refs = _publish(tmp_path) artifacts = tmp_path / "logs" / "audit-artifacts"