diff --git a/CONTEXT.md b/CONTEXT.md index e4b26bc08..34bc39756 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -2548,13 +2548,30 @@ sha1 of `json.dumps(message, ensure_ascii=False, default=str)` — key order preserved, since sorting would make resolution hand back a reordered copy. Beside `_blobs/` and never inside it: `raven.tracing.compact` sweeps a blob whose link count is 1, and a Message Blob's is permanently 1 because a shell references it from its -JSON text rather than by hard link. `compact` names both stores in -`_CONTENT_STORES` and walks neither as an artifact tree -- their layout is +JSON text rather than by hard link. `compact` names the three content stores +(`_blobs/`, `_messages/`, `_message_packs/`) in `_CONTENT_STORES` and walks none +of them as an artifact tree -- their layout is indistinguishable from `//`, so a store left in the walk is rehashed -whole on every run and hard-linked into `_blobs/`, breaking this boundary. No reclaim path yet; growth is bounded only by use. +whole on every run and hard-linked into `_blobs/`, breaking this boundary. Its +reclaim path is Message Pack: `raven tracing pack-messages` moves the files into +packs; until that runs, growth is bounded only by use. _Avoid_: storing one under `_blobs/` — that directory holds whole-artifact payloads, and `compact` sweeps any member of it whose link count is 1. +**Message Pack** (`raven/tracing/message_pack.py`): +Message Blobs concatenated into multi-megabyte zstd frames under +`_message_packs/.pack.zst`, with `index.json` mapping each sha1 to +`(pack, plaintext offset, length)`. A migration, not a writer change: +`raven tracing pack-messages` writes the packs, replaces the index atomically, and +only then deletes the message files, so a never-packed store, a half-packed one, +and a crash at any step all leave every address readable (a reader that finds no +file reads its slice). The index travels with an imported store and is untrusted: +a pack name is a generated basename and the range fields are non-negative +integers, validated before any path is built. A reader keeps at most 32 MB of +decompressed packs; a larger pack is streamed per read. +_Avoid_: "archive" — that names the rotated-log directory; a pack is not a +whole-store backup. + **Artifact Shell** (`raven/observability/semconv.py`): An `llm.input` artifact in `audit.artifact.v2` form: the call's identity plus one `{"$msg": ""}` reference per message, with `systemPrompt` and `prompt` aliasing diff --git a/docs-site/docs/commands.md b/docs-site/docs/commands.md index b4c3b279e..7dcd58c76 100644 --- a/docs-site/docs/commands.md +++ b/docs-site/docs/commands.md @@ -36,6 +36,8 @@ lists its subcommands and options. | `raven sandbox` | List sandbox VMs, run commands, or open a shell; requires `tools.sandbox.debug.enabled=true` | | `raven tracing` | Open the local trace dashboard | | `raven tracing compact` | Consolidate duplicate trace artifacts to free disk space | +| `raven tracing compress [--logs]` | Rewrite trace payloads as zstd (with `--logs`, the rotated span and event logs too) to free disk space | +| `raven tracing pack-messages` | Pack content-addressed message blobs into a few Message Packs to free disk space | | `raven trajectory` | Save, replay, redact, label, and preserve execution trajectories for debugging, and run the bug report receiver | Run `raven --help` for the command list, or `raven --help` for details diff --git a/docs-site/docs/commands.zh.md b/docs-site/docs/commands.zh.md index 90440dee0..617490cf5 100644 --- a/docs-site/docs/commands.zh.md +++ b/docs-site/docs/commands.zh.md @@ -35,6 +35,8 @@ | `raven sandbox` | 列出沙箱虚拟机、执行命令或打开 Shell；需要设置 `tools.sandbox.debug.enabled=true` | | `raven tracing` | 打开本地追踪面板 | | `raven tracing compact` | 合并重复的追踪产物，回收磁盘空间 | +| `raven tracing compress [--logs]` | 以 zstd 重写追踪载荷（加 `--logs` 时还包括轮转后的日志），回收磁盘空间 | +| `raven tracing pack-messages` | 将内容寻址的消息载荷打包成少量消息包（Message Pack），回收磁盘空间 | | `raven trajectory` | 保存、回放、脱敏、标注和保留执行轨迹，用于调试；也运行问题报告接收器 | 运行 `raven --help` 查看命令列表，或运行 `raven --help` 查看指定命令的详细说明。 diff --git a/pyproject.toml b/pyproject.toml index 2636c4bff..03aa46b9e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -122,6 +122,7 @@ dependencies = [ # `AsyncOpenAI`, the only client LiteLLM's OpenAI-SDK route adopts. It # already arrives with litellm, and the range is litellm's own. "openai>=2.20.0,<3.0.0", + "zstandard>=0.25.0", ] [project.optional-dependencies] diff --git a/raven/cli/tracing_commands.py b/raven/cli/tracing_commands.py index 1656eb01b..ffb63d22f 100644 --- a/raven/cli/tracing_commands.py +++ b/raven/cli/tracing_commands.py @@ -42,6 +42,18 @@ console = Console() +#: zstd decoding landed in Node's zlib in 22.15 and 23.8, and the viewer reads +#: payloads compressed by default. Its floor is therefore higher than the TUI's. +_MIN_VIEWER_NODE_VERSION = (22, 15, 0) +_FIRST_VIEWER_NODE_23_VERSION = (23, 8, 0) + + +def _node_supports_zstd(version: tuple[int, ...]) -> bool: + """Whether this Node has zlib zstd: 22.15+, 23.8+, or any later major.""" + if version[0] == 23: + return version >= _FIRST_VIEWER_NODE_23_VERSION + return version >= _MIN_VIEWER_NODE_VERSION + def _viewer_dir() -> Path: return Path(__file__).resolve().parent / "tracing_viewer" @@ -190,15 +202,17 @@ def _stop_viewer() -> None: def _resolve_node() -> str: - from raven.cli.tui_commands import _MIN_NODE_VERSION, find_node - - node, version = find_node() - # find_node returns the best node it saw even below the minimum, so the - # version check is the caller's -- the TUI's own launch does the same. - if not node or version is None or version < _MIN_NODE_VERSION: + from raven.cli.tui_commands import find_node + + # The viewer's floor is stricter than the TUI's, so the search has to use + # it: a PATH node the TUI accepts would otherwise hide a zstd-capable one + # later in the order. find_node still returns the best node it saw when + # none qualifies, so the version check stays the caller's. + node, version = find_node(_node_supports_zstd) + if not node or version is None or not _node_supports_zstd(version): console.print( - "[red]Node (>= 22) not found.[/red] The tracing dashboard needs the " - "same Node runtime as the TUI.\n" + "[red]Node with zstd support not found (22.15+, or 23.8+).[/red] The tracing dashboard reads " + "zstd-compressed payloads and needs a Node with zlib zstd support.\n" "Install: https://nodejs.org/ or brew install node@22 or nvm install 22" ) raise typer.Exit(1) @@ -306,6 +320,70 @@ def _run_compact(*, dry_run: bool) -> None: console.print(f"[yellow]... and {hidden} more[/yellow]") +def _run_compress(*, dry_run: bool, logs: bool = False) -> None: + from raven.tracing.compress import compress, compress_logs + from raven.tracing.store import TraceStore + + store = TraceStore(tracing_config.state_dir()) + if dry_run: + console.print("[dim]dry run - nothing is written[/dim]") + artifacts_dir = store.artifacts_dir + if artifacts_dir.is_dir(): + result = compress(artifacts_dir, dry_run=dry_run) + console.print( + f"blobs {result.blobs}: compressed {result.compressed}, already {result.already}, relinked {result.relinked}" + ) + console.print(f"messages {result.messages}: compressed {result.messages_compressed}") + console.print( + f"{result.bytes_before / (1024 * 1024):.1f} MB -> {result.bytes_after / (1024 * 1024):.1f} MB, " + f"saved {result.saved / (1024 * 1024):.1f} MB" + ) + _print_skipped(result.errors) + else: + console.print(f"[yellow]No artifacts at {escape(str(artifacts_dir))}.[/yellow]") + if logs: + log_result = compress_logs(store.logs_dir, dry_run=dry_run) + console.print( + f"logs {log_result.files}: compressed {log_result.compressed}, already {log_result.already}, " + f"left for the next rotation {log_result.newest}" + ) + console.print( + f"{log_result.bytes_before / (1024 * 1024):.1f} MB -> {log_result.bytes_after / (1024 * 1024):.1f} MB, " + f"saved {log_result.saved / (1024 * 1024):.1f} MB" + ) + _print_skipped(log_result.errors) + + +def _run_pack_messages(*, dry_run: bool) -> None: + from raven.tracing.message_pack import pack_messages + from raven.tracing.store import TraceStore + + artifacts_dir = TraceStore(tracing_config.state_dir()).artifacts_dir + if dry_run: + console.print("[dim]dry run - nothing is written[/dim]") + if not artifacts_dir.is_dir(): + console.print(f"[yellow]No artifacts at {escape(str(artifacts_dir))}.[/yellow]") + return + result = pack_messages(artifacts_dir, dry_run=dry_run) + console.print( + f"messages {result.scanned}: packed {result.packed}, already {result.already}, " + f"packs {result.packs}, removed files {result.removed}" + ) + console.print( + f"{result.bytes_before / (1024 * 1024):.1f} MB -> {result.bytes_after / (1024 * 1024):.1f} MB, " + f"saved {result.saved / (1024 * 1024):.1f} MB" + ) + _print_skipped(result.errors) + + +def _print_skipped(errors: list[str]) -> None: + for message in errors[:10]: + console.print(f"[yellow]skipped:[/yellow] {escape(message)}") + hidden = max(len(errors) - 10, 0) + if hidden: + console.print(f"[yellow]... and {hidden} more[/yellow]") + + def register(app: typer.Typer) -> None: """Attach the ``tracing`` command to ``app``. @@ -320,7 +398,9 @@ def tracing( None, help=( "Optional action: 'stop' shuts down the background viewer; " - "'compact' folds duplicate artifacts (terminal only, not available from the TUI)." + "'compact' folds duplicate artifacts; 'compress' rewrites payloads as zstd " + "(--logs adds the rotated span and event logs); 'pack-messages' moves message blobs " + "into packs. All but 'stop' are terminal only, not available from the TUI." ), ), port: int = typer.Option(None, "--port", "-p", help="Port to bind (default: config or 4318)."), @@ -328,18 +408,30 @@ def tracing( False, "--foreground", "-f", help="Run the viewer in the foreground (blocks; Ctrl-C to stop)." ), dry_run: bool = typer.Option( - False, "--dry-run", help="With 'compact': report what would change without writing." + False, + "--dry-run", + help="With 'compact', 'compress' or 'pack-messages': report what would change without writing.", ), + logs: bool = typer.Option(False, "--logs", help="With 'compress': also compress rotated span and event logs."), ) -> None: """Open the tracing dashboard (captured LLM/tool/memory spans).""" if action == "compact": _run_compact(dry_run=dry_run) return + if action == "compress": + _run_compress(dry_run=dry_run, logs=logs) + return + if action == "pack-messages": + _run_pack_messages(dry_run=dry_run) + return if action == "stop": _stop_viewer() return if action is not None: - console.print(f"[red]Unknown action '{escape(action)}'.[/red] Supported actions: stop, compact") + console.print( + f"[red]Unknown action '{escape(action)}'.[/red] " + "Supported actions: stop, compact, compress, pack-messages" + ) raise typer.Exit(2) bind_port = port if port is not None else tracing_config.port() if foreground: diff --git a/raven/cli/tracing_viewer/log-store.js b/raven/cli/tracing_viewer/log-store.js index 8777152fd..4dfb60eeb 100644 --- a/raven/cli/tracing_viewer/log-store.js +++ b/raven/cli/tracing_viewer/log-store.js @@ -1,8 +1,40 @@ const fs = require('fs'); const os = require('os'); const path = require('path'); +const zlib = require('zlib'); const DEFAULT_MAX_BYTES = 50 * 1024 * 1024; +const MAX_STORED_BYTES = 256 * 1024 * 1024; +const ZSTD_MAGIC = Buffer.from([0x28, 0xb5, 0x2f, 0xfd]); + +function readStored(filePath) { + const data = fs.readFileSync(filePath); + if (data.length >= 4 && data.subarray(0, 4).equals(ZSTD_MAGIC)) { + // zstd landed in Node's zlib in 22.15 / 23.8. An older viewer can still + // read a plain store; a compressed one is reported rather than parsed as + // mojibake, because JSON.parse of the frame would only say "invalid". + if (typeof zlib.zstdDecompressSync !== 'function') { + console.warn('tracing viewer: this Node has no zstd support; compressed files need Node >= 22.15'); + return Buffer.alloc(0); + } + try { + return zlib.zstdDecompressSync(data, { maxOutputLength: MAX_STORED_BYTES }); + } catch (err) { + if (err && err.code === 'ERR_BUFFER_TOO_LARGE') { + // A compressed file can be imported with a store; a frame that expands + // past the read limit is treated as unreadable rather than allocated. + console.warn('tracing viewer: a compressed file expands past the read limit; treating it as empty'); + return Buffer.alloc(0); + } + return data; + } + } + return data; +} + +function readStoredText(filePath) { + return readStored(filePath).toString('utf8'); +} const KIND_FILES = { events: 'audit-events.log', @@ -65,8 +97,7 @@ function statOrNull(filePath) { function readJsonlFile(filePath) { if (!fs.existsSync(filePath)) return []; - return fs - .readFileSync(filePath, 'utf8') + return readStoredText(filePath) .split('\n') .map((line) => line.trim()) .filter(Boolean) @@ -199,6 +230,8 @@ module.exports = { ensureDir, readJsonl, readJsonlFile, + readStored, + readStoredText, statLogFiles, appendJsonl, rotateIfNeeded, diff --git a/raven/cli/tracing_viewer/server.js b/raven/cli/tracing_viewer/server.js index bd01a385b..fbe37b8a0 100644 --- a/raven/cli/tracing_viewer/server.js +++ b/raven/cli/tracing_viewer/server.js @@ -7,7 +7,7 @@ const os = require('os'); const { URL } = require('url'); const { applyStateDirArg } = require('./state-dir'); applyStateDirArg(); -const { readJsonl, readJsonlFile, statLogFiles, getLogsDir } = require('./log-store'); +const { readJsonl, readJsonlFile, readStored, readStoredText, statLogFiles, getLogsDir } = require('./log-store'); const everosDeposits = require('./everos-deposits'); const shardIndex = require('./shard-index'); @@ -18,6 +18,7 @@ const STATE_DIR = path.join(os.homedir(), '.openclaw'); const LOGS_DIR = getLogsDir(); const ARTIFACTS_DIR = path.join(LOGS_DIR, 'audit-artifacts'); + const STATIC_DIR = path.join(__dirname, 'ui'); const SHELL_HTML = require('./ui/shell.js'); const BUNDLED_DESCRIPTORS_DIR = path.join(__dirname, 'descriptors'); @@ -110,7 +111,9 @@ function readJsonFileMaybe(filePath) { if (!filePath || typeof filePath !== 'string') return null; if (!fs.existsSync(filePath)) return null; try { - return JSON.parse(fs.readFileSync(filePath, 'utf8')); + // Persisted artifacts are stored encoded; readStoredText picks the + // encoding from the frame, so a compressed artifact still parses here. + return JSON.parse(readStoredText(filePath)); } catch { return null; } @@ -1060,10 +1063,75 @@ function isSafeArtifactPath(filePath) { // compares the two, so a rule changed there changes here in the same commit. const ARTIFACT_FORMAT_V2 = 'audit.artifact.v2'; const MESSAGES_DIR = path.join(ARTIFACTS_DIR, '_messages'); +const MESSAGE_PACKS_DIR = path.join(ARTIFACTS_DIR, '_message_packs'); +const MESSAGE_PACK_INDEX = path.join(MESSAGE_PACKS_DIR, 'index.json'); +const MESSAGE_PACK_FORMAT = 'raven.message-pack-index.v1'; const MSG_REF_KEY = '$msg'; const TEXT_FIELDS = ['systemPrompt', 'prompt']; const SHA1_RE = /^[0-9a-f]{40}$/; +// Mirror of raven/tracing/message_pack.py: message files can be moved into +// per-chunk zstd packs, addressed by an sha1 -> (pack, plaintext offset, +// length) index. The index is cached until its size or mtime changes, and a +// few decompressed packs are kept because one conversation page rereads the +// same pack for every message on it. +let packIndexCache = { stamp: null, index: null }; +const packCache = new Map(); + +function messagePackIndex() { + let stat; + try { + stat = fs.statSync(MESSAGE_PACK_INDEX); + } catch { + return null; + } + const stamp = `${stat.mtimeMs}:${stat.size}`; + if (packIndexCache.index && packIndexCache.stamp === stamp) return packIndexCache.index; + try { + const parsed = JSON.parse(fs.readFileSync(MESSAGE_PACK_INDEX, 'utf8')); + if (!parsed || parsed.format !== MESSAGE_PACK_FORMAT || typeof parsed.messages !== 'object') return null; + packIndexCache = { stamp, index: parsed }; + return parsed; + } catch { + return null; + } +} + +function readPackedMessage(sha1) { + const index = messagePackIndex(); + const entry = index && index.messages ? index.messages[sha1] : null; + if (!Array.isArray(entry) || entry.length !== 3) return null; + // The index travels with an imported store; a name is a generated basename, + // never a path, and the range fields are non-negative integers. + if (typeof entry[0] !== 'string' || !/^[0-9a-f]{32}\.pack\.zst$/.test(entry[0])) return null; + if (!Number.isInteger(entry[1]) || !Number.isInteger(entry[2]) || entry[1] < 0 || entry[2] < 0) return null; + const packPath = path.join(MESSAGE_PACKS_DIR, entry[0]); + if (!path.resolve(packPath).startsWith(path.resolve(MESSAGE_PACKS_DIR) + path.sep)) return null; + let plain = packCache.get(packPath); + if (!plain) { + try { + plain = readStored(packPath); + } catch { + return null; + } + if (plain.length <= 64 * 1024 * 1024) { + if (packCache.size >= 4) packCache.delete(packCache.keys().next().value); + packCache.set(packPath, plain); + } + } + return plain.subarray(entry[1], entry[1] + entry[2]); +} + +function messageSha1FromPath(filePath) { + const name = path.basename(filePath); + if (!name.endsWith('.json')) return null; + const sha1 = name.slice(0, -'.json'.length); + if (!SHA1_RE.test(sha1)) return null; + if (path.basename(path.dirname(filePath)) !== sha1.slice(0, 2)) return null; + if (path.basename(path.dirname(path.dirname(filePath))) !== '_messages') return null; + return sha1; +} + function refSha1(value) { if (!value || typeof value !== 'object' || Array.isArray(value)) return null; const keys = Object.keys(value); @@ -1094,9 +1162,16 @@ function resolvePayload(payload) { const file = path.join(MESSAGES_DIR, sha1.slice(0, 2), `${sha1}.json`); let value; try { - value = JSON.parse(fs.readFileSync(file, 'utf8')); + value = JSON.parse(readStoredText(file)); } catch { - value = { role: 'unknown', content: `[message blob missing: ${sha1}]` }; + const packed = readPackedMessage(sha1); + try { + value = packed + ? JSON.parse(packed.toString('utf8')) + : { role: 'unknown', content: `[message blob missing: ${sha1}]` }; + } catch { + value = { role: 'unknown', content: `[message blob missing: ${sha1}]` }; + } } cache.set(sha1, value); } @@ -1127,8 +1202,15 @@ function resolvePayload(payload) { function readArtifact(filePath) { if (!filePath || !isSafeArtifactPath(filePath)) return null; - if (!fs.existsSync(filePath)) return null; - const content = fs.readFileSync(filePath, 'utf8'); + let content = null; + if (fs.existsSync(filePath)) { + content = readStoredText(filePath); + } else { + const sha1 = messageSha1FromPath(filePath); + const packed = sha1 === null ? null : readPackedMessage(sha1); + if (packed) content = packed.toString('utf8'); + } + if (content === null) return null; let parsed = null; try { parsed = resolvePayload(JSON.parse(content)); @@ -1213,7 +1295,7 @@ function artifactTextForSearch(filePath) { let text = ''; if (stat.isFile() && stat.size <= MAX_ARTIFACT_SEARCH_BYTES) { try { - text = fs.readFileSync(filePath, 'utf8'); + text = readStoredText(filePath).slice(0, MAX_ARTIFACT_SEARCH_BYTES); } catch { text = ''; } diff --git a/raven/cli/tui_commands.py b/raven/cli/tui_commands.py index 81c0cb2ce..a28d61fc5 100644 --- a/raven/cli/tui_commands.py +++ b/raven/cli/tui_commands.py @@ -23,7 +23,7 @@ import sys import threading from pathlib import Path -from typing import Optional, Tuple +from typing import Callable, Optional, Tuple import typer @@ -128,8 +128,17 @@ def _is_windows() -> bool: return os.name == "nt" -def find_node() -> Tuple[Optional[str], Optional[Tuple[int, int, int]]]: - """Find a usable node executable (>= 22). +def _meets_tui_floor(version: Tuple[int, int, int]) -> bool: + return version >= _MIN_NODE_VERSION + + +def find_node( + accept: Callable[[Tuple[int, int, int]], bool] = _meets_tui_floor, +) -> Tuple[Optional[str], Optional[Tuple[int, int, int]]]: + """Find a usable node executable (>= 22, or whatever ``accept`` says). + + ``accept`` lets a caller with a stricter floor keep searching past a node + the TUI would take, instead of being handed that one and refusing it. Returns (path, version_tuple) or (None, None) if not found. """ @@ -208,7 +217,7 @@ def find_node() -> Tuple[Optional[str], Optional[Tuple[int, int, int]]]: version = (int(match.group(1)), int(match.group(2)), int(match.group(3))) except (subprocess.SubprocessError, FileNotFoundError, OSError): continue - if version >= _MIN_NODE_VERSION: + if accept(version): return (node_path, version) if best_below_min is None or version > best_below_min[1]: best_below_min = (node_path, version) diff --git a/raven/node/install.py b/raven/node/install.py index 11d61c67b..09d82a2c9 100644 --- a/raven/node/install.py +++ b/raven/node/install.py @@ -65,8 +65,8 @@ from raven.utils import mirrors #: Roughly what a node takes, for the free-space check: Raven's code with -#: pillow, loguru and ripgrep-bin (measured 2026-10-10), a Python 3.12 when -#: the machine has none of its own, and uv when the machine has none. +#: pillow, loguru, zstandard and ripgrep-bin, a Python 3.12 when the machine +#: has none of its own, and uv when the machine has none. _NODE_KB = 60 * 1024 _PYTHON_KB = 80 * 1024 _UV_KB = 25 * 1024 diff --git a/raven/node/protocol.py b/raven/node/protocol.py index 8ca0b3267..39e20b158 100644 --- a/raven/node/protocol.py +++ b/raven/node/protocol.py @@ -123,9 +123,10 @@ TASK_KEEP_S = 7 * 24 * 3600 #: Third-party packages the daemon cannot run without, beside Raven's own -#: code: the only ones its file tools import (measured 2026-10-10), pinned to -#: the host's versions. -REQUIREMENTS = ("loguru", "pillow") +#: code: the file tools' two, plus zstandard, which the tracing store the +#: daemon links imports for its payload frames (measured 2026-10-29), pinned +#: to the host's versions. +REQUIREMENTS = ("loguru", "pillow", "zstandard") #: Installed when the machine can get it, not required. ``ripgrep-bin`` puts the #: same ``rg`` beside the daemon's Python that the host has beside its own, so a diff --git a/raven/rpc/methods/cli_dispatch.py b/raven/rpc/methods/cli_dispatch.py index 1e77039b6..9cd2276ed 100644 --- a/raven/rpc/methods/cli_dispatch.py +++ b/raven/rpc/methods/cli_dispatch.py @@ -98,9 +98,12 @@ # backing thread is not cancelled on timeout — it would keep running # detached after the RPC call returns, risking Ink reconciler corruption. # Bare ``tracing`` and ``tracing stop`` are unaffected: only this action - # is blacklisted. ``compact`` is a positional action, so this prefix - # catches it only at argv[1] — _is_tracing_compact covers the rest. + # is blacklisted. The maintenance actions are positional arguments, so this + # prefix catches only the argv[1] spelling — _is_tracing_maintenance covers + # every other order. ("tracing", "compact"), # full-store hash walk can outrun the RPC timeout — terminal-only + ("tracing", "compress"), # same walk (and, with --logs, a whole archive rewrite) + ("tracing", "pack-messages"), # same walk over the message store # ``agent`` (no -m) — handled specially in _is_dispatch_compatible } @@ -117,22 +120,21 @@ def _is_agent_repl(argv: list[str]) -> bool: return not any(a in ("-m", "--message") for a in argv[1:]) -def _is_tracing_compact(argv: list[str]) -> bool: - """Return True for a ``tracing compact`` invocation, at any argv position. +def _is_tracing_maintenance(argv: list[str]) -> bool: + """Return True for a ``tracing compact``, ``compress`` or ``pack-messages`` invocation. - ``compact`` is a positional argument of the ``tracing`` leaf command, so - Click accepts it after any option: ``tracing --dry-run compact`` and - ``tracing --port 1 compact`` both run it. The ``_DISPATCH_BLACKLIST`` - prefix match compares fixed positions and therefore only sees the action - at ``argv[1]``, so it alone would let those forms reach dispatch. + All three are positional arguments of the ``tracing`` leaf command, so Click + accepts them after any option: ``tracing --dry-run compact`` and + ``tracing --port 1 compress`` both run. The ``_DISPATCH_BLACKLIST`` prefix + match compares fixed positions and therefore only sees the action at + ``argv[1]``, so it alone would let those forms reach dispatch. - ``raven tracing compact`` -> blacklist hit - ``raven tracing --dry-run compact`` -> blacklist hit (this predicate) - ``raven tracing`` / ``tracing stop`` -> OK + ``raven tracing compact`` / ``tracing compress`` / ``tracing pack-messages`` + -> terminal-only; ``raven tracing`` / ``tracing stop`` -> OK """ if not argv or argv[0] != "tracing": return False - return "compact" in argv[1:] + return any(action in argv[1:] for action in ("compact", "compress", "pack-messages")) # Default timeout if caller did not override. @@ -198,7 +200,7 @@ def _is_dispatch_compatible(argv: list[str]) -> bool: return False if _is_agent_repl(argv): return False - if _is_tracing_compact(argv): + if _is_tracing_maintenance(argv): return False # Reflection-based positive check (replaces the v0.0.2 hardcoded # _DISPATCH_WHITELIST 19-tuple set). The helper module also serves diff --git a/raven/tracing/artifact_v2.py b/raven/tracing/artifact_v2.py index 4f6852a44..85aab9eba 100644 --- a/raven/tracing/artifact_v2.py +++ b/raven/tracing/artifact_v2.py @@ -6,8 +6,10 @@ the viewer's. The canonical serialization, the envelope key, the reference validation and the per-field resolution rules all live here and nowhere else. -Stdlib only, like :mod:`raven.tracing.store`, so the tracing package stays -installable on its own. +The wire shape itself is stdlib; the default reader decodes the store's zstd +frames through :mod:`raven.tracing.codec`, and falls back to a packed slice +through :mod:`raven.tracing.message_pack`, both imported lazily so this module +stays cheap to import. """ from __future__ import annotations @@ -18,6 +20,8 @@ from pathlib import Path from typing import Any, Callable +from .codec import MAX_PLAINTEXT_BYTES, read_plaintext + ARTIFACT_FORMAT = "audit.artifact.v2" MESSAGES_DIR_NAME = "_messages" REF_KEY = "$msg" @@ -108,12 +112,38 @@ def placeholder(sha1: str) -> dict[str, str]: return {"role": "unknown", "content": f"[message blob missing: {sha1}]"} +def _read_stored_text(path: Path, limit: int = MAX_PLAINTEXT_BYTES) -> str: + """A blob's plaintext, whichever encoding it was stored under.""" + plain = read_plaintext(path.read_bytes(), limit) + if plain is None: + raise ValueError("corrupt or oversized compressed message blob") + return plain.decode("utf-8") + + +def _packed_text(artifacts_dir: Path | str, sha1: str, limit: int = MAX_PLAINTEXT_BYTES) -> str | None: + """A packed message's text, or None when it is missing or cut mid-character. + + ``limit`` can end the slice inside a multi-byte character; that is not a + message this reader can hand back, so it degrades like a missing one. + """ + from . import message_pack + + packed = message_pack.read_message(artifacts_dir, sha1, limit) + if packed is None: + return None + try: + return packed.decode("utf-8") + except UnicodeDecodeError: + return None + + def resolve_payload( payload: Any, artifacts_dir: Path | str, *, on_missing: Callable[[str], Any] | None = None, read: Callable[[Path], str | None] | None = None, + max_total_bytes: int = MAX_PLAINTEXT_BYTES, ) -> Any: """A v2 shell as its v1 equivalent; any other payload returned unchanged. @@ -123,20 +153,41 @@ def resolve_payload( position, so omitting one shifts every later message and turns a locatable gap into a spurious full divergence. ``read`` replaces how a blob's text is read (None means it cannot be), for a caller that must bound what it reads. + + ``max_total_bytes`` bounds the plaintext one shell may pull in. A shell can + name any number of addresses, and an imported one is untrusted input, so + the addresses past the budget resolve to the missing-blob stand-in instead + of being read. """ if not is_v2(payload): return payload missing = placeholder if on_missing is None else on_missing cache: dict[str, Any] = {} + used = 0 def load(sha1: str) -> Any: + nonlocal used if sha1 not in cache: + text: str | None = None + remaining = max_total_bytes - used + if remaining > 0: + try: + path = message_path(artifacts_dir, sha1) + text = _read_stored_text(path, remaining) if read is None else read(path) + except (OSError, ValueError): + text = None + if text is not None and len(text.encode("utf-8")) > remaining: + text = None + if text is None and read is None: + # A packed address has no file; the other encoding of the + # same store is a pack slice. A caller-supplied read owns + # its own fallback, because it may bound or confine it. + text = _packed_text(artifacts_dir, sha1, remaining) try: - path = message_path(artifacts_dir, sha1) - text = path.read_text(encoding="utf-8") if read is None else read(path) if text is None: raise OSError(f"blob {sha1} could not be read") cache[sha1] = json.loads(text) + used += len(text.encode("utf-8")) except (OSError, ValueError): cache[sha1] = missing(sha1) return cache[sha1] diff --git a/raven/tracing/codec.py b/raven/tracing/codec.py new file mode 100644 index 000000000..7b832555e --- /dev/null +++ b/raven/tracing/codec.py @@ -0,0 +1,166 @@ +"""The zstd framing every stored payload shares. + +A payload is written as a zstd frame under the path it always had, so a reader +tells the encoding from the frame's magic and never from the file name. That +is what keeps a store written before compression readable and a half-migrated +one coherent. + +Only the encoding lives here: paths, dedupe and migration stay in +``store``/``compress``, so both can import this module without reaching back +into each other. +""" + +from __future__ import annotations + +import io +from pathlib import Path +from typing import BinaryIO + +import zstandard + +#: A zstd frame always opens with these four bytes. A JSON or text payload +#: cannot start with them (0xB5 is not a UTF-8 leading byte), which is what +#: makes the sniff safe for everything the store writes. +ZSTD_MAGIC = b"\x28\xb5\x2f\xfd" +COMPRESSION_LEVEL = 3 + +#: Ceiling on what one stored frame may decode to. The store's writer caps a +#: rotated log at 50 MiB and the largest captured payload observed is 32.9 MB, +#: so this is headroom rather than a routine limit. It exists because a trace +#: store can be imported from another machine: a small frame that expands to +#: gigabytes must be refused, not allocated. +MAX_PLAINTEXT_BYTES = 256 * 1024 * 1024 + + +def is_compressed(data: bytes) -> bool: + return data[:4] == ZSTD_MAGIC + + +def compress_bytes(data: bytes) -> bytes: + # A fresh compressor per call: ZstdCompressor instances are not documented + # as thread safe, and the store writes from more than one worker thread. + return zstandard.ZstdCompressor(level=COMPRESSION_LEVEL).compress(data) + + +def decompress_bytes(data: bytes, limit: int | None = None) -> bytes: + """The plaintext in ``data``, at most ``limit`` bytes of it. + + ``limit`` bounds the output, not just the returned slice: a caller that + only wants a preview must not pay for a whole multi-megabyte payload. + """ + if not data: + return b"" + with zstandard.ZstdDecompressor().stream_reader(io.BytesIO(data), read_across_frames=False) as reader: + return reader.read() if limit is None else reader.read(limit) + + +def read_plaintext(data: bytes, limit: int = MAX_PLAINTEXT_BYTES) -> bytes | None: + """The payload's plaintext, or None when unreadable or larger than ``limit``. + + A compressed frame is decoded only up to ``limit + 1`` bytes, so a crafted + frame is refused instead of allocated. Plain bytes were already read by the + caller and are returned as they are. + """ + if not is_compressed(data): + return data + try: + plain = decompress_bytes(data, limit + 1) + except zstandard.ZstdError: + return None + return None if len(plain) > limit else plain + + +def plaintext_size(path: Path) -> int | None: + """The payload's uncompressed length, from the frame header when possible.""" + try: + with path.open("rb") as handle: + head = handle.read(4) + if not is_compressed(head): + return path.stat().st_size + handle.seek(0) + try: + size = zstandard.frame_content_size(handle.read(18)) + except zstandard.ZstdError: + return None + if size >= 0: + return int(size) + # A frame written without a content size is only measurable by + # decoding it; the store's own writer always records the size. + # `read(18)` above consumed the header, so rewind before decoding. + handle.seek(0) + data = handle.read() + except OSError: + return None + plain = read_plaintext(data) + return None if plain is None else len(plain) + + +def read_head(path: Path, size: int) -> bytes | None: + """The first ``size`` plaintext bytes of a stored file, decoding no further.""" + try: + with path.open("rb") as handle: + if not is_compressed(handle.read(4)): + handle.seek(0) + return handle.read(size) + handle.seek(0) + head = b"" + with zstandard.ZstdDecompressor().stream_reader(handle, read_across_frames=False) as reader: + while len(head) < size: + chunk = reader.read(size - len(head)) + if not chunk: + break + head += chunk + return head + except (OSError, zstandard.ZstdError): + return None + + +def read_plaintext_file(path: Path, limit: int = MAX_PLAINTEXT_BYTES) -> bytes | None: + """A stored file's plaintext, or None when it cannot be read or decoded.""" + try: + return read_plaintext(path.read_bytes(), limit) + except OSError: + return None + + +def open_plaintext(path: Path, limit: int = MAX_PLAINTEXT_BYTES) -> BinaryIO: + """Open a stored file as a plaintext byte stream. + + A caller that seeks or sizes the stream works in plaintext offsets, which + is the only coordinate the store records (a span log's cursor, an + artifact's range). A compressed payload is decompressed whole, up to + ``limit`` bytes; a frame that expands past it is refused as unreadable. + """ + handle = path.open("rb") + try: + head = handle.read(4) + handle.seek(0) + if not is_compressed(head): + return handle + data = handle.read() + except BaseException: + handle.close() + raise + handle.close() + try: + plain = decompress_bytes(data, limit + 1) + except zstandard.ZstdError as exc: + raise OSError(f"corrupt compressed payload: {path}") from exc + if len(plain) > limit: + raise OSError(f"compressed payload over {limit} bytes: {path}") + return io.BytesIO(plain) + + +__all__ = [ + "COMPRESSION_LEVEL", + "MAX_PLAINTEXT_BYTES", + "ZSTD_MAGIC", + "compress_bytes", + "decompress_bytes", + "is_compressed", + "open_plaintext", + "plaintext_size", + "read_head", + "read_plaintext", + "read_plaintext_file", +] diff --git a/raven/tracing/compact.py b/raven/tracing/compact.py index 0ecfbe992..2d6c3307b 100644 --- a/raven/tracing/compact.py +++ b/raven/tracing/compact.py @@ -49,7 +49,11 @@ from dataclasses import dataclass, field from pathlib import Path +import zstandard + from raven.tracing import artifact_v2 +from raven.tracing.codec import ZSTD_MAGIC +from raven.tracing.message_pack import PACKS_DIR_NAME from raven.tracing.store import BLOBS_DIR_NAME # Directories under ``audit-artifacts/`` that are content-addressed stores, not @@ -66,7 +70,7 @@ _MAX_ERRORS = 500 -_CONTENT_STORES = frozenset({BLOBS_DIR_NAME, artifact_v2.MESSAGES_DIR_NAME}) +_CONTENT_STORES = frozenset({BLOBS_DIR_NAME, artifact_v2.MESSAGES_DIR_NAME, PACKS_DIR_NAME}) @dataclass @@ -110,10 +114,26 @@ def _record_error(result: CompactResult, message: str) -> None: def _sha1_of(path: Path) -> str: + """The plaintext digest: a compressed artifact hashes what it decodes to. + + A blob's name is the digest of the payload, never of its zstd frame, so a + compressed artifact has to be decoded before it can be compared with, or + folded onto, the blob it belongs to. + """ digest = hashlib.sha1() with path.open("rb") as handle: - for chunk in iter(lambda: handle.read(_READ_CHUNK), b""): - digest.update(chunk) + head = handle.read(4) + handle.seek(0) + if head == ZSTD_MAGIC: + try: + with zstandard.ZstdDecompressor().stream_reader(handle, read_across_frames=False) as reader: + for chunk in iter(lambda: reader.read(_READ_CHUNK), b""): + digest.update(chunk) + except zstandard.ZstdError as exc: + raise OSError(f"corrupt compressed artifact: {exc}") from exc + else: + for chunk in iter(lambda: handle.read(_READ_CHUNK), b""): + digest.update(chunk) return digest.hexdigest() diff --git a/raven/tracing/compress.py b/raven/tracing/compress.py new file mode 100644 index 000000000..85f8be1b6 --- /dev/null +++ b/raven/tracing/compress.py @@ -0,0 +1,259 @@ +"""Compress a trace store's payloads in place, under the paths they already have. + +A span records its artifact's path when it is written, and those records are +the audit trail: a migration cannot rename them. So this keeps every path and +changes only the bytes, then re-points each artifact hard link at the one +compressed copy of its payload. Readers pick the encoding from the zstd frame, +never from the name, which is also what lets a half-migrated store read. + +Idempotent: a blob or message already compressed is counted and left alone, so +a second pass writes nothing. A dry run compresses in memory to report the real +saving and writes no file at all. + +The rotated span/event logs are the same story one level up: closed files under +``archive/``, re-encoded in place. The active log and the ACP frame journals +are not touched -- the first is still being appended to, and a span records a +plaintext byte range into the second. +""" + +from __future__ import annotations + +import os +import uuid +from dataclasses import dataclass, field +from pathlib import Path +from typing import Iterator + +from . import artifact_v2 +from .codec import compress_bytes, is_compressed +from .message_pack import PACKS_DIR_NAME +from .store import BLOBS_DIR_NAME + + +@dataclass +class CompressResult: + """What one pass saw, did, and saved.""" + + blobs: int = 0 + compressed: int = 0 + already: int = 0 + relinked: int = 0 + messages: int = 0 + messages_compressed: int = 0 + bytes_before: int = 0 + bytes_after: int = 0 + errors: list[str] = field(default_factory=list) + + @property + def saved(self) -> int: + return max(self.bytes_before - self.bytes_after, 0) + + +def _files(root: Path) -> Iterator[Path]: + for directory, _, names in os.walk(root): + for name in names: + if name.startswith("."): + continue + path = Path(directory) / name + if path.is_file(): + yield path + + +def _replace(path: Path, payload: bytes, *, times: tuple[int, int] | None = None) -> None: + """Swap ``path``'s bytes without ever leaving the name pointing at nothing. + + ``times`` (atime, mtime in ns) goes on the new file before the swap, for a + file whose readers order by mtime. + """ + tmp = path.parent / f".{path.name}.{os.getpid()}.{uuid.uuid4().hex}.compress.tmp" + try: + tmp.write_bytes(payload) + if times is not None: + os.utime(tmp, ns=times) + os.replace(tmp, path) + finally: + tmp.unlink(missing_ok=True) + + +def _relink(blob: Path, artifact: Path, old_inode: int) -> bool: + """Point ``artifact`` at ``blob``, leaving it readable across the swap. + + A hard link is staged beside the artifact and renamed over it, so the path + never passes through a state where it does not exist. The inode check + keeps a path that something else has already replaced from being clobbered. + """ + try: + if artifact.stat().st_ino != old_inode: + return False + staged = artifact.parent / f".{artifact.name}.{os.getpid()}.{uuid.uuid4().hex}.relink" + try: + os.link(blob, staged) + os.replace(staged, artifact) + finally: + staged.unlink(missing_ok=True) + return True + except OSError: + return False + + +def compress(artifacts_dir: Path, *, dry_run: bool = False) -> CompressResult: + """Compress every plain payload under ``artifacts_dir``; see the module doc.""" + result = CompressResult() + artifacts_dir = Path(artifacts_dir) + if not artifacts_dir.is_dir(): + return result + blobs_dir = artifacts_dir / BLOBS_DIR_NAME + messages_dir = artifact_v2.messages_dir(artifacts_dir) + packs_dir = artifacts_dir / PACKS_DIR_NAME + + # One walk to learn which artifact paths share each blob inode; the links + # must be moved after the blob's own name is swapped. + links: dict[int, list[Path]] = {} + for path in _files(artifacts_dir): + if path.is_relative_to(blobs_dir) or path.is_relative_to(messages_dir) or path.is_relative_to(packs_dir): + continue + try: + inode = path.stat().st_ino + except OSError as exc: + result.errors.append(f"{path}: {exc}") + continue + links.setdefault(inode, []).append(path) + + for blob in _files(blobs_dir): + result.blobs += 1 + try: + data = blob.read_bytes() + inode = blob.stat().st_ino + except OSError as exc: + result.errors.append(f"{blob}: {exc}") + continue + if is_compressed(data): + result.already += 1 + continue + packed = compress_bytes(data) + result.bytes_before += len(data) + result.bytes_after += len(packed) + result.compressed += 1 + if dry_run: + continue + # Stage beside the blob first, move the artifact links onto the staged + # inode, and only then give the blob name that inode: a crash in the + # middle leaves every path readable, one encoding or the other. + staged = blob.parent / f".{blob.name}.{os.getpid()}.{uuid.uuid4().hex}.compress.tmp" + try: + staged.write_bytes(packed) + for artifact in links.get(inode, []): + if _relink(staged, artifact, inode): + result.relinked += 1 + os.replace(staged, blob) + except OSError as exc: + result.errors.append(f"{blob}: {exc}") + finally: + staged.unlink(missing_ok=True) + + for path in _files(messages_dir): + result.messages += 1 + try: + data = path.read_bytes() + except OSError as exc: + result.errors.append(f"{path}: {exc}") + continue + if is_compressed(data): + continue + packed = compress_bytes(data) + result.bytes_before += len(data) + result.bytes_after += len(packed) + result.messages_compressed += 1 + if dry_run: + continue + try: + _replace(path, packed) + except OSError as exc: + result.errors.append(f"{path}: {exc}") + return result + + +def _is_rotated_log(name: str) -> bool: + return (name.startswith("audit-spans-") or name.startswith("audit-events-")) and name.endswith(".log") + + +def _newest_of_each_kind(paths: list[Path]) -> set[Path]: + """The last rotated log of each kind (spans, events), by mtime.""" + newest: dict[str, tuple[int, Path]] = {} + for path in paths: + try: + mtime = path.stat().st_mtime_ns + except OSError: + continue + kind = path.name.split("-", 2)[1] + if kind not in newest or mtime > newest[kind][0]: + newest[kind] = (mtime, path) + return {path for _, path in newest.values()} + + +@dataclass +class LogCompressResult: + """What one pass over the rotated logs saw, did, and saved.""" + + files: int = 0 + compressed: int = 0 + already: int = 0 + newest: int = 0 + bytes_before: int = 0 + bytes_after: int = 0 + errors: list[str] = field(default_factory=list) + + @property + def saved(self) -> int: + return max(self.bytes_before - self.bytes_after, 0) + + +def compress_logs(logs_dir: Path, *, dry_run: bool = False) -> LogCompressResult: + """Compress rotated span and event logs in place. + + Only ``archive/`` is touched: the active log is still being appended to. + ACP frame journals are left alone -- a span records a plaintext byte range + into one, so re-encoding it would invalidate that audit record. + + The newest rotated log of each kind is left for the store to compress at + its next rotation: a writer that opened the active log just before it was + renamed may still append its last line to that inode, and a rewrite would + drop the line. Every older one is closed for good, which is why it can be + rewritten here and why a running writer needs no coordination. A compressed + log keeps its mtime, which is how the viewer orders logs. Idempotent: a + compressed file is counted and left alone; a dry run writes nothing. + """ + result = LogCompressResult() + archive = Path(logs_dir) / "archive" + if not archive.is_dir(): + return result + logs = sorted(p for p in _files(archive) if _is_rotated_log(p.name)) + newest = _newest_of_each_kind(logs) + for path in logs: + result.files += 1 + if path in newest: + result.newest += 1 + continue + try: + data = path.read_bytes() + stat = path.stat() + except OSError as exc: + result.errors.append(f"{path}: {exc}") + continue + if is_compressed(data): + result.already += 1 + continue + packed = compress_bytes(data) + result.bytes_before += len(data) + result.bytes_after += len(packed) + result.compressed += 1 + if dry_run: + continue + try: + _replace(path, packed, times=(stat.st_atime_ns, stat.st_mtime_ns)) + except OSError as exc: + result.errors.append(f"{path}: {exc}") + return result + + +__all__ = ["CompressResult", "LogCompressResult", "compress", "compress_logs"] diff --git a/raven/tracing/message_pack.py b/raven/tracing/message_pack.py new file mode 100644 index 000000000..7952b4518 --- /dev/null +++ b/raven/tracing/message_pack.py @@ -0,0 +1,533 @@ +"""Message blobs packed into a few large zstd frames. + +The content-addressed message store writes one small file per distinct message. +Most messages are a few kilobytes, so a file system spends a block on each and +the store is mostly block slack: on a developer machine 40 thousand messages +held 224 MB while their payloads were a fraction of that. Packing concatenates +message texts into multi-megabyte frames and keeps an ``sha1 -> (pack, offset, +length)`` index beside them. A reader that finds no file for an address reads +its slice instead. + +Packing is a migration, not a writer change: :func:`pack_messages` appends the +files to packs, writes the index naming their slices, and only then deletes the +files. A store that has never been packed reads exactly as before, a +half-packed one resolves each address from whichever of the two holds it, and a +crash at any point leaves every address readable. + +An index entry stores plaintext offsets, so a reader can slice a pack without +knowing how the frame compressed it. The index and a few decompressed packs are +cached per process: a conversation page rereads the same pack for every message +it shows. +""" + +from __future__ import annotations + +import json +import os +import re +import threading +import uuid +from collections import OrderedDict +from dataclasses import dataclass, field +from pathlib import Path +from typing import Iterator + +import zstandard + +from .codec import MAX_PLAINTEXT_BYTES, compress_bytes, plaintext_size, read_plaintext + +PACKS_DIR_NAME = "_message_packs" +MESSAGES_DIR_NAME = "_messages" +#: The message directory's name is a literal rather than an import of +#: ``artifact_v2``: that module resolves packed addresses back through this +#: one, so importing it here would close an import cycle. The pack test pins +#: the two constants equal. +INDEX_NAME = "index.json" +PACK_FORMAT = "raven.message-pack.v1" +INDEX_FORMAT = "raven.message-pack-index.v1" +PACK_TARGET_BYTES = 8 * 1024 * 1024 + +#: Decompressed packs kept by one process. Sized for a few conversation pages; +#: a pack larger than the whole budget is streamed on every read instead. +CACHE_BYTES = 32 * 1024 * 1024 + +_SHA1_RE = re.compile(r"^[0-9a-f]{40}$") +#: A pack name is generated by this module, never taken from a payload. The +#: index lives in the store and can be imported with it, so a name with a path +#: separator or ``..`` must be refused before it is joined under the pack +#: directory. +_PACK_NAME_RE = re.compile(r"^[0-9a-f]{32}\.pack\.zst$") + + +def packs_dir(artifacts_dir: Path | str) -> Path: + return Path(artifacts_dir) / PACKS_DIR_NAME + + +def index_path(artifacts_dir: Path | str) -> Path: + return packs_dir(artifacts_dir) / INDEX_NAME + + +@dataclass(frozen=True) +class _Index: + messages: dict[str, tuple[str, int, int]] + packs: dict[str, int] + + +_INDEX_CACHE: dict[Path, tuple[tuple[int, int], _Index]] = {} +_INDEX_LOCK = threading.Lock() +_PACK_CACHE: "OrderedDict[Path, bytes]" = OrderedDict() +_PACK_CACHE_BYTES = 0 +_PACK_LOCK = threading.Lock() + + +def _evict_index(path: Path) -> None: + with _INDEX_LOCK: + _INDEX_CACHE.pop(path, None) + + +def load_index(artifacts_dir: Path | str) -> _Index | None: + """The artifact store's pack index, cached until the file changes.""" + target = index_path(artifacts_dir) + try: + stat_result = target.stat() + except OSError: + return None + stamp = (stat_result.st_mtime_ns, stat_result.st_size) + with _INDEX_LOCK: + cached = _INDEX_CACHE.get(target) + if cached is not None and cached[0] == stamp: + return cached[1] + try: + raw = json.loads(target.read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + if not isinstance(raw, dict) or raw.get("format") != INDEX_FORMAT: + return None + messages_raw = raw.get("messages") + packs_raw = raw.get("packs") + if not isinstance(messages_raw, dict) or not isinstance(packs_raw, dict): + return None + messages: dict[str, tuple[str, int, int]] = {} + for sha1, entry in messages_raw.items(): + if not isinstance(sha1, str) or not _SHA1_RE.match(sha1): + continue + if ( + isinstance(entry, list) + and len(entry) == 3 + and isinstance(entry[0], str) + and _PACK_NAME_RE.match(entry[0]) + and all(isinstance(value, int) and not isinstance(value, bool) and value >= 0 for value in entry[1:]) + ): + messages[sha1] = (entry[0], entry[1], entry[2]) + packs: dict[str, int] = {} + for name, info in packs_raw.items(): + if not isinstance(name, str) or not isinstance(info, dict): + continue + size = info.get("bytes", 0) + if isinstance(size, int) and not isinstance(size, bool): + packs[name] = size + index = _Index(messages=messages, packs=packs) + with _INDEX_LOCK: + _INDEX_CACHE[target] = (stamp, index) + return index + + +def _cached_pack(path: Path) -> bytes | None: + """A pack's plaintext when it fits the cache budget, else None to stream it. + + A pack larger than the whole budget is never decoded here: the caller + streams to the slice it needs instead, which is the memory the budget + exists to save. + """ + global _PACK_CACHE_BYTES + with _PACK_LOCK: + cached = _PACK_CACHE.get(path) + if cached is not None: + _PACK_CACHE.move_to_end(path) + return cached + size = plaintext_size(path) + if size is None or size > CACHE_BYTES: + return None + try: + data = path.read_bytes() + plain = read_plaintext(data, CACHE_BYTES + 1) + except OSError: + return None + if plain is None or len(plain) > CACHE_BYTES: + return None + with _PACK_LOCK: + _PACK_CACHE[path] = plain + _PACK_CACHE_BYTES += len(plain) + while _PACK_CACHE_BYTES > CACHE_BYTES and len(_PACK_CACHE) > 1: + _, dropped = _PACK_CACHE.popitem(last=False) + _PACK_CACHE_BYTES -= len(dropped) + return plain + + +def _stream_slice(path: Path, offset: int, length: int) -> bytes | None: + try: + with ( + path.open("rb") as handle, + zstandard.ZstdDecompressor().stream_reader(handle, read_across_frames=False) as reader, + ): + remaining = offset + while remaining > 0: + chunk = reader.read(min(remaining, 1 << 20)) + if not chunk: + return None + remaining -= len(chunk) + return reader.read(length) + except (OSError, zstandard.ZstdError): + return None + + +def _slice(artifacts_dir: Path, sha1: str, limit: int | None) -> bytes | None: + index = load_index(artifacts_dir) + if index is None: + return None + located = index.messages.get(sha1) + if located is None: + return None + name, offset, length = located + want = length if limit is None else min(length, limit) + # The index can be imported with the store; an offset and length pair that + # asks for gigabytes is refused rather than allocated. The bound covers + # the skip too: the streaming reader decompresses up to offset + want. + want = min(want, MAX_PLAINTEXT_BYTES) + if offset < 0 or length < 0 or offset + want > MAX_PLAINTEXT_BYTES: + return None + pack = packs_dir(artifacts_dir) / name + plain = _cached_pack(pack) + if plain is None: + data = _stream_slice(pack, offset, want) + if data is None or len(data) != want: + return None + return data + if offset < 0 or offset + length > len(plain): + return None + return plain[offset : offset + want] + + +_LENGTH_CACHE: dict[Path, tuple[tuple[int, int], int | None]] = {} +_LENGTH_LOCK = threading.Lock() + + +def _decoded_length(path: Path) -> int | None: + """A pack's decoded plaintext length, or None when it does not decode. + + Streamed, so the plaintext is never held, and cached per process because a + pack is written once under a generated name; the stamp changes only when + the file is replaced. A frame cut short decodes to a shorter length + instead of raising, which is what tells an entry past the cut from one + before it. + """ + try: + stat_result = path.stat() + except OSError: + return None + stamp = (stat_result.st_mtime_ns, stat_result.st_size) + with _LENGTH_LOCK: + cached = _LENGTH_CACHE.get(path) + if cached is not None and cached[0] == stamp: + return cached[1] + length: int | None = None + try: + with ( + path.open("rb") as handle, + zstandard.ZstdDecompressor().stream_reader(handle, read_across_frames=False) as reader, + ): + total = 0 + for chunk in iter(lambda: reader.read(1 << 20), b""): + total += len(chunk) + length = total + except (OSError, zstandard.ZstdError): + length = None + with _LENGTH_LOCK: + _LENGTH_CACHE[path] = (stamp, length) + return length + + +def _pack_plaintext_length(artifacts_dir: Path, name: str, lengths: dict[str, int | None]) -> int | None: + """A pack's decoded length, or None when it does not decode whole. + + ``lengths`` memoizes one run, so several index entries naming a pack share + a check; ``_decoded_length`` keeps it across runs. A store copied + incompletely can leave a pack truncated, and the frame header still records + the original size then, so the decoded length is what decides whether an + entry still resolves. + """ + if name not in lengths: + lengths[name] = _decoded_length(packs_dir(artifacts_dir) / name) + return lengths[name] + + +def _slice_is_readable(artifacts_dir: Path, entry: tuple[str, int, int], lengths: dict[str, int | None]) -> bool: + """Whether the pack an index entry names decodes and covers its slice. + + A store can be copied or cleaned outside this module, so an address the + index lists may have no readable slice left; the message file is then the + only copy and must be kept. + """ + name, offset, length = entry + size = _pack_plaintext_length(artifacts_dir, name, lengths) + return size is not None and offset + length <= size + + +def slice_readable(artifacts_dir: Path | str, sha1: str) -> bool: + """Whether the pack a stored index names still decodes and covers this address. + + A store copied incompletely can leave the index naming a pack that is gone + or cut short; the address then has no readable copy in the packs. + """ + index = load_index(artifacts_dir) + located = index.messages.get(sha1) if index is not None else None + if located is None: + return False + name, offset, length = located + size = _decoded_length(packs_dir(artifacts_dir) / name) + return size is not None and offset + length <= size + + +def read_message(artifacts_dir: Path | str, sha1: str, limit: int | None = None) -> bytes | None: + """One packed message's plaintext, or None when the index does not hold it.""" + if not _SHA1_RE.match(sha1): + return None + return _slice(Path(artifacts_dir), sha1, limit) + + +def message_size(artifacts_dir: Path | str, sha1: str) -> int | None: + index = load_index(artifacts_dir) + if index is None: + return None + located = index.messages.get(sha1) + return located[2] if located is not None else None + + +def sha1_from_path(path: Path | str) -> str | None: + """The address a message-blob path names, or None when it is not one.""" + path = Path(path) + if not path.name.endswith(".json"): + return None + sha1 = path.name[: -len(".json")] + if not _SHA1_RE.match(sha1): + return None + if path.parent.name != sha1[:2] or path.parent.parent.name != MESSAGES_DIR_NAME: + return None + return sha1 + + +def _artifacts_from_path(path: Path) -> Path | None: + if sha1_from_path(path) is None: + return None + return path.parent.parent.parent + + +def read_path(path: Path | str, limit: int | None = None) -> bytes | None: + """The packed slice a message-file path would have held, or None.""" + target = Path(path) + if not target.is_absolute(): + target = target.resolve() + artifacts_dir = _artifacts_from_path(target) + sha1 = sha1_from_path(target) + if artifacts_dir is None or sha1 is None: + return None + return _slice(artifacts_dir, sha1, limit) + + +def size_path(path: Path | str) -> int | None: + target = Path(path) + if not target.is_absolute(): + target = target.resolve() + artifacts_dir = _artifacts_from_path(target) + sha1 = sha1_from_path(target) + if artifacts_dir is None or sha1 is None: + return None + return message_size(artifacts_dir, sha1) + + +def path_mtime(path: Path | str) -> float | None: + """When the slice's pack was written, for callers that need an mtime.""" + target = Path(path) + if not target.is_absolute(): + target = target.resolve() + artifacts_dir = _artifacts_from_path(target) + if artifacts_dir is None: + return None + index = load_index(artifacts_dir) + sha1 = sha1_from_path(target) + if index is None or sha1 is None: + return None + located = index.messages.get(sha1) + if located is None: + return None + try: + return (packs_dir(artifacts_dir) / located[0]).stat().st_mtime + except OSError: + return None + + +def _walk(root: Path) -> Iterator[Path]: + for directory, _, names in os.walk(root): + for name in names: + if name.startswith("."): + continue + path = Path(directory) / name + if path.is_file(): + yield path + + +def _atomic_write_index(artifacts_dir: Path, index: _Index) -> None: + payload = { + "format": INDEX_FORMAT, + "packs": {name: {"bytes": size} for name, size in index.packs.items()}, + "messages": {sha1: list(entry) for sha1, entry in index.messages.items()}, + } + target = index_path(artifacts_dir) + target.parent.mkdir(parents=True, exist_ok=True) + tmp = target.parent / f".{target.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp" + try: + tmp.write_text(json.dumps(payload, separators=(",", ":")), encoding="utf-8") + os.replace(tmp, target) + finally: + tmp.unlink(missing_ok=True) + _evict_index(target) + + +@dataclass +class PackResult: + """What one packing pass saw, did, and saved.""" + + scanned: int = 0 + packed: int = 0 + already: int = 0 + packs: int = 0 + removed: int = 0 + bytes_before: int = 0 + bytes_after: int = 0 + errors: list[str] = field(default_factory=list) + + @property + def saved(self) -> int: + return max(self.bytes_before - self.bytes_after, 0) + + +def pack_messages( + artifacts_dir: Path | str, *, dry_run: bool = False, target_bytes: int = PACK_TARGET_BYTES +) -> PackResult: + """Move unpacked message files into packs; see the module doc. + + Idempotent: an address already in the index is counted and left alone (its + file is removed if a previous run left one behind). A dry run compresses in + memory to report the real saving and writes or deletes nothing. + """ + artifacts_dir = Path(artifacts_dir) + messages_dir = artifacts_dir / MESSAGES_DIR_NAME + result = PackResult() + if not messages_dir.is_dir(): + return result + existing = load_index(artifacts_dir) + known = dict(existing.messages) if existing is not None else {} + packs = dict(existing.packs) if existing is not None else {} + lengths: dict[str, int | None] = {} + added: dict[str, tuple[str, int, int]] = {} + pending: list[tuple[str, bytes]] = [] + pending_bytes = 0 + candidates: list[tuple[Path, str]] = [] + + def flush() -> None: + nonlocal pending, pending_bytes + if not pending: + return + payload = b"".join(data for _, data in pending) + packed = compress_bytes(payload) + name = f"{uuid.uuid4().hex}.pack.zst" + offset = 0 + for sha1, data in pending: + added[sha1] = (name, offset, len(data)) + offset += len(data) + result.bytes_after += len(packed) + result.packs += 1 + if not dry_run: + target = packs_dir(artifacts_dir) / name + target.parent.mkdir(parents=True, exist_ok=True) + tmp = target.parent / f".{name}.{os.getpid()}.{uuid.uuid4().hex}.tmp" + try: + tmp.write_bytes(packed) + os.replace(tmp, target) + finally: + tmp.unlink(missing_ok=True) + packs[name] = len(payload) + pending = [] + pending_bytes = 0 + + for path in sorted(_walk(messages_dir)): + sha1 = path.name[: -len(".json")] if path.name.endswith(".json") else "" + if not _SHA1_RE.match(sha1) or path.name != f"{sha1}.json": + continue + result.scanned += 1 + candidates.append((path, sha1)) + if sha1 in known: + result.already += 1 + continue + try: + data = read_plaintext(path.read_bytes()) + except OSError as exc: + result.errors.append(f"{path}: {exc}") + continue + if data is None: + result.errors.append(f"{path}: corrupt compressed message") + continue + pending.append((sha1, data)) + pending_bytes += len(data) + result.packed += 1 + result.bytes_before += len(data) + if pending_bytes >= target_bytes: + flush() + flush() + + if added and not dry_run: + try: + _atomic_write_index(artifacts_dir, _Index(messages={**known, **added}, packs=packs)) + except OSError as exc: + result.errors.append(f"{index_path(artifacts_dir)}: {exc}") + return result + for path, sha1 in candidates: + if sha1 not in known and sha1 not in added: + continue + entry = known.get(sha1) or added.get(sha1) + if entry is not None and not _slice_is_readable(artifacts_dir, entry, lengths): + continue + try: + path.unlink() + result.removed += 1 + except OSError as exc: + result.errors.append(f"{path}: {exc}") + for orphan in packs_dir(artifacts_dir).glob("*.pack.zst"): + if orphan.name not in packs: + try: + orphan.unlink() + except OSError: + pass + return result + + +__all__ = [ + "CACHE_BYTES", + "INDEX_FORMAT", + "MESSAGES_DIR_NAME", + "PACKS_DIR_NAME", + "PACK_FORMAT", + "PACK_TARGET_BYTES", + "PackResult", + "index_path", + "load_index", + "message_size", + "pack_messages", + "packs_dir", + "path_mtime", + "read_message", + "read_path", + "sha1_from_path", + "size_path", + "slice_readable", +] diff --git a/raven/tracing/store.py b/raven/tracing/store.py index 9ad59e20f..35098ac6a 100644 --- a/raven/tracing/store.py +++ b/raven/tracing/store.py @@ -1,13 +1,14 @@ """JSONL + artifact storage for the trace core (audit.span.v1). -Dependency-free (stdlib only). Copied from the shared tracing-plugin core so -this package stays self-contained for a clean ``pip install``. +The one third-party dependency is ``zstandard``: a payload is stored as a zstd +frame under the path it always had, so readers tell the encoding from the +frame and a store written before compression stays readable. Layout under the state dir:: /logs/audit-events.log # one JSON event record per line /logs/audit-spans.log # one JSON span per line - /logs/audit-artifacts/_blobs//. # one copy per payload + /logs/audit-artifacts/_blobs//. # one zstd copy per payload /logs/audit-artifacts///... # hard links to those blobs /logs/archive//... # rotated logs """ @@ -23,7 +24,10 @@ from pathlib import Path from typing import Any +import zstandard + from . import artifact_v2 +from .codec import compress_bytes, is_compressed DEFAULT_MAX_BYTES = 50 * 1024 * 1024 @@ -114,9 +118,53 @@ def _rotate_if_needed(self, kind: str, next_text: str) -> Path: day_dir = self._ensure(self.archive_dir / current_day) suffix = datetime.now(timezone.utc).strftime("%H%M%S%f") base = _KIND_FILES[kind].replace(".log", "") - path.rename(day_dir / f"{base}-{current_day}-{suffix}.log") + rotated = day_dir / f"{base}-{current_day}-{suffix}.log" + path.rename(rotated) + self._compress_previous_archive(base, rotated) return path + def _compress_previous_archive(self, base: str, rotated: Path) -> None: + """Compress the log this kind's previous rotation archived. + + Not ``rotated`` itself: another writer may have opened the active log + just before the rename and still write into that inode, and a rewrite + would drop the line. Every append opens and closes the file, so nothing + holds a log one rotation older. One file per rotation bounds the cost + here; an older plain backlog is left to ``raven tracing compress --logs``. + """ + try: + archived = [p for p in self.archive_dir.glob(f"*/{base}-*.log") if p != rotated] + previous = max(archived, key=lambda p: p.stat().st_mtime_ns, default=None) + except OSError: + return + if previous is not None: + self._compress_rotated(previous) + + def _compress_rotated(self, path: Path) -> None: + """Store a rotated log compressed under the name it already has. + + The mtime is kept: the viewer orders logs by it, and a re-encoded + archive must not sort after the logs written later. A failure leaves + the log plain, which every reader accepts. + """ + try: + with path.open("rb") as handle: + if is_compressed(handle.read(4)): + return + handle.seek(0) + data = handle.read() + stat = os.fstat(handle.fileno()) + packed = compress_bytes(data) + tmp = path.parent / f".{path.name}.{os.getpid()}.{uuid.uuid4().hex}.compress.tmp" + try: + tmp.write_bytes(packed) + os.utime(tmp, ns=(stat.st_atime_ns, stat.st_mtime_ns)) + os.replace(tmp, path) + finally: + tmp.unlink(missing_ok=True) + except OSError: + pass + def append(self, kind: str, record: dict[str, Any]) -> None: text = f"{to_json_text(record)}\n" path = self._rotate_if_needed(kind, text) @@ -165,15 +213,25 @@ def _blob_is_intact(self, blob: Path, sha1: str) -> bool: return True digest = hashlib.sha1() with blob.open("rb") as handle: - for chunk in iter(lambda: handle.read(_BLOB_READ_CHUNK), b""): - digest.update(chunk) + head = handle.read(4) + handle.seek(0) + if is_compressed(head): + try: + with zstandard.ZstdDecompressor().stream_reader(handle, read_across_frames=False) as reader: + for chunk in iter(lambda: reader.read(_BLOB_READ_CHUNK), b""): + digest.update(chunk) + except zstandard.ZstdError: + return False + else: + for chunk in iter(lambda: handle.read(_BLOB_READ_CHUNK), b""): + digest.update(chunk) if digest.hexdigest() != sha1: return False self._verified_blobs[blob.name] = token return True - def _publish_blob(self, blob: Path, text: str, sha1: str) -> None: - """Create ``blob`` holding ``text``, losing gracefully to a racing writer. + def _publish_blob(self, blob: Path, payload: bytes, sha1: str) -> None: + """Create ``blob`` holding ``payload``, losing gracefully to a racing writer. The temp file is named with a ``uuid4`` (not just the pid), so two writers in the same process publishing the same new payload never @@ -186,7 +244,7 @@ def _publish_blob(self, blob: Path, text: str, sha1: str) -> None: self._ensure(blob.parent) tmp = blob.parent / f"{sha1}.{os.getpid()}.{uuid.uuid4().hex}.tmp" try: - tmp.write_text(text, encoding="utf-8") + tmp.write_bytes(payload) os.link(tmp, blob) self._remember_blob(blob) except FileExistsError: @@ -194,7 +252,7 @@ def _publish_blob(self, blob: Path, text: str, sha1: str) -> None: finally: tmp.unlink(missing_ok=True) - def _repair_blob(self, blob: Path, text: str, sha1: str) -> None: + def _repair_blob(self, blob: Path, payload: bytes, sha1: str) -> None: """Point ``blob`` back at the bytes its name claims. ``os.replace`` here, unlike the publish path: swapping the directory @@ -206,14 +264,14 @@ def _repair_blob(self, blob: Path, text: str, sha1: str) -> None: """ tmp = blob.parent / f"{sha1}.{os.getpid()}.{uuid.uuid4().hex}.tmp" try: - tmp.write_text(text, encoding="utf-8") + tmp.write_bytes(payload) os.replace(tmp, blob) self._remember_blob(blob) finally: tmp.unlink(missing_ok=True) def _materialize(self, file_path: Path, text: str, sha1: str, extension: str) -> None: - """Link ``file_path`` at the single stored copy of ``text``. + """Link ``file_path`` at the single stored copy of ``text``'s compressed form. Hard link, not copy: the payload occupies its blocks once however many spans reference it, and the link count the filesystem keeps is the @@ -228,17 +286,18 @@ def _materialize(self, file_path: Path, text: str, sha1: str, extension: str) -> per process, not per span, and none at all when the cached (inode, mtime, size) still matches. """ + payload = compress_bytes(text.encode("utf-8")) blob = self._blob_path(sha1, extension) try: if not blob.exists(): - self._publish_blob(blob, text, sha1) + self._publish_blob(blob, payload, sha1) if blob.exists() and not self._blob_is_intact(blob, sha1): - self._repair_blob(blob, text, sha1) + self._repair_blob(blob, payload, sha1) os.link(blob, file_path) except FileExistsError: pass except OSError: - file_path.write_text(text, encoding="utf-8") + file_path.write_bytes(payload) def persist_artifact( self, @@ -296,15 +355,29 @@ def address_items(self, items: list[Any]) -> list[Any]: text = artifact_v2.message_text(item) sha1 = artifact_v2.message_sha1(item) blob = artifact_v2.message_path(self.artifacts_dir, sha1) - if not blob.exists(): - self._publish_blob(blob, text, sha1) + payload = compress_bytes(text.encode("utf-8")) + if not blob.exists() and not self._is_packed(sha1): + self._publish_blob(blob, payload, sha1) if blob.exists() and not self._blob_is_intact(blob, sha1): - self._repair_blob(blob, text, sha1) + self._repair_blob(blob, payload, sha1) refs.append(artifact_v2.make_ref(sha1)) except OSError: refs.append(item) return refs + def _is_packed(self, sha1: str) -> bool: + """Whether a pack still holds this address, so its file is not written again. + + The index alone does not say so: a store copied incompletely keeps an + index naming slices whose pack is gone or cut short, and writing the + file is then how the address heals. The pack's decoded length is cached + per process, so this is one stat per address after each pack's first + check. + """ + from . import message_pack + + return message_pack.slice_readable(self.artifacts_dir, sha1) + @staticmethod def artifact_attributes(prefix: str, artifact: dict[str, Any] | None) -> dict[str, Any]: if not artifact: diff --git a/raven/trajectory/bundle.py b/raven/trajectory/bundle.py index 03907fdeb..4c511dcf1 100644 --- a/raven/trajectory/bundle.py +++ b/raven/trajectory/bundle.py @@ -40,6 +40,7 @@ from raven.config.paths import get_workspace_path from raven.tracing import artifact_v2 from raven.tracing import config as tracing_config +from raven.tracing.codec import read_plaintext from raven.trajectory.selection import read_store_file from raven.trajectory.store import attempt_alias_ids, iter_spans, pin, pin_attempt, resolve_attempt_id from raven.trajectory.verdict import Verdict, read_verdicts @@ -49,6 +50,16 @@ _ARTIFACTS_DIR = "artifacts" +def _stored_bytes(path: Path) -> bytes | None: + """The payload's plaintext: a bundle leaves this machine, so it must not + carry the store's on-disk encoding.""" + try: + data = path.read_bytes() + except OSError: + return None + return read_plaintext(data) + + def _copy_resolved( src: Path, target: Path, @@ -65,8 +76,13 @@ def _copy_resolved( ``read`` bounds how the shell's blobs are read. """ try: - payload = json.loads(src.read_text(encoding="utf-8") if text is None else text) - except (OSError, ValueError): + if text is None: + stored = _stored_bytes(src) + if stored is None: + return False + text = stored.decode("utf-8") + payload = json.loads(text) + except (OSError, ValueError, UnicodeDecodeError): return False if not artifact_v2.is_v2(payload): return False @@ -149,7 +165,11 @@ def _import_artifact( if text is None or not _copy_resolved(src, target, store_artifacts, missing_messages, text=text, read=read): target.write_bytes(data) elif not _copy_resolved(src, target, store_artifacts, missing_messages): - shutil.copy2(src, target) + plain = _stored_bytes(src) + if plain is None: + missing.append(source) + return None + target.write_bytes(plain) rel = f"{_ARTIFACTS_DIR}/{name}" copied[source] = rel return rel diff --git a/raven/trajectory/conversation.py b/raven/trajectory/conversation.py index e1c3d9a09..af7008524 100644 --- a/raven/trajectory/conversation.py +++ b/raven/trajectory/conversation.py @@ -42,8 +42,11 @@ from pathlib import Path from typing import Any, Sequence +import zstandard + from raven.tracing import artifact_v2 from raven.tracing import config as tracing_config +from raven.tracing.codec import ZSTD_MAGIC from raven.trajectory import store as tstore _log = logging.getLogger("raven.trajectory.conversation") @@ -131,6 +134,18 @@ def _kind(name: str) -> str: return _KIND_BY_DOMAIN.get(name.split(".", 1)[0], "other") +def _read_packed_message(target: Path, limit: int) -> tuple[str | None, str | None]: + """A message whose file was moved into a pack; the cap matches a file read.""" + from raven.tracing import message_pack + + data = message_pack.read_path(target, limit + 1) + if data is None: + return None, "artifact missing" + if len(data) > limit: + return data[:limit].decode("utf-8", errors="replace"), f"content over {limit // 1024} KiB — truncated" + return data.decode("utf-8", errors="replace"), None + + def _read_artifact(state: Path, path: str, limit: int = _ARTIFACT_LIMIT) -> tuple[str | None, str | None]: """(text, degraded reason); (None, reason) means unreadable. @@ -148,10 +163,20 @@ def _read_artifact(state: Path, path: str, limit: int = _ARTIFACT_LIMIT) -> tupl return None, "artifact path outside the trace store" if target.exists() and not target.is_file(): return None, "artifact is not a regular file" + if not target.exists(): + return _read_packed_message(target, limit) with target.open("rb") as handle: - data = handle.read(limit + 1) + head = handle.read(4) + handle.seek(0) + if head == ZSTD_MAGIC: + with zstandard.ZstdDecompressor().stream_reader(handle, read_across_frames=False) as reader: + data = reader.read(limit + 1) + else: + data = handle.read(limit + 1) except OSError: return None, "artifact missing" + except zstandard.ZstdError: + return None, "artifact unreadable: corrupt compressed content" except ValueError: # A pointer with an embedded NUL cannot name a file. return None, "artifact unreadable: the path is not valid" diff --git a/raven/trajectory/details.py b/raven/trajectory/details.py index 609890d92..e7da18816 100644 --- a/raven/trajectory/details.py +++ b/raven/trajectory/details.py @@ -39,7 +39,8 @@ from pathlib import Path from typing import Any, Callable, Sequence -from raven.tracing import artifact_v2 +from raven.tracing import artifact_v2, message_pack +from raven.tracing import codec as trace_codec from raven.trajectory import conversation as _conv from raven.trajectory import entries as _entries from raven.trajectory import store as tstore @@ -491,10 +492,9 @@ def blob_prefix(self, sha1: str, limit: int) -> tuple[str | None, int | None] | if not self._allowed(True): self.exhausted = True return None - try: - size: int | None = path.stat().st_size - except OSError: - size = None + size: int | None = trace_codec.plaintext_size(path) + if size is None: + size = message_pack.size_path(path) text, _ = _conv._read_artifact(self.state, str(path), limit) if size is not None else (None, None) self.blobs += 1 self.bytes += len(text.encode("utf-8", errors="replace")) if text else 0 diff --git a/raven/trajectory/index.py b/raven/trajectory/index.py index acf9c667e..39d4276bc 100644 --- a/raven/trajectory/index.py +++ b/raven/trajectory/index.py @@ -13,7 +13,9 @@ - Files are identified by ``(st_dev, st_ino)``, so the rotation rename keeps the same cursor; whether a file has more to read is decided by its current - size against the consumed offset, never by a past EOF. + size against the consumed offset, never by a past EOF. Compressing a closed + log replaces its inode; the scanner recognises it by its plaintext head and + carries the cursor over instead of reading it again. - Trace membership is three-valued. A span whose ``session.key`` matches makes its trace a member; a root span's ``trace.dispatched_in_trace_id`` links a sub-agent trace to its parent, and the decision propagates down @@ -47,6 +49,7 @@ from pathlib import Path from typing import Any, Callable, Sequence +from raven.tracing import codec as trace_codec from raven.tracing import config as tracing_config from raven.trajectory import entries as _entries from raven.trajectory import store as tstore @@ -260,6 +263,22 @@ def __init__(self, state_dir: Path, *, max_line_bytes: int = MAX_LINE_BYTES) -> self.cursors: dict[FileKey, FileCursor] = {} self.active_key: FileKey | None = None self._active_path = state_dir / "logs" / "audit-spans.log" + # key -> (mtime_ns, raw size, plaintext size), so a catalog pass over + # an unchanged archive never reopens it just to learn its encoding. + self._sizes: dict[FileKey, tuple[int, int, int]] = {} + # old key -> new key of a log re-encoded under a new inode, so a + # recovery task holding the old key still finds where it was. + self._moved: dict[FileKey, FileKey] = {} + + def _plaintext_size(self, path: Path, key: FileKey, stat_result: Any) -> int | None: + cached = self._sizes.get(key) + if cached is not None and cached[0] == stat_result.st_mtime_ns and cached[1] == stat_result.st_size: + return cached[2] + size = trace_codec.plaintext_size(path) + if size is None: + return None + self._sizes[key] = (stat_result.st_mtime_ns, stat_result.st_size, size) + return size def _catalog(self) -> list[tuple[Path, FileKey, int, int]]: archive: list[tuple[Path, FileKey, int, int]] = [] @@ -269,7 +288,11 @@ def _catalog(self) -> list[tuple[Path, FileKey, int, int]]: stat_result = path.stat() except OSError: continue - item = (path, _file_key(stat_result), stat_result.st_size, stat_result.st_mtime_ns) + key = _file_key(stat_result) + size = self._plaintext_size(path, key, stat_result) + if size is None: + continue + item = (path, key, size, stat_result.st_mtime_ns) (active if path == self._active_path else archive).append(item) return archive + active @@ -284,6 +307,50 @@ def _fingerprint(handle: Any) -> str | None: return None return hashlib.sha1(head).hexdigest() + @staticmethod + def _head_fingerprint(path: Path) -> str | None: + """sha1 of a file's first FINGERPRINT_BYTES of plaintext, or None when it is shorter.""" + head = trace_codec.read_head(path, FINGERPRINT_BYTES) + if head is None or len(head) < FINGERPRINT_BYTES: + return None + return hashlib.sha1(head).hexdigest() + + def _adopt_reencoded(self, catalog: Sequence[tuple[Path, FileKey, int, int]]) -> None: + """Carry a cursor over to its log re-encoded under a new inode. + + Compressing a closed log writes a new file and renames it over the old + name, so the same plaintext comes back under a new key. Read as a new + file it would be replayed from offset 0: every position-keyed record + listed twice, and an index reset when the old cursor was mid-read. + + The old key is either gone or, since a file system hands a freed inode + to the next file created, held by a file shorter than what the cursor + consumed. Such a cursor whose plaintext head matches a file this + scanner has no cursor for, holding at least what was consumed, moves + to that file; the key's new holder is then read as a new file. + """ + sizes = {key: size for _, key, size, _ in catalog} + unknown = [(path, key, size) for path, key, size, _ in catalog if key not in self.cursors] + if not unknown: + return + vanished = [c for key, c in self.cursors.items() if c.fingerprint is not None and sizes.get(key, -1) < c.offset] + for path, key, size in unknown: + if not vanished: + return + fingerprint = self._head_fingerprint(path) + cursor = next((c for c in vanished if c.fingerprint == fingerprint and c.offset <= size), None) + if cursor is None: + continue + vanished.remove(cursor) + del self.cursors[cursor.key] + self._sizes.pop(cursor.key, None) + self._moved.pop(key, None) + self._moved[cursor.key] = key + if self.active_key == cursor.key: + self.active_key = key + cursor.key, cursor.path, cursor.mtime_seen = key, path, -1 + self.cursors[key] = cursor + def _generation_changed(self, catalog: Sequence[tuple[Path, FileKey, int, int]]) -> bool: present = {key: size for _, key, size, _ in catalog} for key, cursor in list(self.cursors.items()): @@ -291,6 +358,7 @@ def _generation_changed(self, catalog: Sequence[tuple[Path, FileKey, int, int]]) if cursor.offset < cursor.size_seen or cursor.partial: return True del self.cursors[key] + self._sizes.pop(key, None) continue if present[key] < cursor.offset: return True @@ -304,6 +372,7 @@ def scan(self, budget: ScanBudget) -> ScanBatch: batch = ScanBatch() catalog = self._catalog() batch.total_bytes = sum(size for _, _, size, _ in catalog) + self._adopt_reencoded(catalog) if self._generation_changed(catalog): batch.generation_changed = True return batch @@ -334,16 +403,31 @@ def scan(self, budget: ScanBudget) -> ScanBatch: batch.scanned_bytes = sum(c.offset for c in self.cursors.values()) return batch + @staticmethod + def _stream_stat(handle: Any, path: Path) -> tuple[Any, int]: + """The stat describing the opened stream, and its plaintext length. + + A real descriptor is stat-ed through the handle, so identity and size + come from the file actually read. A decompressed buffer has no + descriptor; its length is the frame's plaintext size and the path is + stat-ed for identity, with the fingerprint check beside it catching a + rotation in between. + """ + if hasattr(handle, "getbuffer"): + return path.stat(), len(handle.getbuffer()) + stat_result = os.fstat(handle.fileno()) + return stat_result, stat_result.st_size + def _verify(self, cursor: FileCursor) -> str: """Re-check an unchanged-size file's identity and first bytes; "ok", "retry" or "generation".""" try: - with cursor.path.open("rb") as handle: - stat_result = os.fstat(handle.fileno()) + with trace_codec.open_plaintext(cursor.path) as handle: + stat_result, size = self._stream_stat(handle, cursor.path) if _file_key(stat_result) != cursor.key: return "retry" - if stat_result.st_size < cursor.offset: + if size < cursor.offset: return "generation" - if stat_result.st_size >= FINGERPRINT_BYTES: + if size >= FINGERPRINT_BYTES: fingerprint = self._fingerprint(handle) if fingerprint is not None: if cursor.fingerprint is None: @@ -367,11 +451,10 @@ def _read(self, cursor: FileCursor, budget: ScanBudget, batch: ScanBatch) -> str """ assembler = _LineAssembler(cursor, self.max_line_bytes) try: - with cursor.path.open("rb") as handle: - stat_result = os.fstat(handle.fileno()) + with trace_codec.open_plaintext(cursor.path) as handle: + stat_result, size = self._stream_stat(handle, cursor.path) if _file_key(stat_result) != cursor.key: return "retry" - size = stat_result.st_size if size < cursor.offset: return "generation" cursor.size_seen = size @@ -410,6 +493,10 @@ def rescan(self, task: RecoveryTask, budget: ScanBudget) -> tuple[list[RawRecord keys = [key for _, key, _, _ in catalog] if not keys: return [], True + for _ in range(len(self._moved)): + if task.key not in self._moved: + break + task.key = self._moved[task.key] if task.key not in keys: # Removed out of band: what that file held of the trace is lost, # and the rest is read again from the oldest file still there. @@ -425,11 +512,10 @@ def rescan(self, task: RecoveryTask, budget: ScanBudget) -> tuple[list[RawRecord task.discarding = False assembler = _LineAssembler(task, self.max_line_bytes) try: - with path.open("rb") as handle: - stat_result = os.fstat(handle.fileno()) + with trace_codec.open_plaintext(path) as handle: + stat_result, size = self._stream_stat(handle, path) if _file_key(stat_result) != key: return records, False - size = stat_result.st_size handle.seek(task.offset) while task.offset < size: want = min(SCAN_CHUNK, size - task.offset, budget.remaining()) diff --git a/raven/trajectory/selection.py b/raven/trajectory/selection.py index 4a9e15be1..0d2cfddc5 100644 --- a/raven/trajectory/selection.py +++ b/raven/trajectory/selection.py @@ -39,6 +39,7 @@ from raven.tracing import artifact_v2 from raven.tracing import config as tracing_config +from raven.tracing.codec import ZSTD_MAGIC, read_plaintext, read_plaintext_file from raven.trajectory import entries as _entries from raven.trajectory import store as tstore @@ -133,10 +134,10 @@ def _records( themselves only. """ for path in tstore.span_log_paths(state_dir): - try: - raw_lines = path.read_bytes().splitlines() - except OSError: + data = read_plaintext_file(path) + if data is None: continue + raw_lines = data.splitlines() for raw_line in raw_lines: if needles and not any(needle in raw_line for needle in needles): continue @@ -348,18 +349,25 @@ def read_store_file(path: Path, root: Path) -> tuple[bytes, float] | None: """ try: target = Path(path).resolve() - if not target.is_relative_to(root): - return None - fd = os.open(target, os.O_RDONLY | getattr(os, "O_NONBLOCK", 0) | getattr(os, "O_BINARY", 0)) except (OSError, ValueError): return None + if not target.is_relative_to(root): + return None + try: + fd = os.open(target, os.O_RDONLY | getattr(os, "O_NONBLOCK", 0) | getattr(os, "O_BINARY", 0)) + except OSError: + return _read_packed_store_file(target, root) try: info = os.fstat(fd) if not stat.S_ISREG(info.st_mode): return None with os.fdopen(fd, "rb") as handle: fd = -1 - return handle.read(), info.st_mtime + data = handle.read() + from raven.tracing.codec import read_plaintext + + plain = read_plaintext(data) + return (plain, info.st_mtime) if plain is not None else None except OSError: return None finally: @@ -367,6 +375,23 @@ def read_store_file(path: Path, root: Path) -> tuple[bytes, float] | None: os.close(fd) +def _read_packed_store_file(target: Path, root: Path) -> tuple[bytes, float] | None: + """A message whose file moved into a pack, confined to the same root.""" + from raven.tracing import message_pack + + artifacts_dir = target.parent.parent.parent + try: + if not message_pack.packs_dir(artifacts_dir).resolve().is_relative_to(Path(root).resolve()): + return None + except (OSError, ValueError): + return None + data = message_pack.read_path(target) + mtime = message_pack.path_mtime(target) + if data is None or mtime is None: + return None + return data, mtime + + def _read_session(path: Path) -> tuple[dict[str, Any] | None, list[dict[str, Any]]] | None: try: lines = path.read_text(encoding="utf-8").splitlines() @@ -436,11 +461,23 @@ def _turn_input(turn: TurnSummary, state_dir: Path) -> str | None: return None try: with target.open("rb") as handle: - data = handle.read(_INPUT_READ_LIMIT + 1) - if len(data) > _INPUT_READ_LIMIT: - return None - payload = json.loads(data.decode("utf-8")) - except (OSError, UnicodeDecodeError, ValueError): + head = handle.read(4) + handle.seek(0) + if head == ZSTD_MAGIC: + # A turn input written after compression is a zstd frame; the + # bounded decoder is the same one every other artifact reader + # uses. A plain file keeps the bounded read it always had. + plain = read_plaintext(handle.read(), _INPUT_READ_LIMIT) + else: + data = handle.read(_INPUT_READ_LIMIT + 1) + plain = data if len(data) <= _INPUT_READ_LIMIT else None + except OSError: + return None + if plain is None: + return None + try: + payload = json.loads(plain.decode("utf-8")) + except (UnicodeDecodeError, ValueError): return None payload = artifact_v2.resolve_payload(payload, state_dir / "logs" / "audit-artifacts") return _text_of(payload.get("content")) if isinstance(payload, dict) else None diff --git a/raven/trajectory/store.py b/raven/trajectory/store.py index 250248d8f..08e2706bc 100644 --- a/raven/trajectory/store.py +++ b/raven/trajectory/store.py @@ -52,6 +52,7 @@ from loguru import logger from raven.tracing import config as tracing_config +from raven.tracing.codec import read_plaintext_file from raven.utils.atomic_io import atomic_update T = TypeVar("T") @@ -679,10 +680,10 @@ def iter_spans( if owner is not None: match_ids = set(defs[owner]["traces"]) for path in span_log_paths(state_dir): - try: - raw_lines = path.read_bytes().splitlines() - except OSError: + data = read_plaintext_file(path) + if data is None: continue + raw_lines = data.splitlines() for raw_line in raw_lines: # Decode per line: one invalid-UTF-8 byte must hide one record, # not the whole log file. diff --git a/tests/integration/test_tracing_viewer_e2e.py b/tests/integration/test_tracing_viewer_e2e.py index 17c045af9..210f0fae1 100644 --- a/tests/integration/test_tracing_viewer_e2e.py +++ b/tests/integration/test_tracing_viewer_e2e.py @@ -42,6 +42,7 @@ import subprocess import time import urllib.error +import urllib.parse import urllib.request from contextlib import closing, contextmanager from pathlib import Path @@ -148,6 +149,87 @@ def test_reads_the_active_log_and_the_archive(tmp_path): assert _session_ids(_get(port, "/api/data")) == {"live-session", "archived-session"} +def test_reads_a_compressed_archive(tmp_path): + from raven.tracing.codec import compress_bytes + + logs = tmp_path / "logs" + archive = logs / "archive" / "2026-07-01" / "audit-spans-2026-07-01-1.log" + _write_spans(archive, [_span("archived-session", "span-arch")]) + archive.write_bytes(compress_bytes(archive.read_bytes())) + _write_spans(logs / "audit-spans.log", [_span("live-session", "span-live")]) + + with _viewer(tmp_path) as port: + assert _session_ids(_get(port, "/api/data")) == {"live-session", "archived-session"} + + +def test_artifact_endpoint_resolves_a_packed_message(tmp_path): + from raven.tracing import artifact_v2 + from raven.tracing.message_pack import pack_messages + + logs = tmp_path / "logs" + artifacts = logs / "audit-artifacts" + message = {"role": "user", "content": "packed"} + sha1 = artifact_v2.message_sha1(message) + path = artifact_v2.message_path(artifacts, sha1) + path.parent.mkdir(parents=True) + path.write_text(artifact_v2.message_text(message), encoding="utf-8") + shell = artifacts / "llm.input" / "2026-08-01" / "a.json" + shell.parent.mkdir(parents=True) + shell.write_text( + json.dumps( + { + "artifactFormat": artifact_v2.ARTIFACT_FORMAT, + "messages": [artifact_v2.make_ref(sha1)], + "prompt": artifact_v2.make_ref(sha1), + } + ), + encoding="utf-8", + ) + assert pack_messages(artifacts).packed == 1 + + with _viewer(tmp_path) as port: + found = _get(port, "/api/artifact?path=" + urllib.parse.quote(str(shell), safe="")) + + assert found["parsed"]["messages"] == [message] + assert found["parsed"]["prompt"] == "packed" + + +def test_sessions_spawn_synthesis_reads_a_compressed_artifact(tmp_path): + from raven.tracing.codec import compress_bytes + + logs = tmp_path / "logs" + artifacts = logs / "audit-artifacts" / "tool" / "2026-08-01" + artifacts.mkdir(parents=True) + input_artifact = artifacts / "in.json" + output_artifact = artifacts / "out.json" + input_artifact.write_bytes( + compress_bytes( + json.dumps({"params": {"runtime": "subagent", "agentId": "raven", "task": "do"}}).encode("utf-8") + ) + ) + output_artifact.write_bytes( + compress_bytes(json.dumps({"result": {"status": "accepted", "childSessionKey": "child"}}).encode("utf-8")) + ) + span = _span("live-session", "span-spawn", name="tool.call") + span["attributes"].update( + { + "tool.name": "sessions_spawn", + "tool.call_id": "call-1", + "tool.args_preview": "[truncated]", + "tool.result_preview": "[truncated]", + "tool.input.artifact_path": str(input_artifact), + "tool.output.artifact_path": str(output_artifact), + } + ) + _write_spans(logs / "audit-spans.log", [span]) + + with _viewer(tmp_path) as port: + payload = _get(port, "/api/data") + + names = {s["name"] for session in payload["sessions"] for trace in session["traces"] for s in trace["spans"]} + assert "subagent.call" in names + + def test_ignores_a_log_planted_outside_the_active_and_archive_paths(tmp_path): logs = tmp_path / "logs" _write_spans(logs / "audit-spans.log", [_span("live-session", "span-live")]) @@ -161,6 +243,24 @@ def test_ignores_a_log_planted_outside_the_active_and_archive_paths(tmp_path): assert _session_ids(_get(port, "/api/data")) == {"live-session"} +def test_artifact_endpoint_decodes_a_compressed_payload(tmp_path): + from raven.tracing.codec import compress_bytes + + logs = tmp_path / "logs" + artifact = logs / "audit-artifacts" / "llm.input" / "2026-08-01" / "a.json" + artifact.parent.mkdir(parents=True) + payload = {"messages": [{"role": "user", "content": "hi" * 100}]} + artifact.write_bytes(compress_bytes(json.dumps(payload).encode("utf-8"))) + span = _span("live-session", "span-live") + span["attributes"]["llm.input.artifact_path"] = str(artifact) + _write_spans(logs / "audit-spans.log", [span]) + + with _viewer(tmp_path) as port: + found = _get(port, "/api/artifact?path=" + urllib.parse.quote(str(artifact), safe="")) + + assert found["parsed"] == payload + + def test_repeated_read_is_served_without_rebuilding(tmp_path): _write_spans(tmp_path / "logs" / "audit-spans.log", [_span("live-session", "span-live")]) diff --git a/tests/test_cli_tracing_commands.py b/tests/test_cli_tracing_commands.py index c69e41071..d7093dd50 100644 --- a/tests/test_cli_tracing_commands.py +++ b/tests/test_cli_tracing_commands.py @@ -12,6 +12,7 @@ from __future__ import annotations +import hashlib import json import os import signal @@ -393,12 +394,128 @@ def test_tracing_rejects_an_unknown_action_with_bracketed_markup(tmp_path, monke def test_a_below_minimum_node_is_refused(monkeypatch): """find_node deliberately returns the best node it saw even below the - minimum; the viewer used to discard the version and run on it anyway.""" + minimum; the viewer reads zstd-compressed payloads, whose decoder landed + in Node's zlib in 22.15.""" import pytest import typer import raven.cli.tracing_commands as tc - monkeypatch.setattr("raven.cli.tui_commands.find_node", lambda: ("/usr/bin/node", (20, 0, 0))) + monkeypatch.setattr("raven.cli.tui_commands.find_node", lambda *_accept: ("/usr/bin/node", (22, 14, 0))) with pytest.raises(typer.Exit): tc._resolve_node() + monkeypatch.setattr("raven.cli.tui_commands.find_node", lambda *_accept: ("/usr/bin/node", (22, 15, 0))) + assert tc._resolve_node() == "/usr/bin/node" + + +def test_node_23_before_23_8_is_refused(monkeypatch): + """zstd landed in Node 22.15 and 23.8, so a 23.x below 23.8 cannot decode.""" + import pytest + import typer + + import raven.cli.tracing_commands as tc + + monkeypatch.setattr("raven.cli.tui_commands.find_node", lambda *_accept: ("/usr/bin/node", (23, 0, 0))) + with pytest.raises(typer.Exit): + tc._resolve_node() + monkeypatch.setattr("raven.cli.tui_commands.find_node", lambda *_accept: ("/usr/bin/node", (23, 8, 0))) + assert tc._resolve_node() == "/usr/bin/node" + + +def test_a_zstd_node_behind_an_older_node_on_path_is_chosen(tmp_path, monkeypatch): + """find_node stops at the first node the TUI accepts (22.0); a PATH node + without zstd must not hide the installer's newer private runtime.""" + import subprocess + + from raven.cli import tui_commands + + exe = "node.exe" if tui_commands._is_windows() else "node" + on_path = tmp_path / "bin" / exe + runtime_dir = tmp_path / "home" / "runtime" / "node-v22.20.0" + managed = runtime_dir / exe if tui_commands._is_windows() else runtime_dir / "bin" / exe + for binary in (on_path, managed): + binary.parent.mkdir(parents=True, exist_ok=True) + binary.write_text("") + versions = {str(on_path): "v22.12.0", str(managed): "v22.20.0"} + + def fake_run(argv, **_kwargs): + return subprocess.CompletedProcess(argv, 0, stdout=f"{versions[argv[0]]}\n", stderr="") + + monkeypatch.setattr(tui_commands.subprocess, "run", fake_run) + monkeypatch.delenv("RAVEN_NODE", raising=False) + monkeypatch.delenv("VIRTUAL_ENV", raising=False) + monkeypatch.setenv("PATH", str(on_path.parent)) + monkeypatch.setenv("RAVEN_HOME", str(tmp_path / "home")) + + assert tc._resolve_node() == str(managed) + + +def _plain_blob_with_link(state_dir: Path, text: str) -> tuple[Path, Path]: + sha1 = hashlib.sha1(text.encode("utf-8")).hexdigest() + blob = state_dir / "logs" / "audit-artifacts" / "_blobs" / sha1[:2] / f"{sha1}.json" + blob.parent.mkdir(parents=True) + blob.write_text(text, encoding="utf-8") + artifact = _aged_artifact(state_dir, "a.json", text) + artifact.unlink() + os.link(blob, artifact) + stamp = time.time() - 3600 + os.utime(blob, (stamp, stamp)) + os.utime(artifact, (stamp, stamp)) + return artifact, blob + + +def test_tracing_compress_dry_run_reports_and_writes_nothing(tmp_path, monkeypatch): + monkeypatch.setenv("RAVEN_TRACING_DIR", str(tmp_path)) + text = '{"m":"same"}' + artifact, _ = _plain_blob_with_link(tmp_path, text) + + r = runner.invoke(app, ["tracing", "compress", "--dry-run"]) + + assert r.exit_code == 0 + assert "compressed 1" in r.output + assert "saved" in r.output + assert artifact.read_bytes() == text.encode("utf-8") + + +def test_tracing_compress_rewrites_the_blob_and_keeps_the_link(tmp_path, monkeypatch): + monkeypatch.setenv("RAVEN_TRACING_DIR", str(tmp_path)) + text = '{"m":"same"}' + artifact, blob = _plain_blob_with_link(tmp_path, text) + + r = runner.invoke(app, ["tracing", "compress"]) + + assert r.exit_code == 0 + assert artifact.read_bytes().startswith(b"\x28\xb5\x2f\xfd") + assert artifact.stat().st_ino == blob.stat().st_ino + + +def test_tracing_compress_logs_dry_run_reports_and_writes_nothing(tmp_path, monkeypatch): + monkeypatch.setenv("RAVEN_TRACING_DIR", str(tmp_path)) + archive = tmp_path / "logs" / "archive" / "2026-01-01" / "audit-spans-2026-01-01-1.log" + archive.parent.mkdir(parents=True) + archive.write_text('{"spanId": "s"}\n', encoding="utf-8") + os.utime(archive, ns=(1_000_000_000_000_000_000, 1_000_000_000_000_000_000)) + archive.with_name("audit-spans-2026-01-01-2.log").write_text('{"spanId": "t"}\n', encoding="utf-8") + + r = runner.invoke(app, ["tracing", "compress", "--logs", "--dry-run"]) + + assert r.exit_code == 0 + assert "logs 2: compressed 1, already 0, left for the next rotation 1" in r.output + assert archive.read_text(encoding="utf-8") == '{"spanId": "s"}\n' + + +def test_tracing_pack_messages_packs_the_message_store(tmp_path, monkeypatch): + from raven.tracing import artifact_v2 + + monkeypatch.setenv("RAVEN_TRACING_DIR", str(tmp_path)) + message = {"role": "user", "content": "hi"} + sha1 = artifact_v2.message_sha1(message) + path = artifact_v2.message_path(tmp_path / "logs" / "audit-artifacts", sha1) + path.parent.mkdir(parents=True) + path.write_text(artifact_v2.message_text(message), encoding="utf-8") + + r = runner.invoke(app, ["tracing", "pack-messages"]) + + assert r.exit_code == 0 + assert "packed 1" in r.output + assert not path.exists() diff --git a/tests/test_rpc_cli_dispatch.py b/tests/test_rpc_cli_dispatch.py index c1c3cf2d9..3601930ef 100644 --- a/tests/test_rpc_cli_dispatch.py +++ b/tests/test_rpc_cli_dispatch.py @@ -258,13 +258,13 @@ def test_channels_login_normal_channels_dispatch(): def test_blacklist_full_coverage(): - """9-entry blacklist (5 original + tui + onboard + upgrade + tracing compact). + """11-entry blacklist (5 original + tui + onboard + upgrade + 3 tracing maintenance). `tui` blocks recursive Ink+Node spawn; `onboard` blocks prompt_toolkit wizard stdin hijack; `upgrade` blocks replacing the active Raven process; - `tracing compact` blocks a full-store hash walk that can outrun the RPC - timeout with no cancellation of the backing thread. All are necessary - because reflection will otherwise let them through. + the three `tracing` maintenance actions block a full-store walk that can + outrun the RPC timeout with no cancellation of the backing thread. All are + necessary because reflection will otherwise let them through. """ from raven.rpc.methods.cli_dispatch import _DISPATCH_BLACKLIST @@ -278,9 +278,11 @@ def test_blacklist_full_coverage(): ("onboard",), ("upgrade",), ("tracing", "compact"), + ("tracing", "compress"), + ("tracing", "pack-messages"), } assert _DISPATCH_BLACKLIST == expected_entries, ( - f"blacklist drift; expected 9 prefix tuples (+ agent REPL special-case), got {_DISPATCH_BLACKLIST}" + f"blacklist drift; expected 11 prefix tuples (+ agent REPL special-case), got {_DISPATCH_BLACKLIST}" ) # Hard-reject probe (prefix match) assert _is_dispatch_compatible(["tui"]) is False @@ -291,8 +293,10 @@ def test_blacklist_full_coverage(): assert _is_dispatch_compatible(["upgrade", "--check"]) is False # prefix assert _is_dispatch_compatible(["tracing", "compact"]) is False assert _is_dispatch_compatible(["tracing", "compact", "--dry-run"]) is False # prefix + assert _is_dispatch_compatible(["tracing", "compress"]) is False + assert _is_dispatch_compatible(["tracing", "pack-messages"]) is False # Bare `tracing` and its other action stay dispatchable -- only the - # specific `compact` action is terminal-only. + # maintenance actions are terminal-only. assert _is_dispatch_compatible(["tracing"]) is True assert _is_dispatch_compatible(["tracing", "stop"]) is True @@ -303,7 +307,7 @@ def test_tracing_compact_is_rejected_whatever_the_option_order(): The blacklist compares fixed argv positions, so the prefix tuple alone only catches the action at argv[1]: `tracing --port 1 compact` would otherwise reach dispatch and run a real, writing compaction detached - past the timeout. `_is_tracing_compact` closes every order. + past the timeout. `_is_tracing_maintenance` closes every order. """ for argv in ( ["tracing", "--dry-run", "compact"], @@ -325,6 +329,35 @@ def test_tracing_compact_is_rejected_whatever_the_option_order(): assert _is_dispatch_compatible(argv) is True, argv +def test_tracing_compress_is_rejected_whatever_the_option_order(): + """``compress`` walks the whole artifact store and writes in place, like + ``compact``; it must never run detached from a TUI slash command.""" + for argv in ( + ["tracing", "compress"], + ["tracing", "compress", "--dry-run"], + ["tracing", "--dry-run", "compress"], + ["tracing", "--port", "1", "compress"], + ["tracing", "-p", "1", "compress"], + ): + assert _is_dispatch_compatible(argv) is False, argv + + +def test_tracing_pack_messages_is_rejected_whatever_the_option_order(): + """``pack-messages`` walks the whole message store and deletes files. + + Like the other maintenance actions it is positional, so Click accepts it + after any option and the fixed-position blacklist prefix would miss it. + """ + for argv in ( + ["tracing", "pack-messages"], + ["tracing", "pack-messages", "--dry-run"], + ["tracing", "--dry-run", "pack-messages"], + ["tracing", "--port", "1", "pack-messages"], + ["tracing", "-p", "1", "pack-messages"], + ): + assert _is_dispatch_compatible(argv) is False, argv + + # --------------------------------------------------------------------------- # Reflection-based dispatch compat (_DISPATCH_WHITELIST deleted) # --------------------------------------------------------------------------- diff --git a/tests/test_subagent_acp.py b/tests/test_subagent_acp.py index 19afa60df..cb97b038f 100644 --- a/tests/test_subagent_acp.py +++ b/tests/test_subagent_acp.py @@ -3715,6 +3715,14 @@ async def test_a_failed_turn_marks_the_span_and_still_stores_the_transcript(trac # ---- the cli lane records too ----------------------------------------------- +def _stored_text(path) -> str: + """A stored artifact's plaintext, whichever encoding it was written under.""" + from raven.tracing.codec import decompress_bytes, is_compressed + + data = Path(path).read_bytes() + return (decompress_bytes(data) if is_compressed(data) else data).decode("utf-8") + + def _cli_stub_config(name: str = "clistub", *, exit_code: int = 0) -> ThirdPartyCliSubagentConfig: script = ( "import sys; " @@ -3748,7 +3756,7 @@ async def test_the_cli_lane_records_the_same_span_shape(trace_dir, tmp_path: Pat assert "subagent.external.update_counts" not in attrs assert not span["events"] - stored = json.loads(Path(attrs["subagent.external.transcript.artifact_path"]).read_text(encoding="utf-8")) + stored = json.loads(_stored_text(attrs["subagent.external.transcript.artifact_path"])) assert stored["invocations"][0]["stdout"].strip() == "hello from the cli" assert stored["invocations"][0]["stderr"].strip() == "some diagnostics" @@ -3764,9 +3772,7 @@ async def test_a_failed_cli_invocation_keeps_its_transcript(trace_dir, tmp_path: span = next(s for s in _spans_written(trace_dir) if s.get("name") == SPAN_NAME) assert span["status"]["code"] == "ERROR" assert span["attributes"]["subagent.external.exit_code"] == 3 - stored = json.loads( - Path(span["attributes"]["subagent.external.transcript.artifact_path"]).read_text(encoding="utf-8") - ) + stored = json.loads(_stored_text(span["attributes"]["subagent.external.transcript.artifact_path"])) assert stored["invocations"][0]["stdout"].strip() == "hello from the cli" diff --git a/tests/test_tracing_api.py b/tests/test_tracing_api.py index 5d4ce0331..320b7c41d 100644 --- a/tests/test_tracing_api.py +++ b/tests/test_tracing_api.py @@ -16,6 +16,16 @@ from raven.tracing import spans as _spans from raven.tracing import store as _store_mod + + +def _stored_text(path): + """A stored payload's plaintext, whichever encoding it was written under.""" + from raven.tracing.codec import decompress_bytes, is_compressed + + data = Path(path).read_bytes() + return (decompress_bytes(data) if is_compressed(data) else data).decode("utf-8") + + from raven.tracing import trace @@ -198,7 +208,7 @@ def request_generation(self, **asked): assert attrs["llm.request.temperature"] == 0.95 assert attrs["llm.request.effort_sent"] == "reasoning.effort=high" assert "Z.AI" in attrs["llm.request.provider_fence"] - payload = json.loads(Path(attrs["llm.input.artifact_path"]).read_text(encoding="utf-8")) + payload = json.loads(_stored_text(attrs["llm.input.artifact_path"])) assert payload["generation"]["reasoning"] == {"effort": "high"} assert "secret" not in json.dumps(attrs) and "openrouter.ai" not in json.dumps(attrs) @@ -344,8 +354,7 @@ def test_skill_gate_extractor_records_the_subagent_roster(trace_dir): None, ) a = _spans_written(trace_dir)[0]["attributes"] - with open(a["skill.gate.input.artifact_path"], encoding="utf-8") as fh: - payload = json.load(fh) + payload = json.loads(_stored_text(a["skill.gate.input.artifact_path"])) assert payload["available_subagents"] == "Scribe [stateless] (writes decks)" @@ -508,7 +517,7 @@ def test_a_mutated_artifact_does_not_corrupt_the_next_record(trace_dir): Path(first["path"]).write_text("CORRUPT", encoding="utf-8") second = store.persist_artifact("tool.output", {"traceId": "t2"}, payload) - text = Path(second["path"]).read_text(encoding="utf-8") + text = _stored_text(second["path"]) assert json.loads(text) == payload assert hashlib.sha1(text.encode("utf-8")).hexdigest() == second["sha1"] @@ -548,7 +557,7 @@ def test_an_intact_blob_is_not_rehashed_for_every_span(trace_dir, monkeypatch): real_open = Path.open def counting_open(self, mode="r", *args, **kwargs): - if _store_mod.BLOBS_DIR_NAME in self.parts and "b" in mode: + if _store_mod.BLOBS_DIR_NAME in self.parts and "b" in mode and not self.name.endswith(".tmp"): blob_reads.append(self) return real_open(self, mode, *args, **kwargs) @@ -569,8 +578,8 @@ def test_identical_payloads_share_one_inode(trace_dir): p1, p2 = Path(first["path"]), Path(second["path"]) assert p1 != p2 assert p1.stat().st_ino == p2.stat().st_ino - assert json.loads(p1.read_text(encoding="utf-8")) == payload - assert json.loads(p2.read_text(encoding="utf-8")) == payload + assert json.loads(_stored_text(p1)) == payload + assert json.loads(_stored_text(p2)) == payload assert first["sha1"] == second["sha1"] assert first["bytes"] == second["bytes"] @@ -613,7 +622,7 @@ def _no_links(*_args, **_kwargs): art = store.persist_artifact("tool.input", {"traceId": "t1"}, payload) path = Path(art["path"]) - assert json.loads(path.read_text(encoding="utf-8")) == payload + assert json.loads(_stored_text(path)) == payload assert path.stat().st_nlink == 1 assert art.get("error") is None assert not list(store.blobs_dir.rglob("*.tmp")) @@ -621,7 +630,7 @@ def _no_links(*_args, **_kwargs): def test_failed_temp_write_leaves_no_orphan_tmp_file(trace_dir, monkeypatch): store = _spans._get_store() - real_write_text = Path.write_text + real_write_bytes = Path.write_bytes calls = {"n": 0} def _fail_first_call(self, data, *args, **kwargs): @@ -629,17 +638,17 @@ def _fail_first_call(self, data, *args, **kwargs): if calls["n"] == 1: # Simulate a write that reaches disk before it fails (e.g. ENOSPC # mid-write), so a leftover temp file would prove the bug. - real_write_text(self, data, *args, **kwargs) + real_write_bytes(self, data, *args, **kwargs) raise OSError("simulated disk full") - return real_write_text(self, data, *args, **kwargs) + return real_write_bytes(self, data, *args, **kwargs) - monkeypatch.setattr(Path, "write_text", _fail_first_call) + monkeypatch.setattr(Path, "write_bytes", _fail_first_call) payload = {"m": "diskfull"} art = store.persist_artifact("tool.input", {"traceId": "t1"}, payload) path = Path(art["path"]) - assert json.loads(path.read_text(encoding="utf-8")) == payload + assert json.loads(_stored_text(path)) == payload assert path.stat().st_nlink == 1 assert art.get("error") is None assert not list(store.blobs_dir.rglob("*.tmp")) @@ -652,7 +661,7 @@ def test_string_payloads_keep_the_txt_extension(trace_dir): path = Path(art["path"]) assert path.suffix == ".txt" - assert path.read_text(encoding="utf-8") == "raw text" + assert _stored_text(path) == "raw text" assert (store.blobs_dir / art["sha1"][:2] / f"{art['sha1']}.txt").exists() @@ -781,7 +790,7 @@ class _Resp: assert by["llm.call"]["attributes"]["llm.model"] assert by["tool.call"]["attributes"]["tool.name"] == "grep" output_path = Path(by["llm.call"]["attributes"]["llm.output.artifact_path"]) - assert json.loads(output_path.read_text(encoding="utf-8"))["thinking_blocks"] == _Resp.thinking_blocks + assert json.loads(_stored_text(output_path))["thinking_blocks"] == _Resp.thinking_blocks def test_llm_span_carries_the_request_size_and_the_transport_facts(trace_dir): @@ -820,13 +829,13 @@ def test_llm_span_carries_the_request_size_and_the_transport_facts(trace_dir): assert attrs["llm.request_image_bytes"] == 300 assert attrs["llm.request_bytes"] > 400 - input_payload = json.loads(Path(attrs["llm.input.artifact_path"]).read_text(encoding="utf-8")) + input_payload = json.loads(_stored_text(attrs["llm.input.artifact_path"])) assert input_payload["request"] == { "bytes": attrs["llm.request_bytes"], "images": 1, "imageBytes": 300, } - output_payload = json.loads(Path(attrs["llm.output.artifact_path"]).read_text(encoding="utf-8")) + output_payload = json.loads(_stored_text(attrs["llm.output.artifact_path"])) assert output_payload["call"]["body"] == '{"choices": []}' @@ -1179,7 +1188,7 @@ def test_address_items_repairs_a_mutated_message_blob(trace_dir): store_2 = _store_mod.TraceStore(trace_dir) store_2.address_items([message]) - assert blob.read_text(encoding="utf-8") == v2.message_text(message) + assert _stored_text(blob) == v2.message_text(message) def test_spans_address_items_returns_the_input_when_the_store_fails(trace_dir, monkeypatch): diff --git a/tests/test_tracing_compression_edges.py b/tests/test_tracing_compression_edges.py new file mode 100644 index 000000000..a43aafad7 --- /dev/null +++ b/tests/test_tracing_compression_edges.py @@ -0,0 +1,338 @@ +"""Edge paths of the compressed trace store, exercised directly. + +The happy paths live with their modules; these pin the branches a migration or +a damaged store reaches: an index write that fails, a file that cannot be +removed, a compressed log that is already compressed, and readers that must +degrade to "missing" instead of raising. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import zstandard + +from raven.tracing import artifact_v2, message_pack +from raven.tracing import codec as trace_codec +from raven.tracing.store import TraceStore +from raven.trajectory import bundle, selection +from raven.trajectory import index as tidx + +MAGIC = b"\x28\xb5\x2f\xfd" + + +def _publish(store: TraceStore, content: str = "hi") -> str: + return store.address_items([{"role": "user", "content": content}])[0]["$msg"] + + +def test_path_mtime_and_a_missing_pack(tmp_path: Path) -> None: + store = TraceStore(tmp_path) + sha1 = _publish(store) + message_pack.pack_messages(store.artifacts_dir) + path = artifact_v2.message_path(store.artifacts_dir, sha1) + assert message_pack.path_mtime(path) is not None + + next(message_pack.packs_dir(store.artifacts_dir).glob("*.pack.zst")).unlink() + assert message_pack.path_mtime(path) is None + + +def test_pack_without_a_message_directory(tmp_path: Path) -> None: + result = message_pack.pack_messages(tmp_path / "logs" / "audit-artifacts") + assert result.scanned == 0 + + +def test_pack_skips_a_file_that_is_not_a_message(tmp_path: Path) -> None: + store = TraceStore(tmp_path) + _publish(store) + (store.messages_dir / "notes.txt").write_text("x", encoding="utf-8") + + result = message_pack.pack_messages(store.artifacts_dir) + + assert result.packed == 1 + + +def test_pack_index_write_failure_keeps_the_files(tmp_path: Path, monkeypatch) -> None: + store = TraceStore(tmp_path) + sha1 = _publish(store) + path = artifact_v2.message_path(store.artifacts_dir, sha1) + + def boom(*args, **kwargs): + raise OSError("disk full") + + monkeypatch.setattr(message_pack, "_atomic_write_index", boom) + result = message_pack.pack_messages(store.artifacts_dir) + + assert result.errors + assert path.exists() + + +def test_pack_removes_an_orphan_pack(tmp_path: Path) -> None: + store = TraceStore(tmp_path) + _publish(store) + message_pack.pack_messages(store.artifacts_dir) + orphan = message_pack.packs_dir(store.artifacts_dir) / ("0" * 32 + ".pack.zst") + orphan.write_bytes(b"orphan") + _publish(store, "second") + + message_pack.pack_messages(store.artifacts_dir) + + assert not orphan.exists() + + +def test_pack_reports_a_file_it_cannot_remove(tmp_path: Path, monkeypatch) -> None: + store = TraceStore(tmp_path) + sha1 = _publish(store) + path = artifact_v2.message_path(store.artifacts_dir, sha1) + original = Path.unlink + + def failing(self, *args, **kwargs): + if self == path: + raise OSError("busy") + return original(self, *args, **kwargs) + + monkeypatch.setattr(Path, "unlink", failing) + result = message_pack.pack_messages(store.artifacts_dir) + + assert any("busy" in error for error in result.errors) + + +def test_blob_intact_rejects_a_corrupt_frame(tmp_path: Path) -> None: + store = TraceStore(tmp_path) + blob = store._blob_path("a" * 40, "json") + blob.parent.mkdir(parents=True) + blob.write_bytes(MAGIC + b" broken") + + assert store._blob_is_intact(blob, "a" * 40) is False + + +def test_compress_rotated_skips_a_compressed_log(tmp_path: Path) -> None: + store = TraceStore(tmp_path) + path = tmp_path / "rotated.log" + path.write_bytes(trace_codec.compress_bytes(b"line\n")) + before = path.read_bytes() + + store._compress_rotated(path) + + assert path.read_bytes() == before + + +def test_compress_rotated_swallows_a_missing_file(tmp_path: Path) -> None: + store = TraceStore(tmp_path) + store._compress_rotated(tmp_path / "missing.log") + + +def test_stored_bytes_of_a_missing_file_is_none(tmp_path: Path) -> None: + assert bundle._stored_bytes(tmp_path / "gone.json") is None + + +def test_copy_resolved_rejects_invalid_json(tmp_path: Path) -> None: + source = tmp_path / "src.json" + source.write_text("{not json", encoding="utf-8") + + assert bundle._copy_resolved(source, tmp_path / "out.json", tmp_path / "artifacts", []) is False + + +def test_plaintext_size_cache_hit_and_unreadable_file(tmp_path: Path) -> None: + logs = tmp_path / "logs" + logs.mkdir() + (logs / "audit-spans.log").write_text('{"spanId": "s"}\n', encoding="utf-8") + # A name the chain lists but whose plaintext size cannot be read is skipped. + (logs / "archive" / "2026-01-01" / "audit-spans-1.log").mkdir(parents=True) + + scanner = tidx.SpanLogScanner(tmp_path) + first = scanner._catalog() + second = scanner._catalog() + + assert first == second + assert [path.name for path, *_ in first] == ["audit-spans.log"] + + +def test_a_packed_read_outside_the_root_is_refused(tmp_path: Path) -> None: + store = TraceStore(tmp_path / "store") + sha1 = _publish(store) + message_pack.pack_messages(store.artifacts_dir) + path = artifact_v2.message_path(store.artifacts_dir, sha1) + outside = tmp_path / "elsewhere" + outside.mkdir() + + assert selection.read_store_file(path, outside) is None + + +def test_turn_input_degrades_on_bad_json(tmp_path: Path) -> None: + state = tmp_path / "state" + (state / "logs").mkdir(parents=True) + artifact = state / "logs" / "audit-artifacts" / "turn.input" / "2026-10-01" / "bad.json" + artifact.parent.mkdir(parents=True) + artifact.write_text("{not json", encoding="utf-8") + turn = selection.TurnSummary( + trace_id="t", + span_id="s", + number=1, + start="2026-10-01T10:00:00+00:00", + end=None, + input_preview="", + in_progress=False, + input_artifact=str(artifact), + main_line=True, + ) + + assert selection._turn_input(turn, state) is None + + +def test_stream_slice_and_slice_fallbacks(tmp_path: Path, monkeypatch) -> None: + store = TraceStore(tmp_path / "store") + sha1 = _publish(store) + message_pack.pack_messages(store.artifacts_dir) + pack = next(message_pack.packs_dir(store.artifacts_dir).glob("*.pack.zst")) + assert message_pack._stream_slice(pack, 0, 5) is not None + assert message_pack._stream_slice(pack, 10**9, 1) is None + assert message_pack._stream_slice(tmp_path / "missing", 0, 1) is None + + monkeypatch.setattr(message_pack, "_cached_pack", lambda path: None) + assert message_pack.read_message(store.artifacts_dir, sha1) is not None + + +def test_slice_refuses_an_out_of_range_entry(tmp_path: Path) -> None: + store = TraceStore(tmp_path) + sha1 = _publish(store) + message_pack.pack_messages(store.artifacts_dir) + index_file = message_pack.index_path(store.artifacts_dir) + data = json.loads(index_file.read_text(encoding="utf-8")) + data["messages"][sha1] = [data["messages"][sha1][0], 10**9, 10] + index_file.write_text(json.dumps(data), encoding="utf-8") + + assert message_pack.read_message(store.artifacts_dir, sha1) is None + + +def test_address_and_path_validation() -> None: + assert message_pack.read_message("/tmp/none", "not-a-sha") is None + assert message_pack.sha1_from_path("x.txt") is None + assert message_pack.sha1_from_path("_messages/zz/" + "a" * 40 + ".json") is None + + +def test_relative_message_paths_are_resolved(tmp_path: Path, monkeypatch) -> None: + store = TraceStore(tmp_path / "store") + sha1 = _publish(store) + message_pack.pack_messages(store.artifacts_dir) + path = artifact_v2.message_path(store.artifacts_dir, sha1) + monkeypatch.chdir(tmp_path) + relative = path.relative_to(tmp_path) + + assert message_pack.read_path(relative) is not None + assert message_pack.size_path(relative) is not None + assert message_pack.path_mtime(relative) is not None + + +def test_path_mtime_of_an_address_that_was_not_packed(tmp_path: Path) -> None: + store = TraceStore(tmp_path) + sha1 = "b" * 40 + assert message_pack.path_mtime(artifact_v2.message_path(store.artifacts_dir, sha1)) is None + + +def test_walk_skips_hidden_files(tmp_path: Path) -> None: + store = TraceStore(tmp_path) + _publish(store) + (store.messages_dir / ".hidden").write_text("x", encoding="utf-8") + + result = message_pack.pack_messages(store.artifacts_dir) + + assert result.packed == 1 + + +def test_orphan_sweep_ignores_an_entry_it_cannot_remove(tmp_path: Path) -> None: + store = TraceStore(tmp_path) + _publish(store) + message_pack.pack_messages(store.artifacts_dir) + orphan = message_pack.packs_dir(store.artifacts_dir) / ("0" * 32 + ".pack.zst") + orphan.mkdir() + _publish(store, "second") + + result = message_pack.pack_messages(store.artifacts_dir) + + assert result.packed == 1 + assert orphan.is_dir() + + +def test_import_artifact_reports_a_file_it_cannot_read(tmp_path: Path, monkeypatch) -> None: + source = tmp_path / "src.bin" + source.write_bytes(b"raw") + monkeypatch.setattr(bundle, "_stored_bytes", lambda path: None) + missing: list[str] = [] + + result = bundle._import_artifact(str(source), tmp_path / "out", {}, missing, {}, tmp_path / "artifacts", []) + + assert result is None + assert missing == [str(source)] + + +def test_copy_resolved_rejects_invalid_json_again(tmp_path: Path) -> None: + source = tmp_path / "src.json" + source.write_text("[1, 2", encoding="utf-8") + + assert bundle._copy_resolved(source, tmp_path / "out.json", tmp_path / "artifacts", []) is False + + +def _turn(tmp_path: Path, artifact: Path) -> selection.TurnSummary: + return selection.TurnSummary( + trace_id="t", + span_id="s", + number=1, + start="2026-10-01T10:00:00+00:00", + end=None, + input_preview="", + in_progress=False, + input_artifact=str(artifact), + main_line=True, + ) + + +def test_turn_input_degrades_when_the_file_vanishes(tmp_path: Path, monkeypatch) -> None: + state = tmp_path / "state" + artifact = state / "logs" / "audit-artifacts" / "turn.input" / "2026-10-01" / "a.json" + artifact.parent.mkdir(parents=True) + artifact.write_text('{"content": "hi"}', encoding="utf-8") + original = Path.open + + def failing(self, *args, **kwargs): + if self == artifact: + raise OSError("gone") + return original(self, *args, **kwargs) + + monkeypatch.setattr(Path, "open", failing) + + assert selection._turn_input(_turn(tmp_path, artifact), state) is None + + +def test_turn_input_degrades_on_an_oversized_plain_artifact(tmp_path: Path) -> None: + state = tmp_path / "state" + artifact = state / "logs" / "audit-artifacts" / "turn.input" / "2026-10-01" / "big.json" + artifact.parent.mkdir(parents=True) + artifact.write_text("x" * (selection._INPUT_READ_LIMIT + 1), encoding="utf-8") + + assert selection._turn_input(_turn(tmp_path, artifact), state) is None + + +def test_plaintext_size_decodes_a_frame_without_a_content_size(tmp_path: Path) -> None: + payload = b"x" * 4096 + path = tmp_path / "audit-spans-2026-01-01-1.log" + path.write_bytes(zstandard.ZstdCompressor(write_content_size=False).compress(payload)) + + assert trace_codec.plaintext_size(path) == len(payload) + + +def test_a_frame_without_a_content_size_does_not_reset_the_scanner(tmp_path: Path) -> None: + archive = tmp_path / "logs" / "archive" / "2026-01-01" / "audit-spans-2026-01-01-1.log" + archive.parent.mkdir(parents=True) + record = {"traceId": "t", "spanId": "s", "name": "tool.call", "attributes": {}} + archive.write_bytes( + zstandard.ZstdCompressor(write_content_size=False).compress(json.dumps(record).encode() + b"\n") + ) + + scanner = tidx.SpanLogScanner(tmp_path) + first = scanner.scan(tidx.ScanBudget(max_bytes=10_000, deadline=None)) + second = scanner.scan(tidx.ScanBudget(max_bytes=10_000, deadline=None)) + + assert [raw.span["spanId"] for raw in first.records] == ["s"] + assert second.generation_changed is False + assert second.done is True diff --git a/tests/test_tracing_message_pack.py b/tests/test_tracing_message_pack.py new file mode 100644 index 000000000..e3ac77e4c --- /dev/null +++ b/tests/test_tracing_message_pack.py @@ -0,0 +1,273 @@ +"""Packed message blobs: a few large frames plus an address index. + +A packed store keeps the artifact contract: an address resolves whether its +bytes still live in a file or in a pack slice, every reader that resolved a +file before resolves a packed address, and a migration never deletes a file +before the index naming its slice is durable. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from raven.tracing import artifact_v2, message_pack +from raven.tracing.store import TraceStore +from raven.trajectory import conversation, details, selection + +MESSAGES = [ + {"role": "user", "content": "first " * 100}, + {"role": "assistant", "content": "second " * 100}, +] + + +def _publish(state: Path) -> list[dict]: + return TraceStore(state).address_items(MESSAGES) + + +def test_the_pack_module_names_the_artifact_store_directory() -> None: + """The literal exists to keep artifact_v2 out of this module's imports.""" + assert message_pack.MESSAGES_DIR_NAME == artifact_v2.MESSAGES_DIR_NAME + + +def test_pack_moves_files_into_a_pack_and_every_reader_resolves(tmp_path: Path) -> None: + refs = _publish(tmp_path) + sha1s = [ref["$msg"] for ref in refs] + artifacts = tmp_path / "logs" / "audit-artifacts" + texts = {sha1: artifact_v2.message_text(message).encode() for sha1, message in zip(sha1s, MESSAGES)} + assert all(artifact_v2.message_path(artifacts, sha1).is_file() for sha1 in sha1s) + + result = message_pack.pack_messages(artifacts) + + assert (result.packed, result.packs, result.removed) == (2, 1, 2) + assert result.errors == [] + assert not any(artifact_v2.message_path(artifacts, sha1).exists() for sha1 in sha1s) + for sha1 in sha1s: + assert message_pack.read_message(artifacts, sha1) == texts[sha1] + + shell = {"artifactFormat": artifact_v2.ARTIFACT_FORMAT, "messages": refs, "prompt": refs[0]} + resolved = artifact_v2.resolve_payload(shell, artifacts) + assert resolved["messages"] == MESSAGES + assert resolved["prompt"] == MESSAGES[0]["content"] + + path = str(artifact_v2.message_path(artifacts, sha1s[0])) + text, reason = conversation._read_artifact(tmp_path, path) + assert reason is None + assert json.loads(text) == MESSAGES[0] + limited, note = conversation._read_artifact(tmp_path, path, 10) + assert limited == texts[sha1s[0]][:10].decode() + assert note is not None and "truncated" in note + + reader = details._Reader(tmp_path, max_artifacts=None, max_blobs=None, max_bytes=None) + prefix, size = reader.blob_prefix(sha1s[0], 12) + assert prefix == texts[sha1s[0]][:12].decode() + assert size == len(texts[sha1s[0]]) + + found = selection.read_store_file(artifact_v2.message_path(artifacts, sha1s[0]), artifacts.resolve()) + assert found is not None + assert found[0] == texts[sha1s[0]] + + +def test_a_shell_stops_resolving_after_its_total_budget(tmp_path: Path) -> None: + refs = _publish(tmp_path) + artifacts = tmp_path / "logs" / "audit-artifacts" + first = len(artifact_v2.message_text(MESSAGES[0]).encode()) + shell = {"artifactFormat": artifact_v2.ARTIFACT_FORMAT, "messages": refs} + + resolved = artifact_v2.resolve_payload(shell, artifacts, max_total_bytes=first + 1) + + assert resolved["messages"][0] == MESSAGES[0] + assert "missing" in resolved["messages"][1]["content"] + + +def test_a_pack_name_that_escapes_the_store_is_refused(tmp_path: Path) -> None: + refs = _publish(tmp_path) + artifacts = tmp_path / "logs" / "audit-artifacts" + message_pack.pack_messages(artifacts) + secret = tmp_path / "secret.json" + secret.write_text('{"secret": true}', encoding="utf-8") + index_file = message_pack.index_path(artifacts) + data = json.loads(index_file.read_text(encoding="utf-8")) + data["messages"][refs[0]["$msg"]] = ["../../../../secret.json", 0, 16] + index_file.write_text(json.dumps(data), encoding="utf-8") + + assert message_pack.read_message(artifacts, refs[0]["$msg"]) is None + shell = {"artifactFormat": artifact_v2.ARTIFACT_FORMAT, "messages": [refs[0]]} + resolved = artifact_v2.resolve_payload(shell, artifacts) + assert "missing" in resolved["messages"][0]["content"] + + +def test_a_small_target_splits_messages_across_packs(tmp_path: Path) -> None: + refs = _publish(tmp_path) + artifacts = tmp_path / "logs" / "audit-artifacts" + + result = message_pack.pack_messages(artifacts, target_bytes=1) + + assert result.packs == 2 + for ref, message in zip(refs, MESSAGES): + assert json.loads(message_pack.read_message(artifacts, ref["$msg"])) == message + + +def test_pack_is_idempotent_and_dry_run_writes_nothing(tmp_path: Path) -> None: + refs = _publish(tmp_path) + artifacts = tmp_path / "logs" / "audit-artifacts" + files = [artifact_v2.message_path(artifacts, ref["$msg"]) for ref in refs] + + dry = message_pack.pack_messages(artifacts, dry_run=True) + assert (dry.packed, dry.packs) == (2, 1) + assert all(path.is_file() for path in files) + assert not message_pack.index_path(artifacts).exists() + + once = message_pack.pack_messages(artifacts) + assert once.packed == 2 + again = message_pack.pack_messages(artifacts) + assert again.scanned == 0 + assert again.packed == 0 + + +def test_the_writer_does_not_republish_a_packed_message(tmp_path: Path) -> None: + store = TraceStore(tmp_path) + refs = store.address_items(MESSAGES) + message_pack.pack_messages(store.artifacts_dir) + + assert store.address_items(MESSAGES) == refs + assert not any(artifact_v2.message_path(store.artifacts_dir, ref["$msg"]).exists() for ref in refs) + + +def test_the_writer_republishes_a_packed_message_whose_pack_is_gone(tmp_path: Path) -> None: + """A missing blob is written again by the next turn that sends it; a + packed address whose pack was lost must heal the same way.""" + store = TraceStore(tmp_path) + refs = store.address_items(MESSAGES) + message_pack.pack_messages(store.artifacts_dir) + for pack in message_pack.packs_dir(store.artifacts_dir).glob("*.pack.zst"): + pack.unlink() + + assert store.address_items(MESSAGES) == refs + + assert all(artifact_v2.message_path(store.artifacts_dir, ref["$msg"]).is_file() for ref in refs) + shell = {"artifactFormat": artifact_v2.ARTIFACT_FORMAT, "messages": refs} + assert artifact_v2.resolve_payload(shell, store.artifacts_dir)["messages"] == MESSAGES + + +def test_the_writer_republishes_a_packed_message_whose_pack_is_truncated(tmp_path: Path) -> None: + """An interrupted copy can leave a pack short of its frame. The addresses + whose slice no longer decodes must heal like a missing pack, since the + message files were deleted when the pack was written.""" + store = TraceStore(tmp_path) + refs = store.address_items(MESSAGES) + message_pack.pack_messages(store.artifacts_dir) + pack = next(message_pack.packs_dir(store.artifacts_dir).glob("*.pack.zst")) + pack.write_bytes(pack.read_bytes()[:8]) + + assert store.address_items(MESSAGES) == refs + + assert all(artifact_v2.message_path(store.artifacts_dir, ref["$msg"]).is_file() for ref in refs) + shell = {"artifactFormat": artifact_v2.ARTIFACT_FORMAT, "messages": refs} + assert artifact_v2.resolve_payload(shell, store.artifacts_dir)["messages"] == MESSAGES + + +def test_a_corrupt_message_file_is_reported_and_not_removed(tmp_path: Path) -> None: + refs = _publish(tmp_path) + artifacts = tmp_path / "logs" / "audit-artifacts" + corrupt = artifact_v2.message_path(artifacts, refs[1]["$msg"]) + corrupt.write_bytes(b"\x28\xb5\x2f\xfd not a frame") + + result = message_pack.pack_messages(artifacts) + + assert result.packed == 1 + assert result.errors + assert corrupt.exists() + assert not artifact_v2.message_path(artifacts, refs[0]["$msg"]).exists() + assert message_pack.read_message(artifacts, refs[0]["$msg"]) is not None + + +def test_a_budget_that_splits_a_character_resolves_to_the_placeholder(tmp_path: Path) -> None: + messages = [ + {"role": "user", "content": "a" * 200}, + {"role": "user", "content": "\u00e9" * 100}, + ] + refs = TraceStore(tmp_path).address_items(messages) + artifacts = tmp_path / "logs" / "audit-artifacts" + first = len(artifact_v2.message_text(messages[0]).encode("utf-8")) + second = artifact_v2.message_text(messages[1]).encode("utf-8") + cut = next(i for i in range(1, len(second)) if 0x80 <= second[i] <= 0xBF) + message_pack.pack_messages(artifacts) + + shell = {"artifactFormat": artifact_v2.ARTIFACT_FORMAT, "messages": refs} + resolved = artifact_v2.resolve_payload(shell, artifacts, max_total_bytes=first + cut) + + assert resolved["messages"][0] == messages[0] + assert "missing" in resolved["messages"][1]["content"] + + +def test_a_streamed_slice_refuses_an_offset_past_the_decode_bound(tmp_path: Path, monkeypatch) -> None: + refs = _publish(tmp_path) + artifacts = tmp_path / "logs" / "audit-artifacts" + message_pack.pack_messages(artifacts) + index_file = message_pack.index_path(artifacts) + index = json.loads(index_file.read_text(encoding="utf-8")) + entry = index["messages"][refs[0]["$msg"]] + index["messages"][refs[0]["$msg"]] = [entry[0], message_pack.MAX_PLAINTEXT_BYTES, 1] + index_file.write_text(json.dumps(index), encoding="utf-8") + + monkeypatch.setattr(message_pack, "_cached_pack", lambda path: None) + calls: list[tuple] = [] + monkeypatch.setattr(message_pack, "_stream_slice", lambda *args: calls.append(args) or None) + + assert message_pack.read_message(artifacts, refs[0]["$msg"]) is None + assert calls == [] + + +def test_a_pack_over_the_cache_budget_is_streamed(tmp_path: Path, monkeypatch) -> None: + refs = _publish(tmp_path) + artifacts = tmp_path / "logs" / "audit-artifacts" + message_pack.pack_messages(artifacts) + pack = next(message_pack.packs_dir(artifacts).glob("*.pack.zst")) + monkeypatch.setattr(message_pack, "CACHE_BYTES", 1) + + assert message_pack._cached_pack(pack) is None + for ref, message in zip(refs, MESSAGES): + assert json.loads(message_pack.read_message(artifacts, ref["$msg"])) == message + + +def test_pack_keeps_a_file_whose_pack_is_gone(tmp_path: Path) -> None: + refs = _publish(tmp_path) + artifacts = tmp_path / "logs" / "audit-artifacts" + message_pack.pack_messages(artifacts) + kept = artifact_v2.message_path(artifacts, refs[0]["$msg"]) + for pack in message_pack.packs_dir(artifacts).glob("*.pack.zst"): + pack.unlink() + kept.parent.mkdir(parents=True, exist_ok=True) + kept.write_text(artifact_v2.message_text(MESSAGES[0]), encoding="utf-8") + newcomer = {"role": "user", "content": "a message written after the pack was lost"} + new_path = artifact_v2.message_path(artifacts, artifact_v2.message_sha1(newcomer)) + new_path.parent.mkdir(parents=True, exist_ok=True) + new_path.write_text(artifact_v2.message_text(newcomer), encoding="utf-8") + + message_pack.pack_messages(artifacts) + + assert kept.is_file() + shell = {"artifactFormat": artifact_v2.ARTIFACT_FORMAT, "messages": [refs[0]]} + assert artifact_v2.resolve_payload(shell, artifacts)["messages"][0] == MESSAGES[0] + + +def test_pack_keeps_a_file_whose_pack_is_truncated(tmp_path: Path) -> None: + refs = _publish(tmp_path) + artifacts = tmp_path / "logs" / "audit-artifacts" + message_pack.pack_messages(artifacts) + kept = artifact_v2.message_path(artifacts, refs[0]["$msg"]) + pack = next(message_pack.packs_dir(artifacts).glob("*.pack.zst")) + pack.write_bytes(pack.read_bytes()[:16]) + kept.parent.mkdir(parents=True, exist_ok=True) + kept.write_text(artifact_v2.message_text(MESSAGES[0]), encoding="utf-8") + newcomer = {"role": "user", "content": "a message written after the pack was cut"} + new_path = artifact_v2.message_path(artifacts, artifact_v2.message_sha1(newcomer)) + new_path.parent.mkdir(parents=True, exist_ok=True) + new_path.write_text(artifact_v2.message_text(newcomer), encoding="utf-8") + + message_pack.pack_messages(artifacts) + + assert kept.is_file() + shell = {"artifactFormat": artifact_v2.ARTIFACT_FORMAT, "messages": [refs[0]]} + assert artifact_v2.resolve_payload(shell, artifacts)["messages"][0] == MESSAGES[0] diff --git a/tests/test_tracing_store_compression.py b/tests/test_tracing_store_compression.py new file mode 100644 index 000000000..3eb6c367e --- /dev/null +++ b/tests/test_tracing_store_compression.py @@ -0,0 +1,369 @@ +"""Compressed artifact storage: one zstd copy per payload, paths unchanged. + +The store keeps every payload under the same path it always used; only the +bytes change. That is what lets an old span's recorded ``artifact_path`` keep +pointing at a readable file after a migration, and why every reader decides by +the frame, never by the file name. +""" + +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path + +import pytest +import zstandard + +from raven.tracing import artifact_v2 +from raven.tracing import codec as trace_codec +from raven.tracing.codec import is_compressed, plaintext_size +from raven.tracing.compact import compact +from raven.tracing.compress import compress, compress_logs +from raven.tracing.store import TraceStore +from raven.trajectory import details, selection +from raven.trajectory.conversation import _read_artifact +from raven.trajectory.store import iter_spans + +MAGIC = b"\x28\xb5\x2f\xfd" +PAYLOAD = {"messages": [{"role": "user", "content": "hi " * 4000}]} + + +def _persist(state: Path, payload: object = PAYLOAD) -> dict: + return TraceStore(state).persist_artifact("llm.input", {"traceId": "t", "sessionKey": "s"}, payload) + + +def _plain(data: bytes) -> bytes: + return zstandard.ZstdDecompressor().decompressobj().decompress(data) + + +def test_persist_artifact_writes_a_compressed_blob_under_the_plain_path(tmp_path: Path) -> None: + artifact = _persist(tmp_path) + path = Path(artifact["path"]) + assert path.suffix == ".json" and not path.name.endswith(".zst") + + data = path.read_bytes() + assert data.startswith(MAGIC) + plain = _plain(data) + assert json.loads(plain) == PAYLOAD + + sha1 = hashlib.sha1(plain).hexdigest() + assert artifact["sha1"] == sha1 + assert artifact["bytes"] == len(plain) + + blob = tmp_path / "logs" / "audit-artifacts" / "_blobs" / sha1[:2] / f"{sha1}.json" + assert blob.read_bytes().startswith(MAGIC) + assert blob.stat().st_ino == path.stat().st_ino + assert plaintext_size(blob) == len(plain) + + +def test_identical_payloads_share_one_blob(tmp_path: Path) -> None: + first = _persist(tmp_path) + second = _persist(tmp_path) + assert Path(first["path"]).stat().st_ino == Path(second["path"]).stat().st_ino + + +def test_the_reader_decompresses_and_still_bounds_what_it_reads(tmp_path: Path) -> None: + artifact = _persist(tmp_path) + path = str(artifact["path"]) + + text, reason = _read_artifact(tmp_path, path) + assert reason is None + assert json.loads(text) == PAYLOAD + + limited, note = _read_artifact(tmp_path, path, 64) + assert limited == text[:64] + assert note is not None and "truncated" in note + + +def test_plain_artifacts_from_before_compression_still_read(tmp_path: Path) -> None: + plain = tmp_path / "logs" / "llm.input" / "2026-01-01" / "old.json" + plain.parent.mkdir(parents=True) + plain.write_text('{"old": true}', encoding="utf-8") + + text, reason = _read_artifact(tmp_path, str(plain)) + + assert reason is None + assert text == '{"old": true}' + + +def test_messages_are_compressed_and_addressable(tmp_path: Path) -> None: + message = {"role": "user", "content": "x" * 500} + store = TraceStore(tmp_path) + + refs = store.address_items([message]) + + sha1 = artifact_v2.message_sha1(message) + assert refs == [{"$msg": sha1}] + path = artifact_v2.message_path(tmp_path / "logs" / "audit-artifacts", sha1) + assert path.read_bytes().startswith(MAGIC) + text, reason = _read_artifact(tmp_path, str(path)) + assert reason is None and json.loads(text) == message + + +def test_compact_reads_compressed_artifacts(tmp_path: Path) -> None: + artifact = _persist(tmp_path) + artifacts = tmp_path / "logs" / "audit-artifacts" + + result = compact(artifacts) + + assert result.errors == [] + assert result.scanned == 1 + assert result.folded == 0 + assert Path(artifact["path"]).read_bytes().startswith(MAGIC) + + +def test_compact_folds_a_second_copy_of_the_same_payload(tmp_path: Path) -> None: + artifact = _persist(tmp_path) + source = Path(artifact["path"]) + copy = source.parent / "copy.json" + copy.write_bytes(source.read_bytes()) + os.utime(copy, (0, 0)) + + result = compact(tmp_path / "logs" / "audit-artifacts") + + assert result.folded == 1 + assert copy.stat().st_ino == source.stat().st_ino + + +def _plant_plain_store(state: Path) -> tuple[Path, Path, str]: + """A pre-compression store: a plain blob, its artifact link, and a message.""" + text = '{"legacy": "payload"}' + sha1 = hashlib.sha1(text.encode("utf-8")).hexdigest() + artifacts = state / "logs" / "audit-artifacts" + blob = artifacts / "_blobs" / sha1[:2] / f"{sha1}.json" + blob.parent.mkdir(parents=True) + blob.write_text(text, encoding="utf-8") + artifact = artifacts / "llm.input" / "2026-01-01" / "old.json" + artifact.parent.mkdir(parents=True) + os.link(blob, artifact) + message = {"role": "user", "content": "m" * 300} + msha = artifact_v2.message_sha1(message) + mpath = artifact_v2.message_path(artifacts, msha) + mpath.parent.mkdir(parents=True) + mpath.write_text(artifact_v2.message_text(message), encoding="utf-8") + return artifact, mpath, sha1 + + +def test_compress_migrates_plain_blobs_and_keeps_every_path(tmp_path: Path) -> None: + artifact, message_path, sha1 = _plant_plain_store(tmp_path) + artifacts = tmp_path / "logs" / "audit-artifacts" + + result = compress(artifacts) + + assert result.compressed == 1 + assert result.errors == [] + assert artifact.read_bytes().startswith(MAGIC) + assert message_path.read_bytes().startswith(MAGIC) + blob = artifacts / "_blobs" / sha1[:2] / f"{sha1}.json" + assert artifact.stat().st_ino == blob.stat().st_ino + text, reason = _read_artifact(tmp_path, str(artifact)) + assert reason is None and text == '{"legacy": "payload"}' + text, reason = _read_artifact(tmp_path, str(message_path)) + assert reason is None and json.loads(text)["content"] == "m" * 300 + + +def test_compress_is_idempotent_and_dry_run_writes_nothing(tmp_path: Path) -> None: + artifact, message_path, _ = _plant_plain_store(tmp_path) + artifacts = tmp_path / "logs" / "audit-artifacts" + before = artifact.read_bytes() + + dry = compress(artifacts, dry_run=True) + assert dry.compressed == 1 + assert artifact.read_bytes() == before + + once = compress(artifacts) + assert once.compressed == 1 + again = compress(artifacts) + assert again.compressed == 0 + assert again.already >= 1 + assert artifact.read_bytes().startswith(MAGIC) + assert message_path.read_bytes().startswith(MAGIC) + + +def test_blob_prefix_reports_the_plaintext_size(tmp_path: Path) -> None: + message = {"role": "user", "content": "x" * 1000} + TraceStore(tmp_path).address_items([message]) + sha1 = artifact_v2.message_sha1(message) + plain = artifact_v2.message_text(message) + reader = details._Reader(tmp_path, max_artifacts=None, max_blobs=None, max_bytes=None) + + text, size = reader.blob_prefix(sha1, 10) + + assert text == plain[:10] + assert size == len(plain.encode("utf-8")) + + +def test_a_corrupt_frame_is_reported_not_raised(tmp_path: Path) -> None: + bad = tmp_path / "logs" / "llm.input" / "2026-01-01" / "bad.json" + bad.parent.mkdir(parents=True) + bad.write_bytes(MAGIC + b" not a frame") + + text, reason = _read_artifact(tmp_path, str(bad)) + + assert text is None + assert reason is not None and "corrupt" in reason + + +def test_decompression_is_bounded(tmp_path: Path) -> None: + frame = zstandard.ZstdCompressor().compress(b"x" * 4096) + assert trace_codec.read_plaintext(frame) == b"x" * 4096 + assert trace_codec.read_plaintext(frame, 1024) is None + + path = tmp_path / "bomb" + path.write_bytes(frame) + assert trace_codec.read_plaintext_file(path, 1024) is None + with pytest.raises(OSError): + trace_codec.open_plaintext(path, 1024) + + +def test_compact_records_a_corrupt_frame(tmp_path: Path) -> None: + bad = tmp_path / "logs" / "audit-artifacts" / "llm.input" / "2026-01-01" / "bad.json" + bad.parent.mkdir(parents=True) + bad.write_bytes(MAGIC + b" not a frame") + os.utime(bad, (0, 0)) + + result = compact(tmp_path / "logs" / "audit-artifacts") + + assert result.errors + assert bad.exists() + + +def _span_record(trace: str, span_id: str) -> dict: + return {"traceId": trace, "spanId": span_id, "name": "tool.call", "attributes": {}} + + +def _plant_logs(state: Path) -> Path: + """A store with two rotated span logs, one active log, and one frame journal. + + Returns the older rotated log: the newest is left for the next rotation. + """ + logs = state / "logs" + archive = logs / "archive" / "2026-07-01" / "audit-spans-2026-07-01-1.log" + archive.parent.mkdir(parents=True) + archive.write_text(json.dumps(_span_record("t1", "s1")) + "\n", encoding="utf-8") + os.utime(archive, ns=(1_000_000_000_000_000_000, 1_000_000_000_000_000_000)) + newest = archive.with_name("audit-spans-2026-07-01-2.log") + newest.write_text(json.dumps(_span_record("t1", "s1b")) + "\n", encoding="utf-8") + os.utime(newest, ns=(1_000_000_100_000_000_000, 1_000_000_100_000_000_000)) + (logs / "audit-spans.log").write_text(json.dumps(_span_record("t2", "s2")) + "\n", encoding="utf-8") + frames = logs / "acp-frames" / "2026-07-01" / "a.jsonl" + frames.parent.mkdir(parents=True) + frames.write_text('{"method": "x"}\n', encoding="utf-8") + return archive + + +def _rotated(state: Path, kind: str, name: str, mtime_ns: int) -> Path: + path = state / "logs" / "archive" / "2026-07-01" / f"audit-{kind}-2026-07-01-{name}.log" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(_span_record("t", f"{kind}-{name}")) + "\n", encoding="utf-8") + os.utime(path, ns=(mtime_ns, mtime_ns)) + return path + + +def test_compress_logs_leaves_the_newest_log_of_each_kind_and_keeps_mtimes(tmp_path: Path) -> None: + """A writer that opened the active log just before it rotated may still + append to the newest archive, so that one waits for the next rotation; the + viewer orders logs by mtime, so a compressed one keeps its own.""" + old_spans = _rotated(tmp_path, "spans", "1", 1_000_000_000_000_000_000) + new_spans = _rotated(tmp_path, "spans", "2", 1_000_000_100_000_000_000) + old_events = _rotated(tmp_path, "events", "1", 1_000_000_000_000_000_000) + new_events = _rotated(tmp_path, "events", "2", 1_000_000_100_000_000_000) + + result = compress_logs(tmp_path / "logs") + + for newest in (new_spans, new_events): + assert not is_compressed(newest.read_bytes()) + for older in (old_spans, old_events): + assert is_compressed(older.read_bytes()) + assert older.stat().st_mtime_ns == 1_000_000_000_000_000_000 + assert (result.files, result.compressed, result.newest) == (4, 2, 2) + + +def test_compress_logs_rewrites_only_rotated_files(tmp_path: Path) -> None: + archive = _plant_logs(tmp_path) + active = tmp_path / "logs" / "audit-spans.log" + frames = tmp_path / "logs" / "acp-frames" / "2026-07-01" / "a.jsonl" + active_before = active.read_bytes() + frames_before = frames.read_bytes() + + result = compress_logs(tmp_path / "logs") + + assert (result.files, result.compressed, result.already, result.newest) == (2, 1, 0, 1) + assert result.errors == [] + assert is_compressed(archive.read_bytes()) + # A span records a plaintext byte range into a frame journal, so that file + # must keep its bytes; the active log is still being appended to. + assert active.read_bytes() == active_before + assert frames.read_bytes() == frames_before + + +def test_iter_spans_reads_a_compressed_archive(tmp_path: Path) -> None: + archive = _plant_logs(tmp_path) + compress_logs(tmp_path / "logs") + assert is_compressed(archive.read_bytes()) + + ids = [span["spanId"] for span in iter_spans(tmp_path)] + + assert ids == ["s1", "s1b", "s2"] + + +def test_selection_reads_a_compressed_archive(tmp_path: Path) -> None: + archive = _plant_logs(tmp_path) + compress_logs(tmp_path / "logs") + assert is_compressed(archive.read_bytes()) + + records = list(selection._records(tmp_path, None)) + + assert [record["spanId"] for record in records] == ["s1", "s1b", "s2"] + + +def test_compress_logs_is_idempotent_and_dry_run_writes_nothing(tmp_path: Path) -> None: + archive = _plant_logs(tmp_path) + before = archive.read_bytes() + + dry = compress_logs(tmp_path / "logs", dry_run=True) + assert dry.compressed == 1 + assert archive.read_bytes() == before + + once = compress_logs(tmp_path / "logs") + assert once.compressed == 1 + again = compress_logs(tmp_path / "logs") + assert again.compressed == 0 + assert (again.already, again.newest) == (1, 1) + + +def test_rotation_compresses_the_previous_archive_and_keeps_its_mtime(tmp_path: Path) -> None: + """The viewer orders logs by mtime, so a re-encoded archive keeps the one + it had; the log just rotated stays plain until the next rotation.""" + store = TraceStore(tmp_path, max_bytes=1) + store.append_span(_span_record("t1", "s1")) + store.append_span(_span_record("t2", "s2")) + archive = tmp_path / "logs" / "archive" + first = next(archive.rglob("audit-spans-*.log")) + assert not is_compressed(first.read_bytes()) + mtime = first.stat().st_mtime_ns + + store.append_span(_span_record("t3", "s3")) + + second = next(path for path in archive.rglob("audit-spans-*.log") if path != first) + assert is_compressed(first.read_bytes()) + assert first.stat().st_mtime_ns == mtime + assert not is_compressed(second.read_bytes()) + assert [span["spanId"] for span in iter_spans(tmp_path)] == ["s1", "s2", "s3"] + + +@pytest.mark.skipif(os.name == "nt", reason="Windows refuses to rename a log another handle holds open") +def test_a_writer_holding_the_active_log_across_rotation_keeps_its_span(tmp_path: Path) -> None: + """Another writer can open the active log just before a rotation renames + it and write a moment later; that line lands in the rotated inode.""" + store = TraceStore(tmp_path, max_bytes=1) + store.append_span(_span_record("t1", "s1")) + held = (tmp_path / "logs" / "audit-spans.log").open("a", encoding="utf-8") + + store.append_span(_span_record("t2", "s2")) + held.write(json.dumps(_span_record("t3", "late")) + "\n") + held.close() + + assert {span["spanId"] for span in iter_spans(tmp_path)} == {"s1", "s2", "late"} diff --git a/tests/test_trajectory_conversation.py b/tests/test_trajectory_conversation.py index 9f42b76a5..0d5231169 100644 --- a/tests/test_trajectory_conversation.py +++ b/tests/test_trajectory_conversation.py @@ -685,6 +685,9 @@ def read(self, n=-1): reads.append(n) return self._handle.read(n) + def __getattr__(self, name): + return getattr(self._handle, name) + def __enter__(self): return self @@ -699,7 +702,10 @@ def spy_open(self, *args, **kwargs): text, degraded = tconv._read_artifact(state, str(big)) assert degraded == "content over 512 KiB — truncated" assert len(text) == tconv._ARTIFACT_LIMIT - assert reads == [tconv._ARTIFACT_LIMIT + 1] + # The reader sniffs the first four bytes for a compressed frame, then reads + # the bounded window; no read may exceed that window. + assert max(reads) == tconv._ARTIFACT_LIMIT + 1 + assert reads.count(tconv._ARTIFACT_LIMIT + 1) == 1 fifo = state / "logs" / "audit-artifacts" / "pipe" os.mkfifo(fifo) diff --git a/tests/test_trajectory_index.py b/tests/test_trajectory_index.py index 16319e7e8..e3d635334 100644 --- a/tests/test_trajectory_index.py +++ b/tests/test_trajectory_index.py @@ -10,6 +10,8 @@ import pytest +from raven.tracing.codec import is_compressed +from raven.tracing.store import TraceStore from raven.trajectory import conversation as tconv from raven.trajectory import entries as tent from raven.trajectory import index as tidx @@ -67,6 +69,11 @@ def _rotate(state: Path, name: str = "2026-09-01-100000000000") -> Path: return target +def _compress(state: Path, archived: Path) -> None: + """Re-encode one rotated log under its own name, as the store does a rotation later.""" + TraceStore(state)._compress_rotated(archived) + + def _artifact(state: Path, payload, name: str) -> str: directory = state / "logs" / "audit-artifacts" directory.mkdir(parents=True, exist_ok=True) @@ -185,6 +192,99 @@ def test_broken_record_position_survives_rename(state, clock): assert after == before +def test_compressed_archive_is_indexed_in_plaintext_offsets(state, clock): + archive = state / "logs" / "archive" / "2026-09-01" / "audit-spans-2026-09-01-100000000000.log" + _append(state, [_turn("t", "before", start=0, end=1)], path=archive) + plaintext_bytes = len(archive.read_bytes()) + _compress(state, archive) + assert is_compressed(archive.read_bytes()) + _append(state, [_turn("t", "after", start=2, end=3)]) + + index = _index(state, clock) + _refresh_until_ready(index, clock) + + assert {e.span_id for e in index.entries()} == {"before", "after"} + assert index.index_state().total_bytes == plaintext_bytes + _log(state).stat().st_size + + +def test_compressing_a_consumed_archive_keeps_the_cursor_moving(state, clock): + _append(state, [_turn("t", "before", start=0, end=1)]) + index = _index(state, clock) + _refresh_until_ready(index, clock) + + _compress(state, _rotate(state)) + _append(state, [_turn("t", "after", start=2, end=3)]) + _refresh_until_ready(index, clock) + + assert {e.span_id for e in index.entries()} == {"before", "after"} + + +def test_a_log_the_store_compresses_after_rotation_is_not_read_again(state, clock): + """The store re-encodes a rotated log under a new inode. A record without + ids is keyed by its log position, so reading that log again as a new file + would list it twice.""" + store = TraceStore(state, max_bytes=4096) + broken = {"name": "memory.enqueue", "startTime": _ts(0), "endTime": _ts(1), "attributes": {"session.key": SESSION}} + store.append_span(broken) + store.append_span(_turn("t", "a", start=1, end=2)) + index = _index(state, clock) + _refresh_until_ready(index, clock) + before = [i for i in _ids(index) if i.startswith("broken:")] + assert len(before) == 1 + + archive = state / "logs" / "archive" + turns = 0 + while len(list(archive.rglob("audit-spans-*.log"))) < 2: + store.append_span(_turn("t", f"b{turns}", start=3 + turns, end=3 + turns)) + turns += 1 + oldest = sorted(archive.rglob("audit-spans-*.log"))[0] + assert is_compressed(oldest.read_bytes()) + _refresh_until_ready(index, clock) + + assert [i for i in _ids(index) if i.startswith("broken:")] == before + assert len(_ids(index)) == 1 + 2 * (1 + turns) + + +def test_compressing_a_log_mid_scan_keeps_the_epoch(state, clock): + _append(state, [_turn("t", f"s{i}", start=i, end=i) for i in range(300)]) + index = _index(state, clock, budget_bytes=16 * 1024) + index.refresh_sync(None) + assert index.index_state().phase == tidx.PHASE_SCANNING + epoch = index.epoch + + _compress(state, _rotate(state)) + _append(state, [_turn("t", "after", start=400, end=401)]) + _refresh_until_ready(index, clock) + + assert index.epoch == epoch + assert len(_ids(index)) == 2 * 301 + + +def test_a_compressed_log_whose_inode_is_reused_keeps_the_epoch(state, clock): + """ext4 hands a freed inode to the next file created, so the active log + written right after a compression can carry the old log's key. The key is + then present but smaller than the cursor's offset; that is the old log + moved, not truncated, when its plaintext turns up under a new key.""" + _append(state, [_turn("t", f"s{i}", start=i, end=i) for i in range(300)]) + index = _index(state, clock, budget_bytes=16 * 1024) + index.refresh_sync(None) + assert index.index_state().phase == tidx.PHASE_SCANNING + epoch = index.epoch + + archived = _rotate(state) + held = state / "logs" / "held-inode" + os.link(archived, held) + _compress(state, archived) + os.replace(held, _log(state)) + with _log(state).open("r+b") as fh: + fh.truncate(0) + _append(state, [_turn("t", "after", start=400, end=401)]) + _refresh_until_ready(index, clock) + + assert index.epoch == epoch + assert len(_ids(index)) == 2 * 301 + + # ── budgets ─────────────────────────────────────────────────────────── @@ -452,6 +552,32 @@ def test_a_spilled_trace_whose_first_file_was_removed_keeps_what_later_files_hol assert index.index_state().recovering_traces == 0 and index.spilled == {} +def test_a_spilled_trace_whose_first_file_was_compressed_loses_nothing(state, clock): + """A spill records a position in a file; compressing that file moves it to + a new inode, and the rescan must follow it rather than count a gap.""" + _append(state, [_tool("S", f"early-{i}", "s-root", start=1, end=2, session="sub") for i in range(6)]) + index = _index(state, clock, pending_spans=5) + index.refresh_sync(None) + assert "S" in index.spilled + archived = _rotate(state) + _append(state, [_tool("S", f"late-{i}", "s-root", start=1, end=2, session="sub") for i in range(6)]) + index.refresh_sync(None) + _compress(state, archived) + attrs = {"trace.dispatched_in_trace_id": "t"} + _append( + state, + [ + _turn("t", "a", start=0, end=9), + _span("S", "s-root", "session.turn", start=0, end=3, session="sub", attrs=attrs), + ], + ) + _refresh_until_ready(index, clock) + recovered = {e.span_id for e in index.entries() if e.trace_id == "S"} + assert recovered == {f"early-{i}" for i in range(6)} | {f"late-{i}" for i in range(6)} | {"s-root"} + assert index.index_state().unresolved_dropped == 0 + assert index.index_state().recovering_traces == 0 and index.spilled == {} + + def test_spilled_foreign_trace_is_forgotten(state, clock): children = [_tool("F", f"f-{i}", "f-root", start=1, end=2, session="other") for i in range(12)] _append(state, children) diff --git a/tests/test_trajectory_selection.py b/tests/test_trajectory_selection.py index a0084de75..858eb7101 100644 --- a/tests/test_trajectory_selection.py +++ b/tests/test_trajectory_selection.py @@ -120,6 +120,27 @@ def _plan(*trace_ids: str, answering: bool = False) -> sel.SelectionPlan: # -- main-line turns --------------------------------------------------------- +def test_turn_input_reads_a_compressed_artifact(state): + from raven.tracing.codec import compress_bytes + + artifact = state / "logs" / "audit-artifacts" / "turn.input" / "2026-10-01" / "a.json" + artifact.parent.mkdir(parents=True) + artifact.write_bytes(compress_bytes(json.dumps({"content": "hello"}).encode("utf-8"))) + turn = sel.TurnSummary( + trace_id="t", + span_id="s", + number=1, + start=_ts(0), + end=None, + input_preview="", + in_progress=False, + input_artifact=str(artifact), + main_line=True, + ) + + assert sel._turn_input(turn, state) == "hello" + + def test_main_line_turns_leave_out_dispatched_and_foreign_turns(state): _write( state, diff --git a/tests/test_utils_mirrors.py b/tests/test_utils_mirrors.py index 14916d71c..22ef5a1d1 100644 --- a/tests/test_utils_mirrors.py +++ b/tests/test_utils_mirrors.py @@ -165,8 +165,8 @@ def test_a_proxys_refusal_of_the_tunnel_answers_for_the_host(self, error: BaseEx assert mirrors.unreachable(error) is reached def test_the_question_needs_no_httpx(self, monkeypatch: pytest.MonkeyPatch) -> None: - """A Raven node runs with loguru and pillow and nothing else, and starts - agents through npx all the same.""" + """A Raven node runs with loguru, pillow and zstandard and nothing else, + and starts agents through npx all the same.""" monkeypatch.setitem(sys.modules, "httpx", None) assert mirrors.unreachable(urllib.error.URLError(ConnectionRefusedError(111, "connection refused"))) @@ -1402,7 +1402,7 @@ def test_a_refused_connection_is_not(self) -> None: assert not mirrors._reachable(f"http://127.0.0.1:{_closed_port()}/") def test_the_probe_needs_no_httpx(self, monkeypatch: pytest.MonkeyPatch, answering_host: str) -> None: - """A Raven node has loguru and pillow, and starts npx agents too.""" + """A Raven node has loguru, pillow and zstandard, and starts npx agents too.""" monkeypatch.setitem(sys.modules, "httpx", None) assert mirrors._reachable(answering_host) diff --git a/uv.lock b/uv.lock index 762155570..eb9c681d1 100644 --- a/uv.lock +++ b/uv.lock @@ -4254,6 +4254,7 @@ dependencies = [ { name = "truststore" }, { name = "typer" }, { name = "watchfiles" }, + { name = "zstandard" }, ] [package.optional-dependencies] @@ -4459,6 +4460,7 @@ requires-dist = [ { name = "websockets", marker = "extra == 'channel-discord'", specifier = ">=16.0,<17.0" }, { name = "websockets", marker = "extra == 'channel-whatsapp'", specifier = ">=16.0,<17.0" }, { name = "wecom-aibot-sdk-python", marker = "extra == 'channel-wecom'", specifier = ">=0.1.5" }, + { name = "zstandard", specifier = ">=0.25.0" }, ] provides-extras = ["channel-telegram", "channel-slack", "channel-discord", "channel-whatsapp", "channel-matrix", "channel-feishu", "channel-wecom", "channel-mochat", "channel-qq", "channel-dingtalk", "channel-email", "channel-weixin", "channels", "tools", "sandbox", "browser", "dev", "eval", "segmenter"] @@ -5807,3 +5809,60 @@ sdist = { url = "https://files.pythonhosted.org/packages/30/21/093488dfc7cc8964d wheels = [ { url = "https://files.pythonhosted.org/packages/08/8a/0861bec20485572fbddf3dfba2910e38fe249796cb73ecdeb74e07eeb8d3/zipp-3.23.1-py3-none-any.whl", hash = "sha256:0b3596c50a5c700c9cb40ba8d86d9f2cc4807e9bedb06bcdf7fac85633e444dc", size = 10378, upload-time = "2026-04-13T23:21:45.386Z" }, ] + +[[package]] +name = "zstandard" +version = "0.25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/aa/3e0508d5a5dd96529cdc5a97011299056e14c6505b678fd58938792794b1/zstandard-0.25.0.tar.gz", hash = "sha256:7713e1179d162cf5c7906da876ec2ccb9c3a9dcbdffef0cc7f70c3667a205f0b", size = 711513, upload-time = "2025-09-14T22:15:54.002Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/fc/f26eb6ef91ae723a03e16eddb198abcfce2bc5a42e224d44cc8b6765e57e/zstandard-0.25.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7b3c3a3ab9daa3eed242d6ecceead93aebbb8f5f84318d82cee643e019c4b73b", size = 795738, upload-time = "2025-09-14T22:16:56.237Z" }, + { url = "https://files.pythonhosted.org/packages/aa/1c/d920d64b22f8dd028a8b90e2d756e431a5d86194caa78e3819c7bf53b4b3/zstandard-0.25.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:913cbd31a400febff93b564a23e17c3ed2d56c064006f54efec210d586171c00", size = 640436, upload-time = "2025-09-14T22:16:57.774Z" }, + { url = "https://files.pythonhosted.org/packages/53/6c/288c3f0bd9fcfe9ca41e2c2fbfd17b2097f6af57b62a81161941f09afa76/zstandard-0.25.0-cp312-cp312-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:011d388c76b11a0c165374ce660ce2c8efa8e5d87f34996aa80f9c0816698b64", size = 5343019, upload-time = "2025-09-14T22:16:59.302Z" }, + { url = "https://files.pythonhosted.org/packages/1e/15/efef5a2f204a64bdb5571e6161d49f7ef0fffdbca953a615efbec045f60f/zstandard-0.25.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dffecc361d079bb48d7caef5d673c88c8988d3d33fb74ab95b7ee6da42652ea", size = 5063012, upload-time = "2025-09-14T22:17:01.156Z" }, + { url = "https://files.pythonhosted.org/packages/b7/37/a6ce629ffdb43959e92e87ebdaeebb5ac81c944b6a75c9c47e300f85abdf/zstandard-0.25.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:7149623bba7fdf7e7f24312953bcf73cae103db8cae49f8154dd1eadc8a29ecb", size = 5394148, upload-time = "2025-09-14T22:17:03.091Z" }, + { url = "https://files.pythonhosted.org/packages/e3/79/2bf870b3abeb5c070fe2d670a5a8d1057a8270f125ef7676d29ea900f496/zstandard-0.25.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:6a573a35693e03cf1d67799fd01b50ff578515a8aeadd4595d2a7fa9f3ec002a", size = 5451652, upload-time = "2025-09-14T22:17:04.979Z" }, + { url = "https://files.pythonhosted.org/packages/53/60/7be26e610767316c028a2cbedb9a3beabdbe33e2182c373f71a1c0b88f36/zstandard-0.25.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5a56ba0db2d244117ed744dfa8f6f5b366e14148e00de44723413b2f3938a902", size = 5546993, upload-time = "2025-09-14T22:17:06.781Z" }, + { url = "https://files.pythonhosted.org/packages/85/c7/3483ad9ff0662623f3648479b0380d2de5510abf00990468c286c6b04017/zstandard-0.25.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:10ef2a79ab8e2974e2075fb984e5b9806c64134810fac21576f0668e7ea19f8f", size = 5046806, upload-time = "2025-09-14T22:17:08.415Z" }, + { url = "https://files.pythonhosted.org/packages/08/b3/206883dd25b8d1591a1caa44b54c2aad84badccf2f1de9e2d60a446f9a25/zstandard-0.25.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:aaf21ba8fb76d102b696781bddaa0954b782536446083ae3fdaa6f16b25a1c4b", size = 5576659, upload-time = "2025-09-14T22:17:10.164Z" }, + { url = "https://files.pythonhosted.org/packages/9d/31/76c0779101453e6c117b0ff22565865c54f48f8bd807df2b00c2c404b8e0/zstandard-0.25.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1869da9571d5e94a85a5e8d57e4e8807b175c9e4a6294e3b66fa4efb074d90f6", size = 4953933, upload-time = "2025-09-14T22:17:11.857Z" }, + { url = "https://files.pythonhosted.org/packages/18/e1/97680c664a1bf9a247a280a053d98e251424af51f1b196c6d52f117c9720/zstandard-0.25.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:809c5bcb2c67cd0ed81e9229d227d4ca28f82d0f778fc5fea624a9def3963f91", size = 5268008, upload-time = "2025-09-14T22:17:13.627Z" }, + { url = "https://files.pythonhosted.org/packages/1e/73/316e4010de585ac798e154e88fd81bb16afc5c5cb1a72eeb16dd37e8024a/zstandard-0.25.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:f27662e4f7dbf9f9c12391cb37b4c4c3cb90ffbd3b1fb9284dadbbb8935fa708", size = 5433517, upload-time = "2025-09-14T22:17:16.103Z" }, + { url = "https://files.pythonhosted.org/packages/5b/60/dd0f8cfa8129c5a0ce3ea6b7f70be5b33d2618013a161e1ff26c2b39787c/zstandard-0.25.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:99c0c846e6e61718715a3c9437ccc625de26593fea60189567f0118dc9db7512", size = 5814292, upload-time = "2025-09-14T22:17:17.827Z" }, + { url = "https://files.pythonhosted.org/packages/fc/5f/75aafd4b9d11b5407b641b8e41a57864097663699f23e9ad4dbb91dc6bfe/zstandard-0.25.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:474d2596a2dbc241a556e965fb76002c1ce655445e4e3bf38e5477d413165ffa", size = 5360237, upload-time = "2025-09-14T22:17:19.954Z" }, + { url = "https://files.pythonhosted.org/packages/ff/8d/0309daffea4fcac7981021dbf21cdb2e3427a9e76bafbcdbdf5392ff99a4/zstandard-0.25.0-cp312-cp312-win32.whl", hash = "sha256:23ebc8f17a03133b4426bcc04aabd68f8236eb78c3760f12783385171b0fd8bd", size = 436922, upload-time = "2025-09-14T22:17:24.398Z" }, + { url = "https://files.pythonhosted.org/packages/79/3b/fa54d9015f945330510cb5d0b0501e8253c127cca7ebe8ba46a965df18c5/zstandard-0.25.0-cp312-cp312-win_amd64.whl", hash = "sha256:ffef5a74088f1e09947aecf91011136665152e0b4b359c42be3373897fb39b01", size = 506276, upload-time = "2025-09-14T22:17:21.429Z" }, + { url = "https://files.pythonhosted.org/packages/ea/6b/8b51697e5319b1f9ac71087b0af9a40d8a6288ff8025c36486e0c12abcc4/zstandard-0.25.0-cp312-cp312-win_arm64.whl", hash = "sha256:181eb40e0b6a29b3cd2849f825e0fa34397f649170673d385f3598ae17cca2e9", size = 462679, upload-time = "2025-09-14T22:17:23.147Z" }, + { url = "https://files.pythonhosted.org/packages/35/0b/8df9c4ad06af91d39e94fa96cc010a24ac4ef1378d3efab9223cc8593d40/zstandard-0.25.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ec996f12524f88e151c339688c3897194821d7f03081ab35d31d1e12ec975e94", size = 795735, upload-time = "2025-09-14T22:17:26.042Z" }, + { url = "https://files.pythonhosted.org/packages/3f/06/9ae96a3e5dcfd119377ba33d4c42a7d89da1efabd5cb3e366b156c45ff4d/zstandard-0.25.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a1a4ae2dec3993a32247995bdfe367fc3266da832d82f8438c8570f989753de1", size = 640440, upload-time = "2025-09-14T22:17:27.366Z" }, + { url = "https://files.pythonhosted.org/packages/d9/14/933d27204c2bd404229c69f445862454dcc101cd69ef8c6068f15aaec12c/zstandard-0.25.0-cp313-cp313-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:e96594a5537722fdfb79951672a2a63aec5ebfb823e7560586f7484819f2a08f", size = 5343070, upload-time = "2025-09-14T22:17:28.896Z" }, + { url = "https://files.pythonhosted.org/packages/6d/db/ddb11011826ed7db9d0e485d13df79b58586bfdec56e5c84a928a9a78c1c/zstandard-0.25.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bfc4e20784722098822e3eee42b8e576b379ed72cca4a7cb856ae733e62192ea", size = 5063001, upload-time = "2025-09-14T22:17:31.044Z" }, + { url = "https://files.pythonhosted.org/packages/db/00/87466ea3f99599d02a5238498b87bf84a6348290c19571051839ca943777/zstandard-0.25.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:457ed498fc58cdc12fc48f7950e02740d4f7ae9493dd4ab2168a47c93c31298e", size = 5394120, upload-time = "2025-09-14T22:17:32.711Z" }, + { url = "https://files.pythonhosted.org/packages/2b/95/fc5531d9c618a679a20ff6c29e2b3ef1d1f4ad66c5e161ae6ff847d102a9/zstandard-0.25.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:fd7a5004eb1980d3cefe26b2685bcb0b17989901a70a1040d1ac86f1d898c551", size = 5451230, upload-time = "2025-09-14T22:17:34.41Z" }, + { url = "https://files.pythonhosted.org/packages/63/4b/e3678b4e776db00f9f7b2fe58e547e8928ef32727d7a1ff01dea010f3f13/zstandard-0.25.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8e735494da3db08694d26480f1493ad2cf86e99bdd53e8e9771b2752a5c0246a", size = 5547173, upload-time = "2025-09-14T22:17:36.084Z" }, + { url = "https://files.pythonhosted.org/packages/4e/d5/ba05ed95c6b8ec30bd468dfeab20589f2cf709b5c940483e31d991f2ca58/zstandard-0.25.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3a39c94ad7866160a4a46d772e43311a743c316942037671beb264e395bdd611", size = 5046736, upload-time = "2025-09-14T22:17:37.891Z" }, + { url = "https://files.pythonhosted.org/packages/50/d5/870aa06b3a76c73eced65c044b92286a3c4e00554005ff51962deef28e28/zstandard-0.25.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:172de1f06947577d3a3005416977cce6168f2261284c02080e7ad0185faeced3", size = 5576368, upload-time = "2025-09-14T22:17:40.206Z" }, + { url = "https://files.pythonhosted.org/packages/5d/35/398dc2ffc89d304d59bc12f0fdd931b4ce455bddf7038a0a67733a25f550/zstandard-0.25.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3c83b0188c852a47cd13ef3bf9209fb0a77fa5374958b8c53aaa699398c6bd7b", size = 4954022, upload-time = "2025-09-14T22:17:41.879Z" }, + { url = "https://files.pythonhosted.org/packages/9a/5c/36ba1e5507d56d2213202ec2b05e8541734af5f2ce378c5d1ceaf4d88dc4/zstandard-0.25.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:1673b7199bbe763365b81a4f3252b8e80f44c9e323fc42940dc8843bfeaf9851", size = 5267889, upload-time = "2025-09-14T22:17:43.577Z" }, + { url = "https://files.pythonhosted.org/packages/70/e8/2ec6b6fb7358b2ec0113ae202647ca7c0e9d15b61c005ae5225ad0995df5/zstandard-0.25.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:0be7622c37c183406f3dbf0cba104118eb16a4ea7359eeb5752f0794882fc250", size = 5433952, upload-time = "2025-09-14T22:17:45.271Z" }, + { url = "https://files.pythonhosted.org/packages/7b/01/b5f4d4dbc59ef193e870495c6f1275f5b2928e01ff5a81fecb22a06e22fb/zstandard-0.25.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:5f5e4c2a23ca271c218ac025bd7d635597048b366d6f31f420aaeb715239fc98", size = 5814054, upload-time = "2025-09-14T22:17:47.08Z" }, + { url = "https://files.pythonhosted.org/packages/b2/e5/fbd822d5c6f427cf158316d012c5a12f233473c2f9c5fe5ab1ae5d21f3d8/zstandard-0.25.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4f187a0bb61b35119d1926aee039524d1f93aaf38a9916b8c4b78ac8514a0aaf", size = 5360113, upload-time = "2025-09-14T22:17:48.893Z" }, + { url = "https://files.pythonhosted.org/packages/8e/e0/69a553d2047f9a2c7347caa225bb3a63b6d7704ad74610cb7823baa08ed7/zstandard-0.25.0-cp313-cp313-win32.whl", hash = "sha256:7030defa83eef3e51ff26f0b7bfb229f0204b66fe18e04359ce3474ac33cbc09", size = 436936, upload-time = "2025-09-14T22:17:52.658Z" }, + { url = "https://files.pythonhosted.org/packages/d9/82/b9c06c870f3bd8767c201f1edbdf9e8dc34be5b0fbc5682c4f80fe948475/zstandard-0.25.0-cp313-cp313-win_amd64.whl", hash = "sha256:1f830a0dac88719af0ae43b8b2d6aef487d437036468ef3c2ea59c51f9d55fd5", size = 506232, upload-time = "2025-09-14T22:17:50.402Z" }, + { url = "https://files.pythonhosted.org/packages/d4/57/60c3c01243bb81d381c9916e2a6d9e149ab8627c0c7d7abb2d73384b3c0c/zstandard-0.25.0-cp313-cp313-win_arm64.whl", hash = "sha256:85304a43f4d513f5464ceb938aa02c1e78c2943b29f44a750b48b25ac999a049", size = 462671, upload-time = "2025-09-14T22:17:51.533Z" }, + { url = "https://files.pythonhosted.org/packages/3d/5c/f8923b595b55fe49e30612987ad8bf053aef555c14f05bb659dd5dbe3e8a/zstandard-0.25.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:e29f0cf06974c899b2c188ef7f783607dbef36da4c242eb6c82dcd8b512855e3", size = 795887, upload-time = "2025-09-14T22:17:54.198Z" }, + { url = "https://files.pythonhosted.org/packages/8d/09/d0a2a14fc3439c5f874042dca72a79c70a532090b7ba0003be73fee37ae2/zstandard-0.25.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:05df5136bc5a011f33cd25bc9f506e7426c0c9b3f9954f056831ce68f3b6689f", size = 640658, upload-time = "2025-09-14T22:17:55.423Z" }, + { url = "https://files.pythonhosted.org/packages/5d/7c/8b6b71b1ddd517f68ffb55e10834388d4f793c49c6b83effaaa05785b0b4/zstandard-0.25.0-cp314-cp314-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:f604efd28f239cc21b3adb53eb061e2a205dc164be408e553b41ba2ffe0ca15c", size = 5379849, upload-time = "2025-09-14T22:17:57.372Z" }, + { url = "https://files.pythonhosted.org/packages/a4/86/a48e56320d0a17189ab7a42645387334fba2200e904ee47fc5a26c1fd8ca/zstandard-0.25.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:223415140608d0f0da010499eaa8ccdb9af210a543fac54bce15babbcfc78439", size = 5058095, upload-time = "2025-09-14T22:17:59.498Z" }, + { url = "https://files.pythonhosted.org/packages/f8/ad/eb659984ee2c0a779f9d06dbfe45e2dc39d99ff40a319895df2d3d9a48e5/zstandard-0.25.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2e54296a283f3ab5a26fc9b8b5d4978ea0532f37b231644f367aa588930aa043", size = 5551751, upload-time = "2025-09-14T22:18:01.618Z" }, + { url = "https://files.pythonhosted.org/packages/61/b3/b637faea43677eb7bd42ab204dfb7053bd5c4582bfe6b1baefa80ac0c47b/zstandard-0.25.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ca54090275939dc8ec5dea2d2afb400e0f83444b2fc24e07df7fdef677110859", size = 6364818, upload-time = "2025-09-14T22:18:03.769Z" }, + { url = "https://files.pythonhosted.org/packages/31/dc/cc50210e11e465c975462439a492516a73300ab8caa8f5e0902544fd748b/zstandard-0.25.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e09bb6252b6476d8d56100e8147b803befa9a12cea144bbe629dd508800d1ad0", size = 5560402, upload-time = "2025-09-14T22:18:05.954Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ae/56523ae9c142f0c08efd5e868a6da613ae76614eca1305259c3bf6a0ed43/zstandard-0.25.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a9ec8c642d1ec73287ae3e726792dd86c96f5681eb8df274a757bf62b750eae7", size = 4955108, upload-time = "2025-09-14T22:18:07.68Z" }, + { url = "https://files.pythonhosted.org/packages/98/cf/c899f2d6df0840d5e384cf4c4121458c72802e8bda19691f3b16619f51e9/zstandard-0.25.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:a4089a10e598eae6393756b036e0f419e8c1d60f44a831520f9af41c14216cf2", size = 5269248, upload-time = "2025-09-14T22:18:09.753Z" }, + { url = "https://files.pythonhosted.org/packages/1b/c0/59e912a531d91e1c192d3085fc0f6fb2852753c301a812d856d857ea03c6/zstandard-0.25.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f67e8f1a324a900e75b5e28ffb152bcac9fbed1cc7b43f99cd90f395c4375344", size = 5430330, upload-time = "2025-09-14T22:18:11.966Z" }, + { url = "https://files.pythonhosted.org/packages/a0/1d/7e31db1240de2df22a58e2ea9a93fc6e38cc29353e660c0272b6735d6669/zstandard-0.25.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:9654dbc012d8b06fc3d19cc825af3f7bf8ae242226df5f83936cb39f5fdc846c", size = 5811123, upload-time = "2025-09-14T22:18:13.907Z" }, + { url = "https://files.pythonhosted.org/packages/f6/49/fac46df5ad353d50535e118d6983069df68ca5908d4d65b8c466150a4ff1/zstandard-0.25.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4203ce3b31aec23012d3a4cf4a2ed64d12fea5269c49aed5e4c3611b938e4088", size = 5359591, upload-time = "2025-09-14T22:18:16.465Z" }, + { url = "https://files.pythonhosted.org/packages/c2/38/f249a2050ad1eea0bb364046153942e34abba95dd5520af199aed86fbb49/zstandard-0.25.0-cp314-cp314-win32.whl", hash = "sha256:da469dc041701583e34de852d8634703550348d5822e66a0c827d39b05365b12", size = 444513, upload-time = "2025-09-14T22:18:20.61Z" }, + { url = "https://files.pythonhosted.org/packages/3a/43/241f9615bcf8ba8903b3f0432da069e857fc4fd1783bd26183db53c4804b/zstandard-0.25.0-cp314-cp314-win_amd64.whl", hash = "sha256:c19bcdd826e95671065f8692b5a4aa95c52dc7a02a4c5a0cac46deb879a017a2", size = 516118, upload-time = "2025-09-14T22:18:17.849Z" }, + { url = "https://files.pythonhosted.org/packages/f0/ef/da163ce2450ed4febf6467d77ccb4cd52c4c30ab45624bad26ca0a27260c/zstandard-0.25.0-cp314-cp314-win_arm64.whl", hash = "sha256:d7541afd73985c630bafcd6338d2518ae96060075f9463d7dc14cfb33514383d", size = 476940, upload-time = "2025-09-14T22:18:19.088Z" }, +]