diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 12348a6..92d9f11 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,20 +4,33 @@ on: push: branches: [main] pull_request: + schedule: + # Weekly security audit (see the `audit` job; the other jobs skip + # scheduled runs). + - cron: "0 4 * * 1" # Cancel in-progress runs for the same ref when new commits are pushed. +# `event_name` is part of the group so the weekly scheduled audit never +# cancels (or is cancelled by) an in-flight push run on main. concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }} cancel-in-progress: true jobs: rust: - name: Rust (fmt, clippy, test) - runs-on: ubuntu-latest + name: Rust (fmt, clippy, test) / ${{ matrix.os }} + if: github.event_name != 'schedule' + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] steps: - uses: actions/checkout@v7 + # libpcap ships with macOS, so only the Linux runner needs an install. - name: Install libpcap (for the `pcap` crate) + if: runner.os == 'Linux' run: | sudo apt-get update sudo apt-get install -y libpcap-dev @@ -35,8 +48,45 @@ jobs: - name: cargo test run: cargo test --all + msrv: + # Matches `rust-version` in Cargo.toml. Bump both together. + name: MSRV (Rust 1.85) + if: github.event_name != 'schedule' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - name: Install libpcap (for the `pcap` crate) + run: | + sudo apt-get update + sudo apt-get install -y libpcap-dev + + - uses: dtolnay/rust-toolchain@master + with: + toolchain: "1.85" + + - name: cargo check + run: cargo check --all-targets --locked + + audit: + # Runs on pull requests and pushes, plus a weekly scheduled sweep so new + # RUSTSEC advisories against pinned deps are caught between commits. + name: Security audit (cargo audit) + runs-on: ubuntu-latest + permissions: + contents: read + issues: write + checks: write + steps: + - uses: actions/checkout@v7 + + - uses: rustsec/audit-check@v2 + with: + token: ${{ secrets.GITHUB_TOKEN }} + nix: name: Nix (build) + if: github.event_name != 'schedule' runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9a61d05..c44951a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,11 +5,39 @@ on: tags: - "v*" +permissions: + contents: read + jobs: - build-and-release: + # Gate the release on the test suite so a tag never ships untested code. + test: + name: Test (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest] + steps: + - uses: actions/checkout@v7 + + # libpcap ships with macOS, so only the Linux runners need an install. + - name: Install libpcap (for the `pcap` crate) + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y libpcap-dev + + - uses: dtolnay/rust-toolchain@stable + + - name: cargo test + run: cargo test --all --locked + + # linux-x86_64 is nix-built (matching the AUR -bin package's patchelf + # expectations); the manpage is generated fresh by the nix build. + build-nix: + name: Build (linux-x86_64, nix) + needs: test runs-on: ubuntu-latest - permissions: - contents: write steps: - uses: actions/checkout@v7 @@ -29,12 +57,112 @@ jobs: # postInstall; copy it out of the nix output as a release asset. run: cp result/share/man/man1/tapgres.1.gz tapgres.1.gz + - uses: actions/upload-artifact@v4 + with: + name: tapgres-linux-x86_64 + path: | + tapgres-linux-x86_64 + tapgres.1.gz + if-no-files-found: error + + # The remaining targets are plain cargo builds. macos-x86_64 is + # cross-compiled on the arm64 macOS runner (the Apple SDK ships both + # architectures, including libpcap). + build-cargo: + name: Build (${{ matrix.name }}) + needs: test + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - name: linux-aarch64 + os: ubuntu-24.04-arm + target: aarch64-unknown-linux-gnu + - name: macos-x86_64 + os: macos-latest + target: x86_64-apple-darwin + - name: macos-arm64 + os: macos-latest + target: aarch64-apple-darwin + steps: + - uses: actions/checkout@v7 + + - name: Install libpcap (for the `pcap` crate) + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y libpcap-dev + + - uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + + - name: cargo build + run: cargo build --release --locked --target ${{ matrix.target }} + + - name: Rename binary + run: cp "target/${{ matrix.target }}/release/tapgres" "tapgres-${{ matrix.name }}" + + - uses: actions/upload-artifact@v4 + with: + name: tapgres-${{ matrix.name }} + path: tapgres-${{ matrix.name }} + if-no-files-found: error + + release: + name: Create release + needs: [build-nix, build-cargo] + runs-on: ubuntu-latest + permissions: + contents: write + id-token: write + attestations: write + outputs: + version: ${{ steps.sha256.outputs.version }} + binary_x86_64: ${{ steps.sha256.outputs.binary_x86_64 }} + binary_aarch64: ${{ steps.sha256.outputs.binary_aarch64 }} + man: ${{ steps.sha256.outputs.man }} + source: ${{ steps.sha256.outputs.source }} + steps: + - uses: actions/download-artifact@v4 + with: + path: dist + merge-multiple: true + + - name: Generate SHA256SUMS + # Bare filenames so users can `sha256sum -c SHA256SUMS` next to the + # downloaded assets. + run: | + cd dist + sha256sum \ + tapgres-linux-x86_64 \ + tapgres-linux-aarch64 \ + tapgres-macos-x86_64 \ + tapgres-macos-arm64 \ + tapgres.1.gz > SHA256SUMS + cat SHA256SUMS + + - name: Attest build provenance + uses: actions/attest-build-provenance@v2 + with: + subject-path: | + dist/tapgres-linux-x86_64 + dist/tapgres-linux-aarch64 + dist/tapgres-macos-x86_64 + dist/tapgres-macos-arm64 + dist/tapgres.1.gz + - name: Create Release uses: softprops/action-gh-release@v3 with: files: | - tapgres-linux-x86_64 - tapgres.1.gz + dist/tapgres-linux-x86_64 + dist/tapgres-linux-aarch64 + dist/tapgres-macos-x86_64 + dist/tapgres-macos-arm64 + dist/tapgres.1.gz + dist/SHA256SUMS name: Release ${{ github.ref_name }} generate_release_notes: true draft: false @@ -42,23 +170,40 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} - name: Get SHA256 checksums + # Consumed by the AUR deploy jobs below to fill in the PKGBUILD + # templates under packaging/. id: sha256 run: | version=${GITHUB_REF_NAME#v} - binary_sha=$(sha256sum tapgres-linux-x86_64 | awk '{print $1}') - man_sha=$(sha256sum tapgres.1.gz | awk '{print $1}') + binary_x86_64_sha=$(sha256sum dist/tapgres-linux-x86_64 | awk '{print $1}') + binary_aarch64_sha=$(sha256sum dist/tapgres-linux-aarch64 | awk '{print $1}') + man_sha=$(sha256sum dist/tapgres.1.gz | awk '{print $1}') source_sha=$(curl -fsSL https://github.com/${{ github.repository }}/archive/refs/tags/${GITHUB_REF_NAME}.tar.gz | sha256sum | awk '{print $1}') { echo "version=$version" - echo "binary=$binary_sha" + echo "binary_x86_64=$binary_x86_64_sha" + echo "binary_aarch64=$binary_aarch64_sha" echo "man=$man_sha" echo "source=$source_sha" } >> "$GITHUB_OUTPUT" + # The AUR deploys are best-effort: a failed AUR push should never strand a + # half-done GitHub release, so both jobs are continue-on-error and can be + # redone by hand from the packaging/ templates if needed. + aur-bin: + name: Deploy tapgres-bin to AUR + needs: release + runs-on: ubuntu-latest + continue-on-error: true + steps: + - uses: actions/checkout@v7 + - name: Update tapgres-bin PKGBUILD run: | - sed -i "s/^pkgver=.*/pkgver=${{ steps.sha256.outputs.version }}/" packaging/tapgres-bin/PKGBUILD - sed -i "s/^sha256sums=.*/sha256sums=('${{ steps.sha256.outputs.binary }}' '${{ steps.sha256.outputs.man }}')/" packaging/tapgres-bin/PKGBUILD + sed -i "s/^pkgver=.*/pkgver=${{ needs.release.outputs.version }}/" packaging/tapgres-bin/PKGBUILD + sed -i "s/^sha256sums=.*/sha256sums=('${{ needs.release.outputs.man }}')/" packaging/tapgres-bin/PKGBUILD + sed -i "s/^sha256sums_x86_64=.*/sha256sums_x86_64=('${{ needs.release.outputs.binary_x86_64 }}')/" packaging/tapgres-bin/PKGBUILD + sed -i "s/^sha256sums_aarch64=.*/sha256sums_aarch64=('${{ needs.release.outputs.binary_aarch64 }}')/" packaging/tapgres-bin/PKGBUILD - name: Deploy tapgres-bin to AUR uses: KSXGitHub/github-actions-deploy-aur@v4.2.0 @@ -69,10 +214,18 @@ jobs: commit_email: n@sunng.info ssh_private_key: ${{ secrets.AUR_SSH_PRIVATE_KEY }} + aur-source: + name: Deploy tapgres to AUR + needs: release + runs-on: ubuntu-latest + continue-on-error: true + steps: + - uses: actions/checkout@v7 + - name: Update tapgres PKGBUILD run: | - sed -i "s/^pkgver=.*/pkgver=${{ steps.sha256.outputs.version }}/" packaging/tapgres/PKGBUILD - sed -i "s/^sha256sums=.*/sha256sums=('${{ steps.sha256.outputs.source }}')/" packaging/tapgres/PKGBUILD + sed -i "s/^pkgver=.*/pkgver=${{ needs.release.outputs.version }}/" packaging/tapgres/PKGBUILD + sed -i "s/^sha256sums=.*/sha256sums=('${{ needs.release.outputs.source }}')/" packaging/tapgres/PKGBUILD - name: Deploy tapgres to AUR uses: KSXGitHub/github-actions-deploy-aur@v4.2.0 diff --git a/.gitignore b/.gitignore index a8c9a15..3cb4b3d 100644 --- a/.gitignore +++ b/.gitignore @@ -4,5 +4,8 @@ result result-* +# macOS Finder metadata +.DS_Store + # Generated by `cargo run --example gen_manpage`. /man/tapgres.1 diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..ec73313 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,26 @@ +# Changelog + +All notable changes to this project are documented here. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Added + +- Save and replay of decoded sessions: `--save FILE` tees every decoded record + to versioned JSONL while capture continues, and `--replay FILE` reopens a + saved session without live capture. In the TUI, `:save` (`:w`) and `:open` + (`:o`) do the same from the command bar. The on-disk schema (version 1) is + documented in [`docs/session-format.md`](docs/session-format.md). + +## [0.3.0] + +## [0.2.0] + +## [0.1.0] + +Release notes for 0.3.0 and earlier are on the project's +[GitHub releases](https://github.com/sunng87/tapgres/releases) page; this file +tracks changes going forward. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..543fbfb --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,74 @@ +# Contributing to tapgres + +Thanks for taking the time to help. tapgres is a small Rust CLI; the loop below +is all you need. + +## Build and test + +The supported toolchain is a Nix dev shell that pins the Rust toolchain, +`libpcap`, `pandoc`, and a local PostgreSQL 18 to point the tap at: + +```sh +nix develop # Rust toolchain + libpcap + PostgreSQL 18 + pandoc +cargo build +cargo test # runs the unit and integration tests +``` + +Without Nix you need a Rust toolchain (edition 2024, so Rust >= 1.85 — the +`rust-version` in `Cargo.toml`) and libpcap's development headers +(`libpcap-dev` on Debian/Ubuntu, `libpcap` on Arch/Fedora): + +```sh +cargo build +cargo test +``` + +Match CI before opening a PR: + +```sh +cargo fmt --all +cargo clippy --all-targets -- -D warnings +cargo test --all +``` + +## Regenerate the manpage + +The manpage's options come from the clap CLI definition (`src/cli.rs`, the +single source of truth) and the prose in `man/sections.md`. It is generated, +never hand-edited, and needs `pandoc` (provided by the Nix shell). Regenerate it +after any change to the CLI or `man/sections.md`: + +```sh +cargo run --example gen_manpage > man/tapgres.1 +``` + +## Code layout + +The reusable pieces live in the library crate (`src/lib.rs`); `src/main.rs` +wires them together with libpcap, and `tests/` exercises them directly. + +| Module | Responsibility | +| --- | --- | +| `cli.rs` | clap CLI definition — the single source of truth for options and the manpage. | +| `net.rs` | Link-layer / TCP segment parsing (`TcpSegment`). | +| `flow.rs` | Per-connection tracking and TCP reassembly (`ConnTable`, `Direction`, `Role`). | +| `decode.rs` | pgwire message decoding and human-readable rendering (`Output`, `EventDetail`). | +| `filter.rs` | The `-Y` display-filter expression language (`DisplayFilter`, `DisplayMessage`). | +| `capture.rs` | The libpcap capture loop (`--mode pcap`). | +| `proxy.rs` | The TLS-terminating MITM proxy (`--mode mitm`). | +| `session.rs` | Versioned JSONL save/replay (`--save` / `--replay`); format in [`docs/session-format.md`](docs/session-format.md). | +| `state.rs` | Live connection/throughput metrics. | +| `tui.rs` | The ratatui full-screen view (`--tui`). | + +## Pull requests + +- Keep `cargo fmt`, `cargo clippy -D warnings`, and `cargo test --all` green. +- Preserve the stated MSRV (`rust-version` in `Cargo.toml`); CI has an MSRV job. +- Regenerate the manpage if you touched `src/cli.rs` or `man/sections.md`. +- Add a line under `## [Unreleased]` in [`CHANGELOG.md`](CHANGELOG.md) for any + user-visible change. +- If you change the saved-session on-disk shape, bump `SCHEMA_VERSION` and + update `docs/session-format.md` and the `tests/fixtures/session-v1.jsonl` + expectations — the format is versioned deliberately. +- Report security-sensitive issues privately (see [`SECURITY.md`](SECURITY.md)), + not in a public PR or issue. diff --git a/Cargo.lock b/Cargo.lock index 5e845c5..f4757f3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -623,6 +623,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd2e7510819d6fbf51a5545c8f922716ecfb14df168a3242f7d33e0239efe6a1" +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + [[package]] name = "filedescriptor" version = "0.8.3" @@ -1676,6 +1682,19 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + [[package]] name = "sha2" version = "0.10.9" @@ -1850,10 +1869,26 @@ dependencies = [ "regex", "rustls", "rustls-pemfile", + "serde", + "serde_json", + "tempfile", "tokio", "tokio-rustls", ] +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "termina" version = "0.3.3" @@ -2494,3 +2529,9 @@ name = "zeroize" version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml index 0e7a569..66df9ec 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,8 @@ bytes = "1" clap = { version = "4", features = ["derive"] } chrono = { version = "0.4", default-features = false, features = ["clock"] } regex = "1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" # --- TUI (`--tui`) --- # ratatui re-exports crossterm (its default backend), so we use that for events. @@ -50,3 +52,4 @@ rcgen = "0.14" # is dev-only: it never reaches a normal `cargo build`/release/`cargo install`. [dev-dependencies] clap_mangen = "0.2" +tempfile = "3" diff --git a/README.md b/README.md index d2a6616..00b9fc0 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,8 @@ tapgres -p 5432 -i eth0 # capture a specific interface tapgres --mode mitm \ # decode an encrypted session via the proxy --listen 127.0.0.1:15432 --upstream 127.0.0.1:5432 tapgres --tui -Y 'message.type == "Query"' # interactive view, filtered +tapgres --save session.jsonl # capture and tee every record to disk +tapgres --replay session.jsonl --tui # reopen it without live capture ``` For making a client trust the mitm proxy's auto-generated CA, see @@ -60,23 +62,65 @@ sudo setcap cap_net_raw+ep $(which tapgres) | `w` / `r` | wrap / rich display | | `c` | clear | | `y` | edit the display filter | -| `Esc` | clear the display filter | +| `/`, `n` / `N` | search message text, next / previous match | +| `:` | command bar (`:save FILE`, `:open FILE`) | +| `Esc` | clear the search, then the display filter | Display filters (`-Y` / `--display-filter`) use a small typed expression language with fields like `message.type`, `message.text`, `client.ip`, and -`client.port`. See `man tapgres` for the full field and operator reference. +`client.port` (with `==`, `!=`, ordered `<`/`>` on the port, `in`, `contains`, +and `matches`). See `man tapgres` for the full field and operator reference. + +## Save and replay + +`--save FILE` continuously writes every output record to versioned JSONL while +stdout or the TUI continues normally. Recording happens before display +filtering and before the TUI's 50,000-record history cap, so hidden or evicted +live records are still saved. An existing destination is replaced. + +`--replay FILE` uses a saved session instead of pcap/mitm capture. Replay is +instant, preserves the original timestamps and structured rich-view data, and +passes through the same display filters and renderers as live traffic: + +```sh +tapgres --replay session.jsonl +tapgres --replay session.jsonl --tui --tui-rich +tapgres --replay session.jsonl -Y 'message.type == "Query"' +``` + +In the TUI, `/` or `:` opens the command bar. `:save FILE` (also `:w`) writes +the currently retained events and then continuously records future traffic. +If older events have already left the TUI history, the footer reports the +omission. `:open FILE` (also `:o`) validates the complete file, replaces the +current view with its newest 50,000 records, and switches the session to replay +mode. It closes any active recorder, and subsequent live-source records are +discarded so live and replayed timelines never mix. + +Schema version 1 and its compatibility rules are defined in +[`docs/session-format.md`](docs/session-format.md). Unsupported schema versions +and malformed records are refused with a file and line-numbered error. + +> **Sensitive data.** Captures and saved sessions are cleartext: they contain +> query text, returned row values, connection parameters, and error messages +> exactly as they crossed the wire. Treat `--save` / `:save` files as sensitive +> and protect them accordingly. See [SECURITY.md](SECURITY.md). ## Installation -**Prebuilt binary** (Linux x86_64, from -[releases](https://github.com/sunng87/tapgres/releases)): +**Prebuilt binaries** (from +[releases](https://github.com/sunng87/tapgres/releases): +`tapgres-linux-x86_64`, `tapgres-linux-aarch64`, `tapgres-macos-x86_64`, +`tapgres-macos-arm64`; a `SHA256SUMS` file in each release covers all of +them): ```sh curl -L -o tapgres https://github.com/sunng87/tapgres/releases/latest/download/tapgres-linux-x86_64 chmod +x tapgres && sudo mv tapgres /usr/local/bin/ ``` -Built with Nix; on a non-Nix Linux it needs `libpcap.so.1` on the library path. +The linux-x86_64 binary is built with Nix; on a non-Nix Linux it needs +`libpcap.so.1` on the library path (see +[Troubleshooting](#troubleshooting)). **Arch Linux (AUR):** @@ -99,6 +143,61 @@ cargo install --path . A manual page is included in the Nix and Arch packages (`man tapgres`). +## Troubleshooting + +**"Permission denied" opening the capture (Linux).** pcap mode needs +`CAP_NET_RAW`. Grant it once instead of running as root: + +```sh +sudo setcap cap_net_raw+ep $(which tapgres) +``` + +**"Permission denied" opening the capture (macOS).** macOS captures packets +through the BPF devices (`/dev/bpf0`, `/dev/bpf1`, …), which are root-only by +default — and `setcap` is a Linux mechanism that does not exist on macOS. +Either run `sudo tapgres`, or install Wireshark's **ChmodBPF** helper +(bundled with the Wireshark installer, or standalone via +`brew install --cask wireshark-chmodbpf`). ChmodBPF installs a launch daemon +that makes `/dev/bpf*` readable by the `access_bpf` group at every boot; make +sure your user is in that group. Note that this grants every group member +capture access to *all* interfaces, not just loopback. + +**macOS refuses to run a downloaded binary** ("cannot be opened because the +developer cannot be verified", or the process is killed immediately). The +release binaries are not code-signed; if your download path added the +quarantine attribute, remove it: + +```sh +xattr -d com.apple.quarantine ./tapgres +``` + +**No traffic appears.** + +- `psql` and many other clients connect over a **Unix domain socket** when no + host is given — invisible to pcap. Force TCP with `-h 127.0.0.1`. +- tapgres captures the **loopback** interface by default (`lo` on Linux, + `lo0` on macOS). For a server on another machine, pick the right interface + with `-i eth0`. libpcap's `any` pseudo-device (`-i any`) captures every + interface at once but is **Linux-only**. +- If the client negotiated **TLS** (`sslmode=require`, …), pcap mode can + observe the SSL negotiation but not the encrypted stream that follows. Use + `--mode mitm` to decode encrypted sessions. + +**Clients reject the mitm proxy's certificate.** In `--mode mitm`, tapgres +terminates TLS with an auto-generated CA written to `--tls-dir` (default +`~/.config/tapgres`). Point each client at that CA: copy `ca.crt` to the client +and connect with `sslrootcert=…/ca.crt` and `sslmode=verify-ca`, for example + +```sh +psql "host=127.0.0.1 port=15432 dbname=postgres sslrootcert=ca.crt sslmode=verify-ca" +``` + +Distribute only `ca.crt`; `ca.key` is the CA's private key and must stay local. + +**`libpcap.so.1: cannot open shared object file`** when running a prebuilt +Linux binary: install your distribution's libpcap runtime package +(`libpcap0.8` on Debian/Ubuntu, `libpcap` on Arch and Fedora). + ## Develop ```sh diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..c719f3d --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,43 @@ +# Security Policy + +## Scope and threat model + +tapgres is a **local debugging tool**. It inspects PostgreSQL wire traffic on a +machine you control, either by passively capturing a port with libpcap +(`--mode pcap`) or by running a TLS-terminating man-in-the-middle proxy +(`--mode mitm`). It is not an authentication boundary, a production monitoring +agent, or a tool for intercepting traffic you do not own. Please only point it +at your own databases and connections. + +Because of what it does, running tapgres has inherent, expected consequences — +these are not vulnerabilities in the tool: + +- **Captures and saved sessions are cleartext.** Decoded output and any + `--save` / `:save` JSONL file contain query text, returned row values, + connection parameters, error messages, and other potentially sensitive + application data in the clear. Treat capture output and saved `.jsonl` + sessions as sensitive and protect them accordingly. +- **The MITM proxy writes a CA private key to disk.** In `--mode mitm` with + auto-generated certificates, tapgres writes `ca.crt`, `ca.key`, `server.crt`, + and `server.key` to `--tls-dir` (default `$XDG_CONFIG_HOME/tapgres`, i.e. + `~/.config/tapgres`). Any client you configure to trust `ca.crt` will accept + certificates minted by that CA, so `ca.key` is a sensitive secret: keep it + local, never distribute it, and distribute only `ca.crt` to the specific + clients that must trust the proxy. Remove the directory when you are done. +- **pcap mode needs elevated capture privileges** (`CAP_NET_RAW` on Linux, BPF + device access on macOS). Grant them narrowly rather than running as root + where possible. + +## Reporting a vulnerability + +If you find a security issue in tapgres itself — for example, a way it exposes +data or credentials beyond the expected behavior above, or a memory-safety or +input-handling bug reachable from captured/replayed data — please report it +**privately**. + +Email the maintainer directly: **sunng@pm.me** + +Please do not open a public GitHub issue or pull request for security reports. +Include the tapgres version, your OS, a description of the impact, and steps to +reproduce if you have them. You will get an acknowledgement, and a fix or +mitigation will be coordinated before any public disclosure. diff --git a/docs/session-format.md b/docs/session-format.md new file mode 100644 index 0000000..8767a74 --- /dev/null +++ b/docs/session-format.md @@ -0,0 +1,82 @@ +# Tapgres saved-session format + +Tapgres saves sessions as UTF-8 JSON Lines (JSONL): one complete JSON object per +line. Version 1 is designed to round-trip every `decode::Output` variant through +the same display-filter and rendering pipeline used by live capture. + +The format is local and stream-oriented. `--save` and `:save` replace an +existing destination, then record events before display filtering or TUI +history eviction. Files may contain sensitive SQL, credentials, errors, and +returned row values and should be protected accordingly. + +## Common fields + +Every line contains: + +| Field | Type | Meaning | +| --- | --- | --- | +| `schema_version` | unsigned integer | Version of this record shape; currently `1`. | +| `timestamp` | RFC 3339 string | Original message capture time, or record time for operational lines/status. | +| `record_type` | string | `message`, `line`, or `status`. | + +Blank lines are ignored. A malformed record aborts replay with the file path and +line number. + +## Message records + +A decoded PostgreSQL message contains the structured values required by display +filters and rich rendering: + +```json +{"schema_version":1,"timestamp":"2026-07-17T12:34:56.789+01:00","record_type":"message","direction":"f2b","message_type":"Query","text":"SELECT * FROM orders","rendered":"[12:34:56.789] [F→B] Query: SELECT * FROM orders","client":"127.0.0.1:40005"} +``` + +| Field | Type | Meaning | +| --- | --- | --- | +| `direction` | string | `f2b` for frontend/client to backend/server, or `b2f` for the reverse. | +| `message_type` | string | Decoded pgwire message name, such as `Query` or `DataRow`. | +| `text` | string | Decoded message body used by `message.text` filters. | +| `rendered` | string | Stable flat terminal representation, including the original display timestamp. | +| `client` | socket-address string | Owning connection's client IP and port. | +| `detail` | object, optional | Structured rich-rendering payload described below. | + +### Rich detail + +`RowDescription` preserves field names, PostgreSQL type OIDs, and format codes: + +```json +{"detail_type":"row_description","columns":[{"name":"id","type_oid":23,"format_code":0}]} +``` + +`DataRow` preserves its already-decoded display value alongside the cached +column name and type OID: + +```json +{"detail_type":"data_row","columns":[{"name":"id","type_oid":23,"value":"'1'"}]} +``` + +The detail object is nested under the message record's `detail` field. Keeping +both forms allows replay to reproduce rich tables without re-decoding pgwire +bytes while retaining the stable flat transcript. + +## Operational records + +Connection/capture lines and status records retain their original text: + +```json +{"schema_version":1,"timestamp":"2026-07-17T12:34:56.789+01:00","record_type":"line","text":"=== new connection 127.0.0.1:40005 -> 127.0.0.1:5432 ==="} +{"schema_version":1,"timestamp":"2026-07-17T12:34:56.790+01:00","record_type":"status","text":"tapgres: capturing on 'lo'"} +``` + +They remain outside display filtering, matching live behavior. + +## Compatibility policy + +- Tapgres writes only the current schema version. +- Version 1 readers require `schema_version: 1` on every non-blank line. +- Unknown older or newer versions are refused; there is no silent best-effort + conversion. +- A future incompatible shape must increment `schema_version` and provide an + explicit migration path if backward compatibility is desired. +- Replay preserves recorded `rendered` output rather than reformatting it, while + filters and rich mode use the structured fields. diff --git a/examples/demo-session.jsonl b/examples/demo-session.jsonl new file mode 100644 index 0000000..fb0f07b --- /dev/null +++ b/examples/demo-session.jsonl @@ -0,0 +1,25 @@ +{"schema_version":1,"timestamp":"2026-07-19T14:22:30.980-07:00","record_type":"status","text":"tapgres: capturing on 'lo' (filter: tcp port 5432)"} +{"schema_version":1,"timestamp":"2026-07-19T14:22:31.000-07:00","record_type":"line","text":"[14:22:31.000] === new connection 127.0.0.1:52413 -> 127.0.0.1:5432 (port 5432) ==="} +{"schema_version":1,"timestamp":"2026-07-19T14:22:31.010-07:00","record_type":"message","direction":"f2b","message_type":"SSLRequest","text":"(awaiting server reply)","rendered":"[14:22:31.010] [F→B] SSLRequest: (awaiting server reply)","client":"127.0.0.1:52413"} +{"schema_version":1,"timestamp":"2026-07-19T14:22:31.020-07:00","record_type":"message","direction":"b2f","message_type":"SslResponse","text":"refuse (continuing in cleartext)","rendered":"[14:22:31.020] [B→F] SslResponse: refuse (continuing in cleartext)","client":"127.0.0.1:52413"} +{"schema_version":1,"timestamp":"2026-07-19T14:22:31.030-07:00","record_type":"message","direction":"f2b","message_type":"Startup","text":"protocol 3.0 user=app, database=shop, client_encoding=UTF8","rendered":"[14:22:31.030] [F→B] Startup: protocol 3.0 user=app, database=shop, client_encoding=UTF8","client":"127.0.0.1:52413"} +{"schema_version":1,"timestamp":"2026-07-19T14:22:31.045-07:00","record_type":"message","direction":"b2f","message_type":"Authentication","text":"Ok","rendered":"[14:22:31.045] [B→F] Authentication: Ok","client":"127.0.0.1:52413"} +{"schema_version":1,"timestamp":"2026-07-19T14:22:31.046-07:00","record_type":"message","direction":"b2f","message_type":"BackendKeyData","text":"pid=8421 key=1998210394","rendered":"[14:22:31.046] [B→F] BackendKeyData: pid=8421 key=1998210394","client":"127.0.0.1:52413"} +{"schema_version":1,"timestamp":"2026-07-19T14:22:31.050-07:00","record_type":"message","direction":"b2f","message_type":"ReadyForQuery","text":"txn=idle","rendered":"[14:22:31.050] [B→F] ReadyForQuery: txn=idle","client":"127.0.0.1:52413"} +{"schema_version":1,"timestamp":"2026-07-19T14:22:33.100-07:00","record_type":"message","direction":"f2b","message_type":"Query","text":"SELECT id, email FROM users ORDER BY id","rendered":"[14:22:33.100] [F→B] Query: SELECT id, email FROM users ORDER BY id","client":"127.0.0.1:52413"} +{"schema_version":1,"timestamp":"2026-07-19T14:22:33.120-07:00","record_type":"message","direction":"b2f","message_type":"RowDescription","text":"id(oid=23, text), email(oid=25, text)","rendered":"[14:22:33.120] [B→F] RowDescription: id(oid=23, text), email(oid=25, text)","client":"127.0.0.1:52413","detail":{"detail_type":"row_description","columns":[{"name":"id","type_oid":23,"format_code":0},{"name":"email","type_oid":25,"format_code":0}]}} +{"schema_version":1,"timestamp":"2026-07-19T14:22:33.121-07:00","record_type":"message","direction":"b2f","message_type":"DataRow","text":"{ id='1', email='alice@example.com' }","rendered":"[14:22:33.121] [B→F] DataRow: { id='1', email='alice@example.com' }","client":"127.0.0.1:52413","detail":{"detail_type":"data_row","columns":[{"name":"id","type_oid":23,"value":"'1'"},{"name":"email","type_oid":25,"value":"'alice@example.com'"}]}} +{"schema_version":1,"timestamp":"2026-07-19T14:22:33.122-07:00","record_type":"message","direction":"b2f","message_type":"DataRow","text":"{ id='2', email='bob@example.com' }","rendered":"[14:22:33.122] [B→F] DataRow: { id='2', email='bob@example.com' }","client":"127.0.0.1:52413","detail":{"detail_type":"data_row","columns":[{"name":"id","type_oid":23,"value":"'2'"},{"name":"email","type_oid":25,"value":"'bob@example.com'"}]}} +{"schema_version":1,"timestamp":"2026-07-19T14:22:33.125-07:00","record_type":"message","direction":"b2f","message_type":"CommandComplete","text":"SELECT 2","rendered":"[14:22:33.125] [B→F] CommandComplete: SELECT 2","client":"127.0.0.1:52413"} +{"schema_version":1,"timestamp":"2026-07-19T14:22:33.126-07:00","record_type":"message","direction":"b2f","message_type":"ReadyForQuery","text":"txn=idle","rendered":"[14:22:33.126] [B→F] ReadyForQuery: txn=idle","client":"127.0.0.1:52413"} +{"schema_version":1,"timestamp":"2026-07-19T14:22:35.200-07:00","record_type":"message","direction":"f2b","message_type":"Parse","text":" [param types: 23] SELECT id, email FROM users WHERE id = $1","rendered":"[14:22:35.200] [F→B] Parse: [param types: 23] SELECT id, email FROM users WHERE id = $1","client":"127.0.0.1:52413"} +{"schema_version":1,"timestamp":"2026-07-19T14:22:35.201-07:00","record_type":"message","direction":"f2b","message_type":"Bind","text":" <- params: ['2'] result: text","rendered":"[14:22:35.201] [F→B] Bind: <- params: ['2'] result: text","client":"127.0.0.1:52413"} +{"schema_version":1,"timestamp":"2026-07-19T14:22:35.202-07:00","record_type":"message","direction":"f2b","message_type":"Execute","text":"","rendered":"[14:22:35.202] [F→B] Execute: ","client":"127.0.0.1:52413"} +{"schema_version":1,"timestamp":"2026-07-19T14:22:35.203-07:00","record_type":"message","direction":"f2b","message_type":"Sync","text":"","rendered":"[14:22:35.203] [F→B] Sync","client":"127.0.0.1:52413"} +{"schema_version":1,"timestamp":"2026-07-19T14:22:35.215-07:00","record_type":"message","direction":"b2f","message_type":"ParseComplete","text":"","rendered":"[14:22:35.215] [B→F] ParseComplete","client":"127.0.0.1:52413"} +{"schema_version":1,"timestamp":"2026-07-19T14:22:35.216-07:00","record_type":"message","direction":"b2f","message_type":"BindComplete","text":"","rendered":"[14:22:35.216] [B→F] BindComplete","client":"127.0.0.1:52413"} +{"schema_version":1,"timestamp":"2026-07-19T14:22:35.220-07:00","record_type":"message","direction":"b2f","message_type":"DataRow","text":"{ id='2', email='bob@example.com' }","rendered":"[14:22:35.220] [B→F] DataRow: { id='2', email='bob@example.com' }","client":"127.0.0.1:52413","detail":{"detail_type":"data_row","columns":[{"name":"id","type_oid":23,"value":"'2'"},{"name":"email","type_oid":25,"value":"'bob@example.com'"}]}} +{"schema_version":1,"timestamp":"2026-07-19T14:22:35.222-07:00","record_type":"message","direction":"b2f","message_type":"CommandComplete","text":"SELECT 1","rendered":"[14:22:35.222] [B→F] CommandComplete: SELECT 1","client":"127.0.0.1:52413"} +{"schema_version":1,"timestamp":"2026-07-19T14:22:35.223-07:00","record_type":"message","direction":"b2f","message_type":"ReadyForQuery","text":"txn=idle","rendered":"[14:22:35.223] [B→F] ReadyForQuery: txn=idle","client":"127.0.0.1:52413"} +{"schema_version":1,"timestamp":"2026-07-19T14:22:40.000-07:00","record_type":"message","direction":"f2b","message_type":"Terminate","text":"","rendered":"[14:22:40.000] [F→B] Terminate","client":"127.0.0.1:52413"} +{"schema_version":1,"timestamp":"2026-07-19T14:22:40.010-07:00","record_type":"line","text":"[14:22:40.010] === connection closed (FIN) ==="} diff --git a/flake.nix b/flake.nix index 0c103e9..38e8400 100644 --- a/flake.nix +++ b/flake.nix @@ -10,6 +10,7 @@ flake-utils.url = "github:numtide/flake-utils"; crane = { url = "github:ipetkov/crane"; + inputs.nixpkgs.follows = "nixpkgs"; }; }; @@ -38,15 +39,15 @@ # Source cleaning: keep cargo's own selection (crane's # commonCargoSources — every .rs/.toml/Cargo.lock across the workspace), - # plus the committed `man/sections.md` that the gen_manpage example - # embeds with include_str!. cleanCargoSource alone strips it (cargo - # doesn't track it), which breaks the example's build; fileset.toSource - # makes the extra include explicit and unambiguous. + # plus committed non-Rust inputs used by builds and tests. Cargo does + # not track these files, so commonCargoSources strips them unless the + # fileset includes them explicitly. src = pkgs.lib.fileset.toSource { root = ./.; fileset = pkgs.lib.fileset.unions [ (craneLib.fileset.commonCargoSources ./.) ./man/sections.md + ./tests/fixtures/session-v1.jsonl ]; }; diff --git a/man/sections.md b/man/sections.md index b0ed48d..ebdebbf 100644 --- a/man/sections.md +++ b/man/sections.md @@ -1,10 +1,49 @@ +# SAVED SESSIONS + +`--save FILE` continuously writes every output record as versioned JSONL while +normal stdout or TUI rendering continues. Recording occurs before display +filtering and before the TUI history cap, so hidden and evicted live records +remain in the saved session. Operational line/status records are saved along +with decoded PostgreSQL messages. An existing destination file is replaced. + +Output records flow through a bounded in-memory channel. If a consumer stalls +badly enough — a paused stdout pager, a wedged terminal, a stuck disk — records +are shed rather than buffered without limit or stalling capture; a count of any +dropped records is reported on exit. + +`--replay FILE` reads a saved session instead of starting pcap or mitm. Records +are replayed immediately through the same stdout/TUI renderer and display +filter as live traffic. Original capture timestamps, client address, direction, +message type/text, and rich RowDescription/DataRow details are preserved. A +replay can be copied to another file with `--save`; the input and output paths +must differ. + +The TUI command bar opens with `:`. `:save FILE` (`:w FILE`) writes the +currently retained history and continuously records future events. If earlier +events have left the 50,000-record TUI history, a footer warning reports the +omission; `:save` refuses to overwrite the file a `--replay` source is reading. +`:open FILE` (`:o FILE`) validates the complete file before replacing the view, +retains its newest 50,000 records, switches the UI to replay mode, and closes +any active recorder. Subsequent live-source display records are discarded so +timelines do not mix. Both commands accept a leading `~/` for the home directory. + +Press `/` to search the message text: matches are highlighted and `n` / `N` +jump to the next / previous match. `Esc` clears an active search. + +The current on-disk schema version is 1. Each JSONL record carries its own +`schema_version` and RFC 3339 timestamp. Unknown versions and malformed records +are refused with the file path and line number; tapgres does not guess at an +incompatible shape. The full format is documented in `docs/session-format.md` +in the source repository. + # DISPLAY FILTER EXPRESSIONS The `-Y` / `--display-filter` option limits decoded PostgreSQL messages in line-oriented output and supplies the initial display filter in `--tui` mode. The `-Y` shorthand mirrors Wireshark's display filters. Its value is parsed -once at startup; a parse error is fatal for stdout mode and is reported in the -TUI footer (the last valid filter stays active). +once at startup; a parse error there is fatal in both modes. Interactive filter +edits inside the TUI (`y`) instead report the error in the footer and keep the +last valid filter active. The expression language is a small, typed subset of Wireshark display-filter syntax: named fields are compared with operators and combined with boolean @@ -17,11 +56,17 @@ context, not decoded protocol messages, so they are never filtered out. : IP address. Example: `client.ip == 127.0.0.1` `client.port` -: integer. Example: `client.port in {40005, 40006}` +: integer. Example: `client.port in {40005, 40006}`. Supports ordered + comparisons (see Operators). Both bare (`40005`) and quoted (`"40005"`) forms + are accepted. `message.type` : string. Example: `message.type == "Query"`. A decoded pgwire message type, - e.g. Query, Parse, Bind, DataRow, RowDescription, ReadyForQuery. + e.g. Query, Parse, Bind, DataRow, RowDescription, ReadyForQuery. Note the + vocabulary is tapgres's own labels, which are case-sensitive for `==`/`!=`/ + `in`: server errors and notices use the short forms `"ERROR"`, `"NOTICE"`, + and `"NOTIFY"` (not `ErrorResponse`/`NoticeResponse`); warnings use + `"Warning"`. Use `matches` (case-insensitive) if unsure of the exact case. `message.text` : string. Example: `message.text contains "orders"`. The text payload: the SQL @@ -37,6 +82,10 @@ context, not decoded protocol messages, so they are never filtered out. : Equality and inequality. Valid for every field. String and direction comparisons are case-sensitive. +`<`, `<=`, `>`, `>=` +: Ordered comparison. Valid only for the numeric `client.port` field, e.g. + `client.port >= 40000 and client.port < 50000`. + `in {value, ...}` : Set membership. Values must match the field's type; a quoted-string set for string/direction fields, a bare-integer or IP set for numeric/address fields. @@ -59,11 +108,12 @@ String values must be double-quoted; backslash escapes (`\n`, `\r`, `\t`, ## In the TUI -Press `y` to edit the display filter. A valid edit is applied immediately to -the full retained message buffer, so previously hidden messages reappear when -the filter changes. An empty filter (or `Esc`) clears it. The message-view -border is green normally, yellow while a filter is active, and red while the -input is invalid. +Press `y` to edit the display filter, then `Enter` to apply it to the full +retained message buffer (previously hidden messages reappear when the filter +changes). While editing, `Esc` cancels the edit and restores the filter that +was active when the editor opened. Outside the editor, `Esc` clears the applied +filter. The message-view border is green normally, yellow while a filter is +active, and red while the input is invalid. # EXAMPLES diff --git a/packaging/tapgres-bin/PKGBUILD b/packaging/tapgres-bin/PKGBUILD index fb1723c..6771543 100644 --- a/packaging/tapgres-bin/PKGBUILD +++ b/packaging/tapgres-bin/PKGBUILD @@ -1,25 +1,43 @@ +# ----------------------------------------------------------------------------- +# TEMPLATE — DO NOT PUBLISH AS-IS. +# +# pkgver and the sha256sums* arrays below are placeholders. At release time +# .github/workflows/release.yml rewrites them with sed (pkgver from the tag, +# checksums from the freshly built release assets) before pushing to the AUR. +# pkgver=0.0.0 is deliberate, to make the template nature obvious. +# ----------------------------------------------------------------------------- # Maintainer: Ning Sun pkgname=tapgres-bin -pkgver=0.1.0 +pkgver=0.0.0 pkgrel=1 pkgdesc="Passively tap a local PostgreSQL port and decode its wire traffic to stdout" -arch=('x86_64') +arch=('x86_64' 'aarch64') url="https://github.com/sunng87/tapgres" provides=('tapgres') conflicts=('tapgres') license=('MIT') depends=('glibc' 'gcc-libs' 'libpcap') makedepends=('patchelf') -source=("$pkgname-$pkgver::https://github.com/sunng87/tapgres/releases/download/v${pkgver}/tapgres-linux-x86_64" - "tapgres-$pkgver.1.gz::https://github.com/sunng87/tapgres/releases/download/v${pkgver}/tapgres.1.gz") -sha256sums=('SKIP' 'SKIP') # Replaced with the real checksums by the release workflow +# The manpage is a shared (arch-independent) source; the prebuilt binary is +# arch-specific. The release publishes tapgres-linux-x86_64 (nix-built) and +# tapgres-linux-aarch64 (plain cargo build on GitHub's arm64 runner). +source=("tapgres-$pkgver.1.gz::https://github.com/sunng87/tapgres/releases/download/v${pkgver}/tapgres.1.gz") +source_x86_64=("$pkgname-$pkgver::https://github.com/sunng87/tapgres/releases/download/v${pkgver}/tapgres-linux-x86_64") +source_aarch64=("$pkgname-$pkgver::https://github.com/sunng87/tapgres/releases/download/v${pkgver}/tapgres-linux-aarch64") +sha256sums=('SKIP') # Replaced with the real checksum by the release workflow +sha256sums_x86_64=('SKIP') # Replaced with the real checksum by the release workflow +sha256sums_aarch64=('SKIP') # Replaced with the real checksum by the release workflow package() { - # The binary is produced by `nix build`, so its ELF interpreter and RUNPATH - # point into /nix/store. Repoint the interpreter to Arch's dynamic loader and - # drop the nix RUNPATH so libpcap.so.1 resolves from the system (/usr/lib). - patchelf --set-interpreter /usr/lib/ld-linux-x86-64.so.2 "$srcdir/$pkgname-$pkgver" - patchelf --remove-rpath "$srcdir/$pkgname-$pkgver" + if [[ $CARCH == x86_64 ]]; then + # The x86_64 binary is produced by `nix build`, so its ELF interpreter and + # RUNPATH point into /nix/store. Repoint the interpreter to Arch's dynamic + # loader and drop the nix RUNPATH so libpcap.so.1 resolves from the system + # (/usr/lib). The aarch64 binary is a plain cargo build on a GitHub arm64 + # runner and already uses the standard interpreter and no baked RUNPATH. + patchelf --set-interpreter /usr/lib/ld-linux-x86-64.so.2 "$srcdir/$pkgname-$pkgver" + patchelf --remove-rpath "$srcdir/$pkgname-$pkgver" + fi install -Dm755 "$srcdir/$pkgname-$pkgver" "$pkgdir/usr/bin/tapgres" install -Dm644 "$srcdir/tapgres-$pkgver.1.gz" "$pkgdir/usr/share/man/man1/tapgres.1.gz" } diff --git a/packaging/tapgres/PKGBUILD b/packaging/tapgres/PKGBUILD index dc1eed9..97facd4 100644 --- a/packaging/tapgres/PKGBUILD +++ b/packaging/tapgres/PKGBUILD @@ -1,6 +1,14 @@ +# ----------------------------------------------------------------------------- +# TEMPLATE — DO NOT PUBLISH AS-IS. +# +# pkgver and sha256sums below are placeholders. At release time +# .github/workflows/release.yml rewrites them with sed (pkgver from the tag, +# sha256sums from the source tarball) before pushing to the AUR. +# pkgver=0.0.0 is deliberate, to make the template nature obvious. +# ----------------------------------------------------------------------------- # Maintainer: Ning Sun pkgname=tapgres -pkgver=0.1.0 +pkgver=0.0.0 pkgrel=1 pkgdesc="Passively tap a local PostgreSQL port and decode its wire traffic to stdout" arch=('x86_64' 'aarch64') diff --git a/src/capture.rs b/src/capture.rs index 276ee0e..de8e194 100644 --- a/src/capture.rs +++ b/src/capture.rs @@ -49,8 +49,16 @@ pub fn run(opts: PcapOpts, metrics: Arc) -> Result<(), Box> .promisc(!opts.no_promisc) .snaplen(opts.snaplen) .timeout(1000) + // Deliver packets as they arrive instead of in kernel-buffered batches, + // so an interactive tap (especially on BSD/macOS BPF) isn't delayed. + .immediate_mode(true) .open()?; - cap.filter(&format!("tcp port {}", opts.port), true)?; + // Also match VLAN-tagged frames; `net::strip_link` already unwraps 802.1Q, + // but the plain `tcp port` filter would never let a tagged frame through. + cap.filter( + &format!("tcp port {p} or (vlan and tcp port {p})", p = opts.port), + true, + )?; let dlt = cap.get_datalink().0; decode::status(format!( @@ -60,14 +68,35 @@ pub fn run(opts: PcapOpts, metrics: Arc) -> Result<(), Box> )); let mut table = flow::ConnTable::with_metrics(metrics); + let mut warned_truncation = false; + let mut last_dropped = 0u32; + let mut since_stats = 0u32; loop { match cap.next_packet() { Ok(packet) => { + // Snaplen truncation guarantees a reassembly gap; warn once so + // the operator can raise --snaplen instead of chasing silence. + if packet.header.caplen < packet.header.len && !warned_truncation { + warned_truncation = true; + decode::status(format!( + "tapgres: warning — packets truncated to {} of {} bytes; \ + raise --snaplen to avoid gaps in decoding", + packet.header.caplen, packet.header.len + )); + } if let Some(seg) = net::parse_frame(packet.data, dlt) { table.handle(&seg, opts.port); } + since_stats += 1; + if since_stats >= 10_000 { + since_stats = 0; + report_drops(&mut cap, &mut last_dropped); + } + } + Err(pcap::Error::TimeoutExpired) => { + report_drops(&mut cap, &mut last_dropped); + continue; } - Err(pcap::Error::TimeoutExpired) => continue, Err(pcap::Error::NoMorePackets) => break, Err(e) => return Err(e.into()), } @@ -75,6 +104,21 @@ pub fn run(opts: PcapOpts, metrics: Arc) -> Result<(), Box> Ok(()) } +/// Report newly kernel-dropped packets (capture couldn't keep up) since the +/// last check, as a status line. Silent when nothing was dropped. +fn report_drops(cap: &mut Capture, last_dropped: &mut u32) { + if let Ok(stats) = cap.stats() { + if stats.dropped > *last_dropped { + let delta = stats.dropped - *last_dropped; + *last_dropped = stats.dropped; + decode::status(format!( + "tapgres: kernel dropped {delta} packets (capture can't keep up); \ + decoding may have gaps" + )); + } + } +} + /// Resolve which capture device to use. /// /// - `None` (the default): the loopback interface, found by its pcap loopback diff --git a/src/cli.rs b/src/cli.rs index 1672c57..e82be3b 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -22,7 +22,8 @@ use crate::state; with the pgwire protocol layer. Use --mode pcap (the default) to passively \ capture a local port with libpcap (cleartext only), or --mode mitm to run a \ local TLS-terminating proxy that decrypts encrypted sessions. Add --tui to \ - either source for an interactive, scrollable, filterable view.", + either source for an interactive, scrollable, filterable view. Use --save to \ + record versioned JSONL or --replay to open a saved session without capture.", before_help = crate::tui::BANNER )] pub struct Args { @@ -30,14 +31,14 @@ pub struct Args { #[arg(long, value_enum, default_value_t = Mode::Pcap)] pub mode: Mode, - /// Interactive TUI instead of line-oriented stdout (works with any --mode). + /// Interactive TUI instead of line-oriented stdout (works with live or replay sources). #[arg(long, default_value_t = false)] pub tui: bool, /// [tui] Start with rich display mode on: per-message key/value tables for /// `DataRow` and typed column lists for `RowDescription`, instead of the - /// flat line view. Toggle at runtime with `r`. - #[arg(long, default_value_t = false)] + /// flat line view. Toggle at runtime with `r`. Only meaningful with --tui. + #[arg(long, default_value_t = false, requires = "tui")] pub tui_rich: bool, /// Display only decoded messages matching this expression. @@ -45,6 +46,34 @@ pub struct Args { #[arg(short = 'Y', long = "display-filter")] pub display_filter: Option, + /// Save every live or replayed output record as versioned JSONL while + /// continuing to render normally. Recording happens before display + /// filtering and before the TUI history cap is applied. An existing file + /// is replaced. + #[arg(long, value_name = "FILE")] + pub save: Option, + + /// Read a saved JSONL session instead of starting pcap or mitm capture. + /// Replay is loaded at full speed and preserves original timestamps. + #[arg( + long, + value_name = "FILE", + conflicts_with_all = [ + "mode", + "port", + "interface", + "no_promisc", + "snaplen", + "listen", + "upstream", + "tls_dir", + "tls_cert", + "tls_key", + "no_upstream_tls" + ] + )] + pub replay: Option, + /// Maximum retained open + recently-closed connection records. /// Open connections are never evicted. #[arg(long, default_value_t = state::DEFAULT_CONNECTION_CAP)] @@ -120,3 +149,34 @@ pub enum Mode { pub fn command() -> clap::Command { Args::command() } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_save_and_replay_as_file_source_options() { + let args = Args::try_parse_from([ + "tapgres", + "--replay", + "capture.jsonl", + "--save", + "copy.jsonl", + "--tui", + ]) + .unwrap(); + + assert_eq!(args.replay, Some(PathBuf::from("capture.jsonl"))); + assert_eq!(args.save, Some(PathBuf::from("copy.jsonl"))); + assert!(args.tui); + } + + #[test] + fn replay_rejects_live_source_options() { + let error = + Args::try_parse_from(["tapgres", "--replay", "capture.jsonl", "--mode", "mitm"]) + .unwrap_err(); + + assert_eq!(error.kind(), clap::error::ErrorKind::ArgumentConflict); + } +} diff --git a/src/decode.rs b/src/decode.rs index 00381ed..6b0cf41 100644 --- a/src/decode.rs +++ b/src/decode.rs @@ -7,12 +7,13 @@ use std::cell::RefCell; use std::fmt::Write as _; -use std::sync::Mutex; +use std::sync::RwLock; +use std::sync::atomic::{AtomicU64, Ordering}; -use crossbeam_channel::Sender; +use crossbeam_channel::{Receiver, Sender}; use bytes::{Buf, Bytes}; -use chrono::Local; +use chrono::{Local, SecondsFormat}; use crate::filter::{DisplayFilter, DisplayMessage, MessageDirection}; use crate::flow::{Direction, Role}; @@ -134,20 +135,42 @@ impl Output { } } +/// Capacity of the output channel. Large enough that ordinary bursts never +/// stall, but bounded: a wedged consumer (a paused stdout pager, a stalled +/// `--save` disk) then sheds records instead of growing memory without limit or +/// — critically for mitm mode — stalling the client↔server relay it sits in. +pub const OUTPUT_CHANNEL_CAPACITY: usize = 131_072; + /// The global producer handle. `None` (the default) means no consumer is wired -/// and `out`/`status` print directly to stdout/stderr. -static OUTPUT_TX: Mutex>> = Mutex::new(None); +/// and `out`/`status` print directly to stdout/stderr. An `RwLock` (not a +/// `Mutex`) so the many concurrent mitm pump tasks don't serialize on the read +/// path — only `set_output`/`close_output` take the write lock. +static OUTPUT_TX: RwLock>> = RwLock::new(None); + +/// Count of records shed because the consumer could not keep up. +static DROPPED: AtomicU64 = AtomicU64::new(0); + +/// Create the bounded output channel. Both the stdout printer and the TUI use +/// this so the backpressure policy is identical. +pub fn channel() -> (Sender, Receiver) { + crossbeam_channel::bounded(OUTPUT_CHANNEL_CAPACITY) +} + +/// How many output records have been dropped because the consumer fell behind. +pub fn dropped_count() -> u64 { + DROPPED.load(Ordering::Relaxed) +} /// Install the channel producers write to. The matching receiver is owned by /// whichever consumer is active (stdout-printer thread, or the TUI). pub fn set_output(tx: Sender) { - *OUTPUT_TX.lock().unwrap() = Some(tx); + *OUTPUT_TX.write().unwrap() = Some(tx); } /// Drop the producer handle so the consumer observes end-of-stream and can /// flush/drain. pub fn close_output() { - *OUTPUT_TX.lock().unwrap() = None; + *OUTPUT_TX.write().unwrap() = None; } fn deliver(record: Output) { @@ -157,8 +180,12 @@ fn deliver(record: Output) { CAPTURE.with(|c| c.borrow_mut().as_mut().unwrap().push(record)); return; } - if let Some(tx) = &*OUTPUT_TX.lock().unwrap() { - let _ = tx.send(record); // unbounded channel: never blocks + if let Some(tx) = &*OUTPUT_TX.read().unwrap() { + // Non-blocking: never stall the capture thread or the mitm relay. If the + // consumer is too far behind, shed this record and count it. + if tx.try_send(record).is_err() { + DROPPED.fetch_add(1, Ordering::Relaxed); + } return; } // No consumer wired: fall back to direct terminal output. @@ -169,6 +196,13 @@ fn deliver(record: Output) { } } +/// Feed a previously decoded record through the active consumer. File replay +/// uses this entry point so it follows the exact same stdout/TUI path as live +/// capture without exposing the decoder's routing internals. +pub fn replay(record: Output) { + deliver(record); +} + /// Emit one decoded protocol line. Routed to the output consumer, or stdout if /// none is wired. pub fn out(line: String) { @@ -224,13 +258,17 @@ impl MessageEmitter { } fn emit_with_detail(self, kind: &str, text: &str, detail: Option) { + let captured_at = Local::now(); + let timestamp = captured_at.to_rfc3339_opts(SecondsFormat::Millis, true); + let display_time = captured_at.format("%H:%M:%S%.3f"); let rendered = if text.is_empty() { - format!("[{}] [{}] {}", ts(), dir_tag(self.role), kind) + format!("[{display_time}] [{}] {kind}", dir_tag(self.role)) } else { - format!("[{}] [{}] {}: {}", ts(), dir_tag(self.role), kind, text) + format!("[{display_time}] [{}] {kind}: {text}", dir_tag(self.role)) }; deliver(Output::Message { message: DisplayMessage { + timestamp, rendered, client: self.client, direction: if self.role == Role::Client { @@ -246,9 +284,13 @@ impl MessageEmitter { } fn warn(self, msg: &str) { - let rendered = format!("[{}] [{}] ⚠ {}", ts(), dir_tag(self.role), msg); + let captured_at = Local::now(); + let timestamp = captured_at.to_rfc3339_opts(SecondsFormat::Millis, true); + let display_time = captured_at.format("%H:%M:%S%.3f"); + let rendered = format!("[{display_time}] [{}] ⚠ {msg}", dir_tag(self.role)); deliver(Output::Message { message: DisplayMessage { + timestamp, rendered, client: self.client, direction: if self.role == Role::Client { @@ -264,6 +306,13 @@ impl MessageEmitter { } } +/// Emit a warning line attributed to a connection direction. Used by the +/// reassembly layer (which has no `MessageEmitter`) to report lost or skipped +/// bytes so the warning is filterable and TUI-attributed like decoded messages. +pub fn warn(role: Role, client: std::net::SocketAddr, msg: &str) { + MessageEmitter { role, client }.warn(msg); +} + /// Signal that the *server* side should now expect a 1-byte SSL or GSS /// response, because the frontend just sent the matching request. /// @@ -302,7 +351,7 @@ pub struct DrainOutcome { /// See [`ServerNegotiationWait`] and [`DrainOutcome`] for how the caller learns /// about the SSL/GSS negotiation handoff and encryption. pub fn drain_direction(dir: &mut Direction, outcome: &mut DrainOutcome) { - if outcome.encrypted { + if outcome.encrypted || dir.dead { return; } if dir.role == Role::Client { @@ -311,13 +360,14 @@ pub fn drain_direction(dir: &mut Direction, outcome: &mut DrainOutcome) { match PgWireFrontendMessage::decode(&mut dir.rxbuf, &dir.ctx) { Ok(None) => return, Ok(Some(msg)) => { + dir.decode_failures = 0; // progress: the stream is in sync let consumed = before.saturating_sub(dir.rxbuf.len()) as u64; if !handle_frontend(dir, msg, outcome, consumed) { return; } } Err(e) => { - decode_error(Role::Client, &e, &mut dir.rxbuf); + decode_error(dir, &e); return; } } @@ -328,11 +378,12 @@ pub fn drain_direction(dir: &mut Direction, outcome: &mut DrainOutcome) { match PgWireBackendMessage::decode(&mut dir.rxbuf, &dir.ctx) { Ok(None) => return, Ok(Some(msg)) => { + dir.decode_failures = 0; let consumed = before.saturating_sub(dir.rxbuf.len()) as u64; handle_backend(dir, msg, outcome, consumed); } Err(e) => { - decode_error(Role::Server, &e, &mut dir.rxbuf); + decode_error(dir, &e); return; } } @@ -340,24 +391,30 @@ pub fn drain_direction(dir: &mut Direction, outcome: &mut DrainOutcome) { } } -fn decode_error(role: Role, e: &pgwire::error::PgWireError, buf: &mut bytes::BytesMut) { - // The buffer is out of sync with the protocol; rather than crash, report and - // drop the remainder so a later, well-formed message can still be seen. - out(format!( - "[{}] [{}] ⚠ decode error ({} lost bytes): {}", - role_dbg(role), - dir_tag(role), - buf.len(), - e - )); - buf.clear(); -} +/// Give up on a direction after this many consecutive decode failures. Occasional +/// failures recover (a resync gap, capture joined mid-message); a persistent run +/// means the stream is desynced and every future segment starts mid-message. +const MAX_DECODE_FAILURES: u32 = 8; -fn role_dbg(role: Role) -> &'static str { - if role == Role::Client { - "client" +fn decode_error(dir: &mut Direction, e: &pgwire::error::PgWireError) { + // The buffer is out of sync with the protocol; rather than crash, report and + // drop the remainder so a later, well-formed message can still be seen. The + // warning is emitted as an attributed message (not a bare line) so it shares + // the client/direction metadata and honors display filters. + let lost = dir.rxbuf.len(); + dir.rxbuf.clear(); + dir.decode_failures += 1; + let emitter = MessageEmitter { + role: dir.role, + client: dir.client, + }; + if dir.decode_failures >= MAX_DECODE_FAILURES { + dir.dead = true; + emitter.warn(&format!( + "decode error ({lost} lost bytes): {e}; stream desynced, giving up decoding this direction" + )); } else { - "server" + emitter.warn(&format!("decode error ({lost} lost bytes): {e}")); } } @@ -414,19 +471,52 @@ fn handle_frontend( dir.ctx.awaiting_frontend_startup = false; emitter.emit("Startup", &format_startup(&s)); } - PgWireFrontendMessage::CancelRequest(_) => { - emitter.emit("CancelRequest", ""); + PgWireFrontendMessage::CancelRequest(c) => { + emitter.emit("CancelRequest", &format_cancel(&c)); } PgWireFrontendMessage::Query(q) => { emitter.emit("Query", &query_text(&q)); } - PgWireFrontendMessage::Parse(p) => emitter.emit("Parse", &format_parse(&p)), - PgWireFrontendMessage::Bind(b) => emitter.emit("Bind", &format_bind(&b)), + PgWireFrontendMessage::Parse(p) => { + // Remember the statement's SQL so later Bind/Execute can show it. + // Bounded so a connection that churns distinct statement names can't + // grow the map without limit. + if dir.prepared.len() < PREPARED_CACHE_CAP { + dir.prepared + .insert(p.name.clone().unwrap_or_default(), p.query.clone()); + } + emitter.emit("Parse", &format_parse(&p)) + } + PgWireFrontendMessage::Bind(b) => { + dir.portals.insert( + b.portal_name.clone().unwrap_or_default(), + b.statement_name.clone().unwrap_or_default(), + ); + let sql = dir.prepared.get(b.statement_name.as_deref().unwrap_or("")); + emitter.emit("Bind", &format_bind(&b, sql.map(String::as_str))) + } PgWireFrontendMessage::Describe(d) => { emitter.emit("Describe", &format_describe_close(d.target_type, &d.name)) } - PgWireFrontendMessage::Execute(e) => emitter.emit("Execute", &format_execute(&e)), + PgWireFrontendMessage::Execute(e) => { + // Resolve portal → statement → SQL. + let sql = dir + .portals + .get(e.name.as_deref().unwrap_or("")) + .and_then(|stmt| dir.prepared.get(stmt)); + emitter.emit("Execute", &format_execute(&e, sql.map(String::as_str))) + } PgWireFrontendMessage::Close(c) => { + // Drop the closed statement/portal so its SQL doesn't linger. + match c.target_type { + b'S' => { + dir.prepared.remove(c.name.as_deref().unwrap_or("")); + } + b'P' => { + dir.portals.remove(c.name.as_deref().unwrap_or("")); + } + _ => {} + } emitter.emit("Close", &format_describe_close(c.target_type, &c.name)) } PgWireFrontendMessage::Sync(_) => emitter.emit("Sync", ""), @@ -492,6 +582,13 @@ fn handle_backend( // in the extended protocol a statement/portal is described once but // may be executed across many ReadyForQuery cycles, so the columns // must outlive a single command cycle. + // + // Known limitation: one cache per direction. Two portals with + // different result shapes executed alternately would label each + // other's `DataRow`s. Correct labelling needs a per-portal map keyed + // off the request pipeline; the single cache is a deliberate + // simplification that is correct for the common one-portal-at-a-time + // case. dir.row_desc = Some(summary); } PgWireBackendMessage::NoData(_) => { @@ -599,7 +696,11 @@ fn format_parse(p: &Parse) -> String { format!("{} [param types: {}] {}", name, types, p.query) } -fn format_bind(b: &Bind) -> String { +/// Cap on remembered prepared statements per direction; guards against a +/// connection that never closes its statements from growing the cache forever. +const PREPARED_CACHE_CAP: usize = 4096; + +fn format_bind(b: &Bind, sql: Option<&str>) -> String { let portal = b.portal_name.as_deref().unwrap_or(""); let stmt = b.statement_name.as_deref().unwrap_or(""); let all_text = b @@ -627,13 +728,17 @@ fn format_bind(b: &Bind) -> String { } }) .collect(); - format!( + let mut out = format!( "{} <- {} params: [{}] result: {}", portal, stmt, params.join(", "), format_format_codes(&b.result_column_format_codes), - ) + ); + if let Some(sql) = sql { + let _ = write!(out, " sql: {sql}"); + } + out } /// Render format codes (text=0, binary=1) compactly: `text`, `binary`, or a @@ -667,13 +772,21 @@ fn format_describe_close(target_type: u8, name: &Option) -> String { format!("{} {}", kind, name.as_deref().unwrap_or("")) } -fn format_execute(e: &Execute) -> String { +fn format_execute(e: &Execute, sql: Option<&str>) -> String { let name = e.name.as_deref().unwrap_or(""); - if e.max_rows == 0 { + let mut out = if e.max_rows == 0 { name.to_string() } else { format!("{} (limit {})", name, e.max_rows) + }; + if let Some(sql) = sql { + let _ = write!(out, " sql: {sql}"); } + out +} + +fn format_cancel(c: &pgwire::messages::cancel::CancelRequest) -> String { + format!("pid={} key={}", c.pid, secret_key_str(&c.secret_key)) } fn format_auth(a: &Authentication) -> String { @@ -689,12 +802,15 @@ fn format_auth(a: &Authentication) -> String { } } -fn format_bkd(b: &BackendKeyData) -> String { - let key = match &b.secret_key { +fn secret_key_str(key: &SecretKey) -> String { + match key { SecretKey::I32(i) => i.to_string(), SecretKey::Bytes(bs) => hex_preview(bs), - }; - format!("pid={} key={}", b.pid, key) + } +} + +fn format_bkd(b: &BackendKeyData) -> String { + format!("pid={} key={}", b.pid, secret_key_str(&b.secret_key)) } fn format_negotiate(n: &NegotiateProtocolVersion) -> String { @@ -899,7 +1015,18 @@ fn hex_preview(b: &[u8]) -> String { fn format_bytes(b: &Bytes) -> String { if is_printable(b) { - String::from_utf8_lossy(b).into_owned() + // A bulk COPY streams whole chunks through here; cap the printable + // preview like the hex path so one CopyData can't emit a multi-MB line. + const MAX: usize = 256; + if b.len() > MAX { + format!( + "{}… ({} bytes)", + String::from_utf8_lossy(&b[..MAX]), + b.len() + ) + } else { + String::from_utf8_lossy(b).into_owned() + } } else { hex_preview(b) } @@ -1007,6 +1134,91 @@ mod tests { assert_eq!(format_columns(&cols, false), "['x', 'y']"); } + #[test] + fn bind_and_execute_resolve_prepared_sql() { + let bind = Bind::new(Some("p1".into()), Some("s1".into()), vec![], vec![], vec![]); + let sql = "SELECT id FROM users WHERE tenant = $1"; + assert!(format_bind(&bind, Some(sql)).contains(sql)); + assert!(!format_bind(&bind, None).contains("sql:")); + + let exec = Execute::new(Some("p1".into()), 0); + assert!(format_execute(&exec, Some(sql)).contains(sql)); + assert_eq!(format_execute(&exec, None), "p1"); + } + + #[test] + fn parse_then_execute_end_to_end_shows_sql() { + use crate::flow::Direction; + start_capture(); + let mut dir = Direction::for_decoding(Role::Client, "127.0.0.1:40000".parse().unwrap()); + let mut outcome = DrainOutcome::default(); + let sql = "SELECT * FROM orders WHERE id = $1"; + // Parse s1, Bind p1<-s1, Execute p1 — driven directly through the handler. + handle_frontend( + &mut dir, + PgWireFrontendMessage::Parse(Parse::new(Some("s1".into()), sql.into(), vec![])), + &mut outcome, + 0, + ); + handle_frontend( + &mut dir, + PgWireFrontendMessage::Bind(Bind::new( + Some("p1".into()), + Some("s1".into()), + vec![], + vec![], + vec![], + )), + &mut outcome, + 0, + ); + handle_frontend( + &mut dir, + PgWireFrontendMessage::Execute(Execute::new(Some("p1".into()), 0)), + &mut outcome, + 0, + ); + let out = take_output_capture(); + let execute = out + .iter() + .find_map(|o| match o { + Output::Message { message, .. } if message.kind == "Execute" => Some(message), + _ => None, + }) + .expect("an Execute message"); + assert!( + execute.text.contains(sql), + "Execute should resolve to its SQL" + ); + } + + #[test] + fn printable_payload_preview_is_truncated() { + let big = Bytes::from(vec![b'x'; 5000]); + let rendered = format_bytes(&big); + assert!(rendered.len() < 400, "preview must be capped"); + assert!(rendered.contains("(5000 bytes)")); + } + + #[test] + fn persistent_decode_failures_mark_direction_dead() { + use crate::flow::Direction; + start_capture(); + let mut dir = Direction::for_decoding(Role::Server, "127.0.0.1:40000".parse().unwrap()); + // Feed junk that never decodes; each drain fails and clears the buffer. + for _ in 0..MAX_DECODE_FAILURES { + dir.rxbuf.extend_from_slice(&[0xff, 0xff, 0xff, 0xff, 0xff]); + let mut outcome = DrainOutcome::default(); + drain_direction(&mut dir, &mut outcome); + } + assert!(dir.dead, "should give up after repeated failures"); + // Once dead, further drains are a no-op even with fresh bytes. + dir.rxbuf.extend_from_slice(&[0xff; 5]); + let mut outcome = DrainOutcome::default(); + drain_direction(&mut dir, &mut outcome); + let _ = take_output_capture(); + } + #[test] fn decoded_output_carries_filter_metadata() { start_capture(); @@ -1039,6 +1251,7 @@ mod tests { .unwrap(); let query = Output::Message { message: DisplayMessage { + timestamp: "2026-07-17T12:34:56.789+01:00".into(), rendered: "query".into(), client: "127.0.0.1:40005".parse().unwrap(), direction: MessageDirection::FrontendToBackend, @@ -1049,6 +1262,7 @@ mod tests { }; let row = Output::Message { message: DisplayMessage { + timestamp: "2026-07-17T12:34:56.789+01:00".into(), rendered: "row".into(), client: "127.0.0.1:40005".parse().unwrap(), direction: MessageDirection::BackendToFrontend, diff --git a/src/filter.rs b/src/filter.rs index cfe2450..5b7919c 100644 --- a/src/filter.rs +++ b/src/filter.rs @@ -6,9 +6,13 @@ //! //! ```text //! client.ip == 127.0.0.1 and client.port == 40005 +//! client.port >= 40000 and client.port < 50000 //! message.type in {"Query", "DataRow"} and message.text contains "orders" //! not (message.direction == "b2f" or message.type matches r"^Error") //! ``` +//! +//! Ordered comparisons (`<`, `<=`, `>`, `>=`) apply to the numeric +//! `client.port` field only. use std::fmt; use std::net::{IpAddr, SocketAddr}; @@ -25,6 +29,9 @@ pub enum MessageDirection { /// A decoded message plus the structured fields used by display filters. #[derive(Clone, Debug)] pub struct DisplayMessage { + /// Original capture time in RFC 3339 with millisecond precision. Kept + /// separately from `rendered` so saved sessions preserve real timestamps. + pub timestamp: String, pub rendered: String, pub client: SocketAddr, pub direction: MessageDirection, @@ -122,10 +129,32 @@ enum Value { Direction(MessageDirection), } +/// Ordered comparison operator, for numeric fields (`client.port`). +#[derive(Clone, Copy, Debug)] +enum OrdOp { + Less, + LessEqual, + Greater, + GreaterEqual, +} + +impl OrdOp { + fn test(self, actual: u16, bound: u16) -> bool { + match self { + Self::Less => actual < bound, + Self::LessEqual => actual <= bound, + Self::Greater => actual > bound, + Self::GreaterEqual => actual >= bound, + } + } +} + #[derive(Clone, Debug)] enum Comparison { Equal(Value), NotEqual(Value), + /// An ordered comparison against a port number (`client.port > 40000`). + Ordered(OrdOp, u16), Contains(String), Matches(Regex), In(Vec), @@ -142,6 +171,7 @@ impl Predicate { match &self.comparison { Comparison::Equal(value) => self.field.equals(value, message), Comparison::NotEqual(value) => !self.field.equals(value, message), + Comparison::Ordered(op, bound) => op.test(message.client.port(), *bound), Comparison::Contains(needle) => self .field .text(message) @@ -221,6 +251,10 @@ enum TokenKind { String(String), Equal, NotEqual, + Less, + LessEqual, + Greater, + GreaterEqual, Contains, Matches, In, @@ -285,6 +319,25 @@ fn lex(input: &str) -> Result, FilterParseError> { }); continue; } + // Ordered comparisons (two-char forms before one-char). + for (symbol, kind) in [ + ("<=", TokenKind::LessEqual), + (">=", TokenKind::GreaterEqual), + ("<", TokenKind::Less), + (">", TokenKind::Greater), + ] { + if input[start..].starts_with(symbol) { + position += symbol.len(); + tokens.push(Token { + kind: kind.clone(), + position: start, + }); + break; + } + } + if position != start { + continue; + } if input[start..].starts_with("&&") { position += 2; tokens.push(Token { @@ -337,8 +390,14 @@ fn lex(input: &str) -> Result, FilterParseError> { while position < input.len() { let ch = input[position..].chars().next().unwrap(); + // A quote or comparison operator ends the bare word, so + // `message.text contains"x"` and `client.port<40000` lex correctly + // instead of swallowing the operator into the field/word. if ch.is_whitespace() - || matches!(ch, '(' | ')' | '{' | '}' | ',' | '=' | '!' | '&' | '|') + || matches!( + ch, + '(' | ')' | '{' | '}' | ',' | '=' | '!' | '&' | '|' | '<' | '>' | '"' + ) { break; } @@ -482,6 +541,27 @@ impl Parser { let comparison = match operator.kind { TokenKind::Equal => Comparison::Equal(self.parse_value(field)?), TokenKind::NotEqual => Comparison::NotEqual(self.parse_value(field)?), + TokenKind::Less + | TokenKind::LessEqual + | TokenKind::Greater + | TokenKind::GreaterEqual => { + let op = match operator.kind { + TokenKind::Less => OrdOp::Less, + TokenKind::LessEqual => OrdOp::LessEqual, + TokenKind::Greater => OrdOp::Greater, + _ => OrdOp::GreaterEqual, + }; + if !matches!(field, Field::ClientPort) { + return Err(FilterParseError::new( + operator.position, + format!( + "ordered comparison is only valid for 'client.port', not '{}'", + field.name() + ), + )); + } + Comparison::Ordered(op, self.parse_port()?) + } TokenKind::Contains => { self.require_text_field(field, operator.position, "contains")?; Comparison::Contains(self.parse_string("contains requires a quoted string")?) @@ -503,7 +583,7 @@ impl Parser { return Err(FilterParseError::new( operator.position, format!( - "expected an operator after '{}' (==, !=, contains, matches, or in)", + "expected an operator after '{}' (==, !=, <, <=, >, >=, contains, matches, or in)", field.name() ), )); @@ -550,12 +630,11 @@ impl Parser { }) } Field::ClientPort => { - let TokenKind::Word(value) = &token.kind else { - return Err(FilterParseError::new( - token.position, - "client.port requires an integer", - )); - }; + // Accept both `client.port == 40005` and the quoted form + // `== "40005"`, mirroring client.ip's leniency. + let value = token_value(&token).ok_or_else(|| { + FilterParseError::new(token.position, "client.port requires an integer") + })?; value.parse().map(Value::Port).map_err(|_| { FilterParseError::new(token.position, format!("invalid client port '{value}'")) }) @@ -594,6 +673,17 @@ impl Parser { Ok(value) } + /// Parse a port number (bare or quoted) for an ordered comparison. + fn parse_port(&mut self) -> Result { + let token = self.take(); + let value = token_value(&token).ok_or_else(|| { + FilterParseError::new(token.position, "expected a port number after the operator") + })?; + value.parse().map_err(|_| { + FilterParseError::new(token.position, format!("invalid client port '{value}'")) + }) + } + fn require_text_field( &self, field: Field, @@ -672,6 +762,7 @@ mod tests { fn message() -> DisplayMessage { DisplayMessage { + timestamp: "2026-07-17T12:34:56.789+01:00".into(), rendered: "line".into(), client: "127.0.0.1:40005".parse().unwrap(), direction: MessageDirection::FrontendToBackend, @@ -680,6 +771,56 @@ mod tests { } } + #[test] + fn ordered_port_comparisons() { + // message() has client.port 40005. + for (expr, expected) in [ + ("client.port > 40000", true), + ("client.port < 40000", false), + ("client.port >= 40005", true), + ("client.port <= 40005", true), + ("client.port < 40005", false), + ("client.port >= 40000 and client.port < 50000", true), + ] { + assert_eq!( + DisplayFilter::parse(expr).unwrap().matches(&message()), + expected, + "{expr}" + ); + } + // Ordered comparison is rejected on non-numeric fields. + assert!(DisplayFilter::parse("message.type > \"Query\"").is_err()); + assert!(DisplayFilter::parse("client.ip < 127.0.0.1").is_err()); + } + + #[test] + fn quoted_port_is_accepted_symmetrically_with_ip() { + // Both quoted and bare forms parse and match, like client.ip. + assert!( + DisplayFilter::parse("client.port == \"40005\"") + .unwrap() + .matches(&message()) + ); + assert!( + DisplayFilter::parse("client.port == 40005") + .unwrap() + .matches(&message()) + ); + } + + #[test] + fn quote_terminates_a_bare_word() { + // `contains"orders"` (no space) must still recognise the operator. + let filter = DisplayFilter::parse("message.text contains\"orders\"").unwrap(); + assert!(filter.matches(&message())); + // And a comparison operator glued to the field lexes correctly. + assert!( + DisplayFilter::parse("client.port<50000") + .unwrap() + .matches(&message()) + ); + } + #[test] fn combines_typed_conditions_with_boolean_operators() { let filter = DisplayFilter::parse( @@ -769,7 +910,7 @@ mod tests { let cases = [ "client == 127.0.0.1", "client.ip contains \"127\"", - "client.port == \"40005\"", + "client.port == \"not-a-number\"", "message.type == Query", "message.direction == \"sideways\"", "message.type in {}", diff --git a/src/flow.rs b/src/flow.rs index ee6c364..90f2dd1 100644 --- a/src/flow.rs +++ b/src/flow.rs @@ -5,6 +5,7 @@ //! direction in order, and feeds the resulting byte stream into the pgwire //! message decoder. +use std::collections::btree_map::Entry; use std::collections::{BTreeMap, HashMap}; use std::net::{IpAddr, SocketAddr}; use std::sync::Arc; @@ -19,6 +20,25 @@ use crate::state::{ConnStats, Metrics, TrafficDirection}; /// (ip, port) pub type Endpoint = (IpAddr, u16); +/// Cap on out-of-order bytes buffered per direction. Normal reordering settles +/// within a handful of segments; passing this means a segment the kernel +/// dropped before we saw it will never arrive, so we resync past the hole +/// rather than stall and grow without bound. +const OOO_BYTES_CAP: usize = 256 * 1024; + +/// A closed connection is kept this many segments (of subsequent global +/// activity) so trailing close-handshake packets and retransmits land on it +/// instead of spawning a phantom new connection, then it is swept. +const CLOSE_GRACE_SEGMENTS: u64 = 256; + +/// How often, in handled segments, to sweep the table for evictable entries. +const SWEEP_INTERVAL: u64 = 512; + +/// Backstop bound on retained connections. Closed ones are swept promptly via +/// the grace period; this caps the pathological case of many connections whose +/// FIN/RST we never observed (client crash, capture started late, NAT rebind). +const MAX_CONNECTIONS: usize = 16_384; + /// Which side of the connection a direction belongs to. #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum Role { @@ -46,12 +66,28 @@ pub struct Direction { next_seq: Option, /// Segments received ahead of `next_seq`, keyed by their start sequence. ooo: BTreeMap>, + /// Total bytes held in `ooo`, kept in step with it so the buffering cap is + /// an O(1) check. + ooo_bytes: usize, /// Reassembled, in-order bytes awaiting decode. pub rxbuf: BytesMut, pub ctx: DecodeContext, /// Most recently seen `RowDescription` (server side), used to label /// `DataRow`s with column names and to pick text/binary rendering. pub row_desc: Option>, + /// Extended-protocol prepared statements (name → SQL) seen on this + /// direction's `Parse` messages, so `Bind`/`Execute` can be annotated with + /// the actual query instead of an opaque statement name. + pub prepared: HashMap, + /// Portal name → statement name, from `Bind`, so an `Execute` on a portal + /// resolves back to its prepared SQL. + pub portals: HashMap, + /// Consecutive decode failures; reset on any successful decode. Used to + /// stop decoding a hopelessly desynced direction rather than spam. + pub decode_failures: u32, + /// Set once a direction is given up as unrecoverably desynced; decoding + /// stops (relaying, in mitm mode, continues regardless). + pub dead: bool, } impl Direction { @@ -61,9 +97,14 @@ impl Direction { client, next_seq: None, ooo: BTreeMap::new(), + ooo_bytes: 0, rxbuf: BytesMut::with_capacity(8 * 1024), ctx: DecodeContext::default(), row_desc: None, + prepared: HashMap::new(), + portals: HashMap::new(), + decode_failures: 0, + dead: false, } } @@ -92,25 +133,31 @@ impl Direction { if data.is_empty() { return; } + // A SYN carrying data (TCP Fast Open) numbers that data from seq+1, not + // seq, since the SYN itself consumed the ISN. + let data_seq = if syn { seq.wrapping_add(1) } else { seq }; let next = match self.next_seq { Some(n) => n, // SYN missed (capture started mid-connection): anchor here. None => { - self.next_seq = Some(seq); - seq + self.next_seq = Some(data_seq); + data_seq } }; - // Signed distance from `next` to `seq` in wrapping 32-bit space. - let diff = seq.wrapping_sub(next) as i32; + // Signed distance from `next` to `data_seq` in wrapping 32-bit space. + let diff = data_seq.wrapping_sub(next) as i32; if diff > 0 { // Gap: this segment is ahead of what we can deliver yet. - self.ooo.insert(seq, data.to_vec()); + self.buffer_ooo(data_seq, data); + if self.ooo_bytes > OOO_BYTES_CAP { + self.resync_after_gap(); + } return; } - let (mut cur, mut chunk) = (seq, data); + let (mut cur, mut chunk) = (data_seq, data); if diff < 0 { // Overlapping / retransmitted: drop the bytes we've already seen. let skip = (-diff) as usize; @@ -121,16 +168,71 @@ impl Direction { cur = cur.wrapping_add(skip as u32); } - // cur == next: append in-order bytes. + // cur == next: append in-order bytes, then pull in any buffered + // segments that overlap or abut the new position. self.rxbuf.extend_from_slice(chunk); - let mut adv = cur.wrapping_add(chunk.len() as u32); + let adv = cur.wrapping_add(chunk.len() as u32); + self.next_seq = Some(self.drain_ooo(adv)); + } + + /// Buffer an out-of-order segment, keeping the longer of any two segments + /// that start at the same sequence — a shorter retransmit must not clobber + /// data we already hold. + fn buffer_ooo(&mut self, seq: u32, data: &[u8]) { + match self.ooo.entry(seq) { + Entry::Occupied(mut e) => { + if data.len() > e.get().len() { + self.ooo_bytes += data.len() - e.get().len(); + e.insert(data.to_vec()); + } + } + Entry::Vacant(e) => { + self.ooo_bytes += data.len(); + e.insert(data.to_vec()); + } + } + } - // Drain any buffered segments that now fit contiguously. - while let Some(b) = self.ooo.remove(&adv) { - self.rxbuf.extend_from_slice(&b); - adv = adv.wrapping_add(b.len() as u32); + /// Deliver every buffered segment that overlaps or abuts `adv`, returning + /// the new in-order position. Correctly handles re-segmented retransmissions + /// where a buffered segment starts before `adv` but extends past it, and + /// drops segments that are now fully behind `adv`. + /// + /// Assumes sequence numbers do not wrap within the buffered set; the + /// `OOO_BYTES_CAP` resync bounds that set well under the 4 GiB wrap window. + fn drain_ooo(&mut self, mut adv: u32) -> u32 { + while let Some((&start, _)) = self.ooo.range(..=adv).next_back() { + let buf = self.ooo.remove(&start).unwrap(); + self.ooo_bytes -= buf.len(); + let end = start.wrapping_add(buf.len() as u32); + if (end.wrapping_sub(adv) as i32) <= 0 { + continue; // entirely already delivered + } + let already = adv.wrapping_sub(start) as usize; + self.rxbuf.extend_from_slice(&buf[already..]); + adv = end; } - self.next_seq = Some(adv); + adv + } + + /// A capture gap has grown past the buffering cap, so a dropped segment will + /// never arrive. Skip the hole: drop the un-decodable prefix, jump to the + /// lowest buffered segment, and drain from there so decoding recovers + /// instead of stalling forever. + fn resync_after_gap(&mut self) { + let Some((&lo, _)) = self.ooo.iter().next() else { + return; + }; + let lost = self.next_seq.map(|n| lo.wrapping_sub(n)).unwrap_or(0); + decode::warn( + self.role, + self.client, + &format!("reassembly gap: ~{lost} bytes lost (capture drop?), resyncing stream"), + ); + // Pre-gap bytes end at the old `next_seq`; splicing them onto the + // post-gap bytes would forge a bogus message boundary, so drop them. + self.rxbuf.clear(); + self.next_seq = Some(self.drain_ooo(lo)); } } @@ -141,24 +243,35 @@ pub struct Connection { /// Set when the client negotiated SSL/GSS — the stream is then encrypted /// and we can no longer decode it. encrypted: bool, - /// Metrics close on the first FIN, while the flow entry remains to absorb - /// the rest of the TCP close handshake. - metrics_closed: bool, + /// Table clock at which metrics closed (first FIN/RST). `Some` marks the + /// entry a tombstone: it lingers to absorb the rest of the close handshake, + /// then the sweep removes it after [`CLOSE_GRACE_SEGMENTS`]. + closed_at: Option, + /// Table clock of the most recent segment on this connection, for LRU + /// eviction of connections whose close we never saw. + last_seen: u64, stats: Arc, } impl Connection { - fn new(stats: Arc) -> Self { + fn new(stats: Arc, now: u64) -> Self { let client = stats.client(); Self { client: Direction::new(Role::Client, client), server: Direction::new(Role::Server, client), encrypted: false, - metrics_closed: false, + closed_at: None, + last_seen: now, stats, } } + /// Whether a client SYN's ISN matches the handshake we already track (a + /// retransmitted SYN) rather than a fresh connection reusing the 4-tuple. + fn is_same_client_syn(&self, seq: u32) -> bool { + self.client.next_seq == Some(seq.wrapping_add(1)) + } + fn handle(&mut self, seg: &TcpSegment, pg_port: u16, metrics: &Metrics) { if self.encrypted { return; @@ -214,6 +327,15 @@ impl Connection { pub struct ConnTable { map: HashMap, metrics: Arc, + /// Monotonic count of handled segments, used as a logical clock for the + /// close-grace and idle/LRU eviction that keep `map` bounded. + clock: u64, + /// Clock value at which to run the next sweep. + next_sweep: u64, + /// Retained-connection cap; a field so tests can shrink it. + max_connections: usize, + /// One-shot guard for the both-ports-are-the-monitored-port warning. + warned_same_port: bool, } impl Default for ConnTable { @@ -234,11 +356,32 @@ impl ConnTable { Self { map: HashMap::new(), metrics, + clock: 0, + next_sweep: SWEEP_INTERVAL, + max_connections: MAX_CONNECTIONS, + warned_same_port: false, } } /// Ingest one captured TCP segment. pub fn handle(&mut self, seg: &TcpSegment, pg_port: u16) { + self.clock = self.clock.wrapping_add(1); + self.maybe_sweep(); + + // Both endpoints on the watched port (server-to-server): direction can't + // be classified, so both sides would collide in one decode buffer. Skip. + if seg.src_port == pg_port && seg.dst_port == pg_port { + if !self.warned_same_port { + self.warned_same_port = true; + decode::status( + "tapgres: ignoring traffic where both endpoints use the monitored port; \ + direction cannot be classified" + .into(), + ); + } + return; + } + // Classify direction by which endpoint owns the watched port. let (client, server) = if seg.dst_port == pg_port { ((seg.src, seg.src_port), (seg.dst, seg.dst_port)) @@ -250,14 +393,28 @@ impl ConnTable { let key = ConnKey { client, server }; - // A closed entry stays in the map after FIN to absorb the trailing - // ACK/FIN/ACK packets. A later SYN on the same 4-tuple is a genuine - // reuse and starts fresh decode and metrics state. - let should_open = self - .map - .get(&key) - .is_none_or(|conn| seg.syn && conn.metrics_closed); + // A client's initial SYN (travelling toward the monitored port). + let client_syn = seg.syn && seg.dst_port == pg_port; + let should_open = match self.map.get(&key) { + // A bare ACK/FIN/RST on an unknown 4-tuple (trailing close packets + // after eviction, a port-scan RST) must not spawn a phantom + // connection — only a SYN or a payload-bearing segment does. + None => seg.syn || !seg.payload.is_empty(), + // Reuse of the 4-tuple: reopen when a closed (tombstoned) entry sees + // any SYN, or when a live entry sees a *new* client handshake (its + // previous close was missed by the capture). + Some(conn) => { + (seg.syn && conn.closed_at.is_some()) + || (client_syn && !conn.is_same_client_syn(seg.seq)) + } + }; if should_open { + // Retire any stale entry we're replacing before opening afresh. + if let Some(old) = self.map.remove(&key) { + if old.closed_at.is_none() { + self.metrics.close_connection(&old.stats); + } + } decode::out(format!( "[{}] === new connection {}:{} -> {}:{} (port {}) ===", decode::ts(), @@ -272,29 +429,63 @@ impl ConnTable { SocketAddr::new(server.0, server.1), false, ); - self.map.insert(key, Connection::new(stats)); + self.map.insert(key, Connection::new(stats, self.clock)); } + let Some(conn) = self.map.get_mut(&key) else { + return; + }; + conn.last_seen = self.clock; + conn.handle(seg, pg_port, &self.metrics); + if seg.rst { - if let Some(conn) = self.map.get_mut(&key) { - conn.handle(seg, pg_port, &self.metrics); - } - decode::out(format!("[{}] === connection reset (RST) ===", decode::ts())); - if let Some(conn) = self.map.remove(&key) { + // Tombstone rather than remove: a duplicate/retransmitted RST or a + // trailing segment then lands on the closed entry instead of + // recreating a phantom connection. The sweep removes it later. + if conn.closed_at.is_none() { + conn.closed_at = Some(self.clock); self.metrics.close_connection(&conn.stats); + decode::out(format!("[{}] === connection reset (RST) ===", decode::ts())); } + } else if seg.fin && conn.closed_at.is_none() { + conn.closed_at = Some(self.clock); + decode::out(format!( + "[{}] === connection closed (FIN) ===", + decode::ts() + )); + self.metrics.close_connection(&conn.stats); + } + } + + /// Periodically evict connections that can no longer receive useful data: + /// tombstoned ones past their close grace, and — as a backstop — the + /// least-recently-seen live ones once the table exceeds its cap. + fn maybe_sweep(&mut self) { + if self.clock < self.next_sweep { return; } + self.next_sweep = self.clock.wrapping_add(SWEEP_INTERVAL); + let clock = self.clock; + self.map.retain(|_, conn| { + conn.closed_at + .is_none_or(|c| clock.wrapping_sub(c) <= CLOSE_GRACE_SEGMENTS) + }); + if self.map.len() > self.max_connections { + self.evict_lru(self.map.len() - self.max_connections); + } + } - if let Some(conn) = self.map.get_mut(&key) { - conn.handle(seg, pg_port, &self.metrics); - if seg.fin && !conn.metrics_closed { - conn.metrics_closed = true; - decode::out(format!( - "[{}] === connection closed (FIN) ===", - decode::ts() - )); - self.metrics.close_connection(&conn.stats); + /// Remove the `n` least-recently-seen connections, closing any that were + /// still live in the metrics registry so its lifecycle stays consistent. + fn evict_lru(&mut self, n: usize) { + let mut by_age: Vec<(u64, ConnKey)> = + self.map.iter().map(|(k, c)| (c.last_seen, *k)).collect(); + by_age.sort_unstable_by_key(|(seen, _)| *seen); + for (_, key) in by_age.into_iter().take(n) { + if let Some(conn) = self.map.remove(&key) { + if conn.closed_at.is_none() { + self.metrics.close_connection(&conn.stats); + } } } } @@ -394,4 +585,158 @@ mod tests { assert_eq!(closed.conns_live, 0); assert_eq!(closed.connections.len(), 2); } + + fn client_dir() -> Direction { + Direction::new(Role::Client, "127.0.0.1:40000".parse().unwrap()) + } + + #[test] + fn syn_with_data_keeps_the_first_byte() { + // TCP Fast Open: the SYN carries payload numbered from seq+1. + let mut d = client_dir(); + d.feed(100, true, b"AB"); + assert_eq!(&d.rxbuf[..], b"AB"); + d.feed(103, false, b"C"); + assert_eq!(&d.rxbuf[..], b"ABC"); + } + + #[test] + fn overlapping_retransmit_delivers_every_byte() { + // Anchor at 100, buffer an early [150,250) segment, then receive an + // in-order [100,200) segment that overlaps it. All of [100,250) must + // arrive contiguously — the classic stranded-suffix reassembly bug. + let mut d = client_dir(); + d.feed(99, true, b""); // SYN: next_seq = 100 + let early: Vec = (50u8..150).collect(); // seq 150..250 -> value seq-100 + d.feed(150, false, &early); + assert!( + d.rxbuf.is_empty(), + "early segment must be buffered, not delivered" + ); + let inorder: Vec = (0u8..100).collect(); // seq 100..200 -> value seq-100 + d.feed(100, false, &inorder); + let expected: Vec = (0u8..150).collect(); + assert_eq!(&d.rxbuf[..], &expected[..]); + assert_eq!(d.ooo_bytes, 0); + } + + #[test] + fn shorter_retransmit_does_not_clobber_a_buffered_segment() { + let mut d = client_dir(); + d.feed(99, true, b""); // next_seq = 100 + d.feed(200, false, b"LONGSEGMENT"); // buffered at 200 + d.feed(200, false, b"SHORT"); // same start, shorter: must be ignored + d.feed(100, false, &[b'.'; 100]); // fills the gap to 200 + assert_eq!(&d.rxbuf[100..], b"LONGSEGMENT"); + } + + #[test] + fn oversized_gap_resyncs_instead_of_stalling() { + decode::start_capture(); + let mut d = client_dir(); + d.feed(999, true, b""); // next_seq = 1000 + // A far-ahead segment larger than the buffering cap: the [1000, far) + // gap will never fill, so feed() must skip it and resync. + let big = vec![0xabu8; OOO_BYTES_CAP + 1]; + d.feed(5000, false, &big); + assert_eq!( + d.rxbuf.len(), + big.len(), + "resync should deliver the buffered segment" + ); + assert_eq!(d.ooo_bytes, 0); + assert_eq!(d.next_seq, Some(5000u32.wrapping_add(big.len() as u32))); + let out = decode::take_output_capture(); + assert!( + out.iter().any(|o| matches!(o, decode::Output::Message { message, .. } if message.kind == "Warning")), + "resync should emit an attributed warning" + ); + } + + fn noise() -> TcpSegment { + // Traffic on neither the client nor the monitored port: advances the + // table clock without creating a connection. + TcpSegment { + src: IpAddr::V4(Ipv4Addr::LOCALHOST), + dst: IpAddr::V4(Ipv4Addr::LOCALHOST), + src_port: 1234, + dst_port: 2345, + seq: 1, + ack: 0, + syn: false, + fin: false, + rst: false, + payload: vec![], + } + } + + #[test] + fn closed_connections_are_swept_after_the_grace_period() { + let metrics = Arc::new(Metrics::new()); + let mut table = ConnTable::with_metrics(metrics.clone()); + clean_close(&mut table); + assert_eq!(table.map.len(), 1, "tombstone retained right after close"); + + // Advance the clock well past the close grace, then sweep. + for _ in 0..(CLOSE_GRACE_SEGMENTS + 4) { + table.handle(&noise(), PG_PORT); + } + table.next_sweep = table.clock; + table.maybe_sweep(); + assert_eq!(table.map.len(), 0, "tombstone swept after grace"); + assert_eq!(metrics.snapshot().conns_live, 0); + } + + fn client_syn(src_port: u16) -> TcpSegment { + TcpSegment { + src: IpAddr::V4(Ipv4Addr::LOCALHOST), + dst: IpAddr::V4(Ipv4Addr::LOCALHOST), + src_port, + dst_port: PG_PORT, + seq: 1000, + ack: 0, + syn: true, + fin: false, + rst: false, + payload: vec![], + } + } + + #[test] + fn lru_eviction_bounds_live_connections() { + let metrics = Arc::new(Metrics::new()); + let mut table = ConnTable::with_metrics(metrics.clone()); + table.max_connections = 2; + // Four never-closed connections (missed FIN/RST); oldest two evicted. + for port in 40_001..=40_004 { + table.handle(&client_syn(port), PG_PORT); + } + assert_eq!(table.map.len(), 4); + table.next_sweep = table.clock; + table.maybe_sweep(); + assert_eq!(table.map.len(), 2, "capped to max_connections"); + // Evicting live connections closes them in the registry. + assert_eq!(metrics.snapshot().conns_live, 2); + } + + #[test] + fn both_endpoints_on_monitored_port_are_ignored() { + let metrics = Arc::new(Metrics::new()); + let mut table = ConnTable::with_metrics(metrics.clone()); + let mut seg = client_syn(40_010); + seg.src_port = PG_PORT; // both ends on PG_PORT + table.handle(&seg, PG_PORT); + assert_eq!(table.map.len(), 0); + assert_eq!(metrics.snapshot().conns_opened, 0); + } + + #[test] + fn bare_ack_on_unknown_tuple_creates_no_connection() { + let metrics = Arc::new(Metrics::new()); + let mut table = ConnTable::with_metrics(metrics.clone()); + // A stray payload-less ACK/RST arriving for a 4-tuple we never opened. + table.handle(&segment(true, false, false, &[]), PG_PORT); + assert_eq!(table.map.len(), 0); + assert_eq!(metrics.snapshot().conns_opened, 0); + } } diff --git a/src/lib.rs b/src/lib.rs index a5ce2c3..cb71959 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,5 +10,6 @@ pub mod filter; pub mod flow; pub mod net; pub mod proxy; +pub mod session; pub mod state; pub mod tui; diff --git a/src/main.rs b/src/main.rs index 0eef08c..0fce33a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -11,19 +11,24 @@ //! instead of the server; the proxy decrypts the client leg, decodes the //! traffic in the middle, and forwards it to the real server. See //! [`tapgres::proxy`]. +//! - `--replay FILE`: opens a versioned JSONL session instead of capturing and +//! feeds its decoded records through the same stdout or TUI renderer. //! //! Add `--tui` to either mode for an interactive, scrollable, filterable view //! instead of line-oriented stdout. See [`tapgres::tui`]. use std::error::Error; +use std::fs; use std::io::Write; +use std::io::{self}; +use std::path::Path; use std::path::PathBuf; use std::sync::Arc; use clap::Parser; use tapgres::cli::{Args, Mode}; -use tapgres::{capture, decode, filter::DisplayFilter, proxy, state, tui}; +use tapgres::{capture, decode, filter::DisplayFilter, proxy, session, state, tui}; fn main() -> Result<(), Box> { let args = Args::parse(); @@ -32,6 +37,18 @@ fn main() -> Result<(), Box> { args.conn_history, args.rate_history, )); + if let Some(replay) = args.replay { + if let Some(save) = &args.save { + ensure_distinct_files(&replay, save)?; + } + return if args.tui { + tui::run_replay(replay, metrics, args.tui_rich, filter, args.save) + } else { + run_stdout(filter, args.save, move || { + session::read_with(replay, decode::replay).map_err(Into::into) + }) + }; + } match args.mode { Mode::Pcap => { let opts = capture::PcapOpts { @@ -41,9 +58,9 @@ fn main() -> Result<(), Box> { snaplen: args.snaplen, }; if args.tui { - tui::run_pcap(opts, metrics, args.tui_rich, filter) + tui::run_pcap(opts, metrics, args.tui_rich, filter, args.save) } else { - run_stdout(filter, move || capture::run(opts, metrics)) + run_stdout(filter, args.save, move || capture::run(opts, metrics)) } } Mode::Mitm => { @@ -56,9 +73,9 @@ fn main() -> Result<(), Box> { no_upstream_tls: args.no_upstream_tls, }; if args.tui { - tui::run_mitm(opts, metrics, args.tui_rich, filter) + tui::run_mitm(opts, metrics, args.tui_rich, filter, args.save) } else { - run_stdout(filter, move || proxy::run(opts, metrics)) + run_stdout(filter, args.save, move || proxy::run(opts, metrics)) } } } @@ -67,18 +84,32 @@ fn main() -> Result<(), Box> { /// Run `source` with its decoded output funneled through a single consumer /// thread: decoded lines to stdout, status to stderr. When `source` returns, /// close the channel and join the consumer so nothing is left unflushed. -fn run_stdout(filter: DisplayFilter, source: F) -> Result<(), Box> +fn run_stdout( + filter: DisplayFilter, + save: Option, + source: F, +) -> Result<(), Box> where F: FnOnce() -> Result<(), Box>, { - let (tx, rx) = crossbeam_channel::unbounded(); + let (tx, rx) = decode::channel(); decode::set_output(tx); + let mut recorder = save.map(session::SessionWriter::create).transpose()?; let printer = std::thread::Builder::new() .name("tapgres-out".into()) - .spawn(move || { + .spawn(move || -> io::Result<()> { let mut stdout = std::io::stdout().lock(); let mut stderr = std::io::stderr().lock(); + let mut recorder_error = None; while let Ok(record) = rx.recv() { + if let Some(writer) = recorder.as_mut() { + if let Err(error) = writer.write(&record) { + let message = format!("tapgres: recording stopped: {error}"); + let _ = writeln!(stderr, "{message}"); + recorder_error = Some(io::Error::other(message)); + recorder = None; + } + } if !record.matches_filter(&filter) { continue; } @@ -94,13 +125,70 @@ where } } } + if let Some(writer) = recorder.as_mut() { + writer.flush().map_err(io::Error::other)?; + } let _ = stdout.flush(); let _ = stderr.flush(); + if let Some(error) = recorder_error { + return Err(error); + } + Ok(()) })?; let result = source(); decode::close_output(); - let _ = printer.join(); - result + let consumer_result = printer + .join() + .map_err(|_| io::Error::other("output consumer thread panicked"))?; + result?; + consumer_result?; + let dropped = decode::dropped_count(); + if dropped > 0 { + eprintln!("tapgres: dropped {dropped} output records (consumer could not keep up)"); + } + Ok(()) +} + +/// Prevent `--replay FILE --save FILE` from truncating the input before it is +/// read. Canonicalising the existing input and the output's parent also catches +/// equivalent relative paths and symlinked directories. +fn ensure_distinct_files(input: &Path, output: &Path) -> Result<(), Box> { + let input = fs::canonicalize(input)?; + let output_exists = output.exists(); + let output = if output_exists { + fs::canonicalize(output)? + } else { + let parent = output + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + let name = output + .file_name() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "invalid --save path"))?; + fs::canonicalize(parent)?.join(name) + }; + if input == output || (output_exists && files_share_identity(&input, &output)?) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "--save must not overwrite the --replay input file", + ) + .into()); + } + Ok(()) +} + +#[cfg(unix)] +fn files_share_identity(left: &Path, right: &Path) -> io::Result { + use std::os::unix::fs::MetadataExt; + + let left = fs::metadata(left)?; + let right = fs::metadata(right)?; + Ok(left.dev() == right.dev() && left.ino() == right.ino()) +} + +#[cfg(not(unix))] +fn files_share_identity(_left: &Path, _right: &Path) -> io::Result { + Ok(false) } /// Default on-disk location for the auto-generated CA + server cert. diff --git a/src/net.rs b/src/net.rs index 38f56fa..cdd930d 100644 --- a/src/net.rs +++ b/src/net.rs @@ -111,11 +111,21 @@ fn parse_ipv4(b: &[u8]) -> Option { if proto != 6 { return None; // not TCP } - let total_len = u16::from_be_bytes([b[2], b[3]]) as usize; - let end = total_len.min(b.len()); - if end < ihl { + // Non-first IP fragment: it carries no TCP header, so parsing its bytes as + // L4 would be garbage. (First fragments, offset 0, still hold the header.) + let frag_offset = u16::from_be_bytes([b[6], b[7]]) & 0x1fff; + if frag_offset != 0 { return None; } + let total_len = u16::from_be_bytes([b[2], b[3]]) as usize; + // Segmentation offload (TSO/GSO) hands us merged super-packets whose IP + // total_length is 0 (or, defensively, anything short of the header); fall + // back to the captured length so we don't silently drop the whole segment. + let end = if total_len >= ihl { + total_len.min(b.len()) + } else { + b.len() + }; let src = IpAddr::V4(Ipv4Addr::new(b[12], b[13], b[14], b[15])); let dst = IpAddr::V4(Ipv4Addr::new(b[16], b[17], b[18], b[19])); parse_tcp(&b[ihl..end], src, dst) @@ -135,6 +145,14 @@ fn parse_ipv6(b: &[u8]) -> Option { // Walk extension headers (best-effort). Stop at TCP, or bail on anything // we don't model (e.g. ESP). while is_extension_header(next_header) && off + 8 <= body_end { + // A non-first fragment (fragment header, offset != 0) carries no TCP + // header — drop it rather than parse its payload as L4. + if next_header == 44 { + let frag = u16::from_be_bytes([b[off + 2], b[off + 3]]); + if (frag >> 3) != 0 { + return None; + } + } let ext_next = b[off]; let ext_len = extension_header_len(next_header, &b[off..body_end])?; next_header = ext_next; @@ -205,3 +223,58 @@ fn parse_tcp(l4: &[u8], src: IpAddr, dst: IpAddr) -> Option { payload, }) } + +#[cfg(test)] +mod tests { + use super::*; + + /// Build a raw IPv4+TCP frame. `total_len_override` forces the IP + /// total-length field (use `Some(0)` to mimic segmentation offload); + /// `frag_offset` sets the IPv4 fragment offset. + fn ipv4_tcp(frag_offset: u16, total_len_override: Option, payload: &[u8]) -> Vec { + let total = 20 + 20 + payload.len(); + let total_len = total_len_override.unwrap_or(total as u16); + let mut f = vec![0u8; total]; + f[0] = 0x45; // version 4, ihl 5 + f[2..4].copy_from_slice(&total_len.to_be_bytes()); + f[6..8].copy_from_slice(&(frag_offset & 0x1fff).to_be_bytes()); + f[9] = 6; // TCP + f[12..16].copy_from_slice(&[127, 0, 0, 1]); + f[16..20].copy_from_slice(&[127, 0, 0, 1]); + // TCP header at offset 20 + f[20..22].copy_from_slice(&40_000u16.to_be_bytes()); // src port + f[22..24].copy_from_slice(&5432u16.to_be_bytes()); // dst port + f[24..28].copy_from_slice(&1000u32.to_be_bytes()); // seq + f[32] = 0x50; // data offset = 5 words + f[33] = 0x18; // PSH|ACK + f[40..].copy_from_slice(payload); + f + } + + #[test] + fn parses_a_well_formed_ipv4_tcp_segment() { + let seg = parse_frame(&ipv4_tcp(0, None, b"hello"), DLT_RAW).unwrap(); + assert_eq!(seg.src_port, 40_000); + assert_eq!(seg.dst_port, 5432); + assert_eq!(seg.seq, 1000); + assert_eq!(seg.payload, b"hello"); + } + + #[test] + fn zero_total_length_falls_back_to_captured_length() { + // TSO/GSO capture: total_length == 0 must not drop the payload. + let seg = parse_frame(&ipv4_tcp(0, Some(0), b"offloaded-bytes"), DLT_RAW).unwrap(); + assert_eq!(seg.payload, b"offloaded-bytes"); + } + + #[test] + fn non_first_fragment_is_dropped() { + // A fragment with non-zero offset has no TCP header of its own. + assert!(parse_frame(&ipv4_tcp(37, None, b"junk"), DLT_RAW).is_none()); + } + + #[test] + fn runt_frame_is_rejected() { + assert!(parse_frame(&[0x45, 0, 0], DLT_RAW).is_none()); + } +} diff --git a/src/proxy.rs b/src/proxy.rs index ac5bb91..6c3a5a6 100644 --- a/src/proxy.rs +++ b/src/proxy.rs @@ -117,7 +117,16 @@ pub async fn serve(opts: ProxyOpts, metrics: Arc) -> Result<(), Box pair, + // A transient per-connection error (EMFILE/ECONNABORTED burst) must + // not tear down the whole proxy; log, back off briefly, keep serving. + Err(e) => { + decode::status(format!("tapgres: accept error: {e}")); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + continue; + } + }; let opts = opts.clone(); let tls = tls.clone(); let metrics = metrics.clone(); @@ -145,33 +154,45 @@ async fn handle_connection( metrics: metrics.clone(), stats: stats.clone(), }; - // Read the first 8 bytes: 4-byte length + 4-byte magic/protocol. + // PG17+ direct SSL (`sslnegotiation=direct`) opens with a raw TLS + // ClientHello instead of an SSLRequest. Its first byte is the TLS handshake + // record type 0x16; every classic PostgreSQL opening message starts its + // Int32 length with 0x00, so one peeked byte disambiguates without + // consuming it (the TLS acceptor then reads the ClientHello itself). + let mut probe = [0u8; 1]; + let direct_tls = matches!(client.peek(&mut probe).await, Ok(n) if n >= 1 && probe[0] == 0x16); + + // Negotiate the client-facing transport. A client may send GssEncRequest + // (which we refuse) and then retry with SSLRequest or a cleartext Startup on + // the same connection, so loop until we settle on TLS or cleartext. let mut head = [0u8; 8]; - if client.read_exact(&mut head).await.is_err() { - return Ok(()); // client sent < 8 bytes (or nothing); nothing to tap - } - let body = &head[4..8]; - - // Cancel requests are one-shot, raw, and must reach the server verbatim on - // their own connection — relay them untouched, no TLS, no decoding. - if body == CANCEL_MAGIC { - let mut server = TcpStream::connect(opts.upstream.as_str()).await?; - server.write_all(&head).await?; - return raw_relay(client, server).await; - } - - // --- Decide the client-facing transport --- - let client_tls = body == SSL_MAGIC; - let initial: Vec = if client_tls { - client.write_all(b"S").await?; // accept SSL locally - Vec::new() - } else if body == GSS_MAGIC { - // We don't speak GSS; refuse so the client falls back to cleartext. - client.write_all(b"N").await?; - Vec::new() + let (client_tls, initial): (bool, Vec) = if direct_tls { + (true, Vec::new()) } else { - // Cleartext Startup (or anything else): these 8 bytes begin it. - head.to_vec() + loop { + if client.read_exact(&mut head).await.is_err() { + return Ok(()); // client sent < 8 bytes (or nothing); nothing to tap + } + let body = &head[4..8]; + if body == CANCEL_MAGIC { + // One-shot cancel: relay verbatim on its own connection, + // negotiating upstream TLS the same way a real client would so + // hostssl-only servers still accept it. + let server = TcpStream::connect(opts.upstream.as_str()).await?; + let mut server = upstream_transport(server, &opts, &tls).await?; + server.write_all(&head).await?; + return raw_relay(client, server).await; + } else if body == SSL_MAGIC { + client.write_all(b"S").await?; // accept SSL locally + break (true, Vec::new()); + } else if body == GSS_MAGIC { + client.write_all(b"N").await?; // we don't speak GSS; client retries + continue; + } else { + // Cleartext Startup (or anything else): these 8 bytes begin it. + break (false, head.to_vec()); + } + } }; // The cleartext Startup's first 8 bytes (read above to detect // SSL/GSS/cancel) are forwarded upstream below and also fed back into the @@ -186,65 +207,95 @@ async fn handle_connection( }; // --- Upstream transport --- - let mut server = TcpStream::connect(opts.upstream.as_str()).await?; - let server_stream: ProxyStream = if opts.no_upstream_tls { - ProxyStream::Plain(server) - } else { - server.write_all(&SSL_REQUEST).await?; // probe the server for TLS - let mut reply = [0u8; 1]; - match server.read_exact(&mut reply).await { - Ok(_) if reply[0] == b'S' => { - let connector = TlsConnector::from(tls.upstream_client_config.clone()); - let name = ServerName::try_from("localhost".to_string()) - .map_err(|e| io::Error::other(format!("invalid upstream name: {e}")))?; - let s = connector.connect(name, server).await?; - ProxyStream::Tls(Box::new(s.into())) - } - _ => ProxyStream::Plain(server), // 'N' or EOF: stay cleartext upstream - } - }; + let server = TcpStream::connect(opts.upstream.as_str()).await?; + let mut server_stream = upstream_transport(server, &opts, &tls).await?; // Forward the client's initial bytes (the Startup) upstream. - let mut server_stream = server_stream; if !initial.is_empty() { server_stream.write_all(&initial).await?; } - // Bidirectional decode + relay. + // Bidirectional decode + relay. Run both directions to completion with + // `join!` (not `select!`+abort) so a half-closed peer — a client that + // shuts down its write side and keeps reading results — isn't cut off + // mid-response. EOF on one leg shuts down the paired write half, which + // propagates the close naturally. let (client_rd, client_wr) = tokio::io::split(client_stream); let (server_rd, server_wr) = tokio::io::split(server_stream); - let mut to_client = tokio::spawn(pump( - server_rd, - client_wr, - Role::Server, - metrics.clone(), - stats.clone(), - Vec::new(), - )); - let mut to_server = tokio::spawn(pump( - client_rd, - server_wr, - Role::Client, - metrics.clone(), - stats.clone(), - initial, - )); - // Finish as soon as either side closes; abort the other half. - tokio::select! { - _ = &mut to_client => { - to_server.abort(); - let _ = to_server.await; - } - _ = &mut to_server => { - to_client.abort(); - let _ = to_client.await; + let (to_client, to_server) = tokio::join!( + pump( + server_rd, + client_wr, + Role::Server, + metrics.clone(), + stats.clone(), + Vec::new(), + ), + pump( + client_rd, + server_wr, + Role::Client, + metrics.clone(), + stats.clone(), + initial, + ), + ); + for result in [to_client, to_server] { + if let Err(e) = result { + decode::status(format!("tapgres: relay ended with error: {e}")); } } Ok(()) } -/// Copy bytes one way without decoding (used for cancel-request connections). -async fn raw_relay(client: TcpStream, server: TcpStream) -> io::Result<()> { +/// Establish the upstream transport, probing the server for TLS unless +/// `--no-upstream-tls` was given. Shared by the normal relay and the cancel +/// path so both negotiate identically. +async fn upstream_transport( + mut server: TcpStream, + opts: &ProxyOpts, + tls: &TlsMaterial, +) -> io::Result { + if opts.no_upstream_tls { + return Ok(ProxyStream::Plain(server)); + } + server.write_all(&SSL_REQUEST).await?; // probe the server for TLS + let mut reply = [0u8; 1]; + match server.read_exact(&mut reply).await { + Ok(_) if reply[0] == b'S' => { + let connector = TlsConnector::from(tls.upstream_client_config.clone()); + let s = connector + .connect(upstream_server_name(opts), server) + .await?; + Ok(ProxyStream::Tls(Box::new(s.into()))) + } + _ => Ok(ProxyStream::Plain(server)), // 'N' or EOF: stay cleartext upstream + } +} + +/// SNI to present to the upstream, derived from the configured host so +/// SNI-routing poolers see the right name. Certificate verification is disabled +/// (local server), so a fallback is harmless. +fn upstream_server_name(opts: &ProxyOpts) -> ServerName<'static> { + let host = opts + .upstream + .rsplit_once(':') + .map(|(h, _)| h) + .unwrap_or(opts.upstream.as_str()) + .trim_start_matches('[') + .trim_end_matches(']'); + ServerName::try_from(host.to_string()) + .unwrap_or_else(|_| ServerName::try_from("localhost".to_string()).unwrap()) +} + +/// Copy bytes both ways without decoding (used for cancel-request connections). +/// Generic over the stream types so a plain client can be relayed against a +/// possibly-TLS upstream. +async fn raw_relay(client: A, server: B) -> io::Result<()> +where + A: AsyncRead + AsyncWrite, + B: AsyncRead + AsyncWrite, +{ let (mut c_rd, mut c_wr) = tokio::io::split(client); let (mut s_rd, mut s_wr) = tokio::io::split(server); let _ = tokio::try_join!( @@ -376,9 +427,13 @@ fn materialize_tls(opts: &ProxyOpts) -> Result> { } }; - let server_config = ServerConfig::builder() + let mut server_config = ServerConfig::builder() .with_no_client_auth() .with_single_cert(certs, key)?; + // Advertise the PostgreSQL ALPN protocol so PG17+ direct-SSL clients + // (`sslnegotiation=direct`), which require ALPN, complete the handshake. + // Clients using the classic SSLRequest negotiation simply don't offer it. + server_config.alpn_protocols = vec![b"postgresql".to_vec()]; // The upstream leg talks to a local, user-controlled server, so we don't // verify its certificate — only that the handshake completes. @@ -548,4 +603,93 @@ mod tests { let _ = std::fs::remove_dir_all(&dir); } + + #[test] + fn upstream_server_name_parses_host_forms() { + let name = |upstream: &str| { + let opts = ProxyOpts { + listen: String::new(), + upstream: upstream.to_string(), + tls_dir: PathBuf::new(), + tls_cert: None, + tls_key: None, + no_upstream_tls: false, + }; + upstream_server_name(&opts) + }; + assert_eq!(name("db.example.com:5432"), name("db.example.com:5432")); + // Hostname form yields a DNS name; IPv6 brackets are stripped; both parse. + assert!(matches!( + name("db.example.com:5432"), + ServerName::DnsName(_) + )); + assert!(matches!(name("[::1]:5432"), ServerName::IpAddress(_))); + assert!(matches!(name("127.0.0.1:5432"), ServerName::IpAddress(_))); + } + + /// End-to-end cleartext path: a client's Startup and the upstream's + /// ReadyForQuery both traverse the proxy and are decoded (reflected in + /// metrics), and the upstream SSL probe is answered and relayed correctly. + #[tokio::test] + async fn cleartext_startup_relays_and_decodes_both_directions() { + // Fake upstream: refuse the SSL probe, read the Startup, reply RFQ. + let upstream = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let upstream_addr = upstream.local_addr().unwrap(); + tokio::spawn(async move { + let (mut s, _) = upstream.accept().await.unwrap(); + let mut probe = [0u8; 8]; + s.read_exact(&mut probe).await.unwrap(); + assert_eq!(probe, SSL_REQUEST, "proxy should probe upstream for TLS"); + s.write_all(b"N").await.unwrap(); // refuse -> cleartext upstream + // Startup: Int32 len, Int32 protocol(196608), "user\0tapgres\0\0". + let params = b"user\0tapgres\0\0"; + let total = 8 + params.len(); + let mut startup = Vec::new(); + startup.extend_from_slice(&(total as u32).to_be_bytes()); + startup.extend_from_slice(&196_608u32.to_be_bytes()); + startup.extend_from_slice(params); + let mut got = vec![0u8; total]; + s.read_exact(&mut got).await.unwrap(); + assert_eq!(got, startup, "full Startup should reach upstream"); + // ReadyForQuery: 'Z', len 5, status 'I'. + s.write_all(&[b'Z', 0, 0, 0, 5, b'I']).await.unwrap(); + }); + + let proxy = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let proxy_addr = proxy.local_addr().unwrap(); + let opts = Arc::new(ProxyOpts { + listen: proxy_addr.to_string(), + upstream: upstream_addr.to_string(), + tls_dir: unique_temp_dir(), + tls_cert: None, + tls_key: None, + no_upstream_tls: false, + }); + let tls = Arc::new(materialize_tls(&opts).unwrap()); + let metrics = Arc::new(Metrics::new()); + let m = metrics.clone(); + let handled = tokio::spawn(async move { + let (client, _) = proxy.accept().await.unwrap(); + let _ = handle_connection(client, opts, tls, m).await; + }); + + let mut client = TcpStream::connect(proxy_addr).await.unwrap(); + let params = b"user\0tapgres\0\0"; + let total = 8 + params.len(); + let mut startup = Vec::new(); + startup.extend_from_slice(&(total as u32).to_be_bytes()); + startup.extend_from_slice(&196_608u32.to_be_bytes()); + startup.extend_from_slice(params); + client.write_all(&startup).await.unwrap(); + let mut rfq = [0u8; 6]; + client.read_exact(&mut rfq).await.unwrap(); + assert_eq!(rfq, [b'Z', 0, 0, 0, 5, b'I'], "RFQ relayed to client"); + drop(client); // half-close; join! must still finish the other leg + handled.await.unwrap(); + + let snap = metrics.snapshot(); + assert!(snap.msgs_in >= 1, "client Startup should be decoded"); + assert!(snap.msgs_out >= 1, "server ReadyForQuery should be decoded"); + assert_eq!(snap.conns_live, 0, "connection guard should close it"); + } } diff --git a/src/session.rs b/src/session.rs new file mode 100644 index 0000000..76e240c --- /dev/null +++ b/src/session.rs @@ -0,0 +1,587 @@ +//! Versioned JSONL persistence for decoded tapgres sessions. +//! +//! Each line is an independent record with a schema version, capture +//! timestamp, record kind, and the structured fields needed by display +//! filters and rich TUI rendering. Replayed records are converted back into +//! [`crate::decode::Output`] so live and file sources share the same renderer. + +use std::collections::VecDeque; +use std::fmt; +use std::fs::File; +use std::io::{self, BufRead, BufReader, LineWriter, Write}; +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; + +use chrono::{Local, SecondsFormat}; +use serde::{Deserialize, Serialize}; + +use crate::decode::{DataColumn, EventDetail, FieldSummary, Output}; +use crate::filter::{DisplayMessage, MessageDirection}; + +/// Current on-disk JSONL schema. Readers refuse every other version so a +/// future incompatible shape cannot be silently misinterpreted. +pub const SCHEMA_VERSION: u32 = 1; + +#[derive(Debug)] +pub enum SessionError { + Io { + path: PathBuf, + source: io::Error, + }, + InvalidRecord { + path: PathBuf, + line: usize, + message: String, + }, + UnsupportedSchema { + path: PathBuf, + line: usize, + found: u32, + }, + Encode { + path: PathBuf, + source: serde_json::Error, + }, +} + +impl fmt::Display for SessionError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Io { path, source } => write!(f, "{}: {source}", path.display()), + Self::InvalidRecord { + path, + line, + message, + } => write!( + f, + "{}: invalid JSONL record at line {line}: {message}", + path.display() + ), + Self::UnsupportedSchema { path, line, found } => write!( + f, + "{}: unsupported schema version {found} at line {line} (supported: {SCHEMA_VERSION})", + path.display() + ), + Self::Encode { path, source } => { + write!( + f, + "{}: could not encode JSONL record: {source}", + path.display() + ) + } + } + } +} + +impl std::error::Error for SessionError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Io { source, .. } => Some(source), + Self::Encode { source, .. } => Some(source), + Self::InvalidRecord { .. } | Self::UnsupportedSchema { .. } => None, + } + } +} + +/// A streaming JSONL writer. It owns the file so a live consumer can record +/// every event before display filtering or in-memory history eviction. +pub struct SessionWriter { + path: PathBuf, + writer: LineWriter, +} + +impl SessionWriter { + pub fn create(path: impl AsRef) -> Result { + let path = path.as_ref().to_path_buf(); + let file = File::create(&path).map_err(|source| SessionError::Io { + path: path.clone(), + source, + })?; + Ok(Self { + path, + // Flush on every completed JSONL line. Recording is opt-in, and + // this prevents the final buffered events from disappearing when + // a long-running CLI capture is stopped with Ctrl-C. + writer: LineWriter::new(file), + }) + } + + pub fn path(&self) -> &Path { + &self.path + } + + pub fn write(&mut self, output: &Output) -> Result<(), SessionError> { + let record = StoredRecord::from_output(output); + serde_json::to_writer(&mut self.writer, &record).map_err(|source| { + SessionError::Encode { + path: self.path.clone(), + source, + } + })?; + self.writer + .write_all(b"\n") + .map_err(|source| SessionError::Io { + path: self.path.clone(), + source, + }) + } + + pub fn flush(&mut self) -> Result<(), SessionError> { + self.writer.flush().map_err(|source| SessionError::Io { + path: self.path.clone(), + source, + }) + } +} + +impl Drop for SessionWriter { + fn drop(&mut self) { + let _ = self.writer.flush(); + } +} + +/// Load a complete saved session. The TUI uses this for `:open`, where the +/// current retained view is replaced atomically only after all lines validate. +pub fn read_all(path: impl AsRef) -> Result, SessionError> { + let mut outputs = Vec::new(); + read_with(path, |output| outputs.push(output))?; + Ok(outputs) +} + +/// Validate a complete session while retaining only its newest `cap` records. +/// The TUI uses this to keep replay memory bounded without exposing a partial +/// file if a later line is malformed. +pub fn read_tail(path: impl AsRef, cap: usize) -> Result<(Vec, usize), SessionError> { + let mut outputs = VecDeque::with_capacity(cap.min(4_096)); + let mut dropped = 0usize; + read_with(path, |output| { + if cap == 0 { + dropped = dropped.saturating_add(1); + return; + } + if outputs.len() == cap { + outputs.pop_front(); + dropped = dropped.saturating_add(1); + } + outputs.push_back(output); + })?; + Ok((outputs.into(), dropped)) +} + +/// Stream a session into a consumer without first retaining the entire file. +/// This is the CLI replay path and keeps large transcripts bounded. +pub fn read_with( + path: impl AsRef, + mut consume: impl FnMut(Output), +) -> Result<(), SessionError> { + let path = path.as_ref().to_path_buf(); + let file = File::open(&path).map_err(|source| SessionError::Io { + path: path.clone(), + source, + })?; + for (index, line) in BufReader::new(file).lines().enumerate() { + let line_number = index + 1; + let line = line.map_err(|source| SessionError::Io { + path: path.clone(), + source, + })?; + if line.trim().is_empty() { + continue; + } + let value: serde_json::Value = + serde_json::from_str(&line).map_err(|source| SessionError::InvalidRecord { + path: path.clone(), + line: line_number, + message: source.to_string(), + })?; + let found = value + .get("schema_version") + .and_then(serde_json::Value::as_u64) + .and_then(|version| u32::try_from(version).ok()) + .ok_or_else(|| SessionError::InvalidRecord { + path: path.clone(), + line: line_number, + message: "schema_version must be an unsigned 32-bit integer".into(), + })?; + if found != SCHEMA_VERSION { + return Err(SessionError::UnsupportedSchema { + path, + line: line_number, + found, + }); + } + let record: StoredRecord = + serde_json::from_value(value).map_err(|source| SessionError::InvalidRecord { + path: path.clone(), + line: line_number, + message: source.to_string(), + })?; + consume(record.into_output(&path, line_number)?); + } + Ok(()) +} + +#[derive(Debug, Serialize, Deserialize)] +struct StoredRecord { + schema_version: u32, + timestamp: String, + #[serde(flatten)] + payload: StoredPayload, +} + +impl StoredRecord { + fn from_output(output: &Output) -> Self { + let (timestamp, payload) = match output { + Output::Message { message, detail } => ( + message.timestamp.clone(), + StoredPayload::Message { + direction: StoredDirection::from(message.direction), + message_type: message.kind.clone(), + text: message.text.clone(), + rendered: message.rendered.clone(), + client: message.client.to_string(), + detail: detail.as_ref().map(StoredDetail::from), + }, + ), + Output::Line(text) => (now_timestamp(), StoredPayload::Line { text: text.clone() }), + Output::Status(text) => ( + now_timestamp(), + StoredPayload::Status { text: text.clone() }, + ), + }; + Self { + schema_version: SCHEMA_VERSION, + timestamp, + payload, + } + } + + fn into_output(self, path: &Path, line: usize) -> Result { + chrono::DateTime::parse_from_rfc3339(&self.timestamp).map_err(|error| { + SessionError::InvalidRecord { + path: path.to_path_buf(), + line, + message: format!("timestamp must be RFC 3339: {error}"), + } + })?; + match self.payload { + StoredPayload::Message { + direction, + message_type, + text, + rendered, + client, + detail, + } => { + let client = + client + .parse::() + .map_err(|error| SessionError::InvalidRecord { + path: path.to_path_buf(), + line, + message: format!("invalid client address {client:?}: {error}"), + })?; + Ok(Output::Message { + message: DisplayMessage { + timestamp: self.timestamp, + rendered, + client, + direction: direction.into(), + kind: message_type, + text, + }, + detail: detail.map(Into::into), + }) + } + StoredPayload::Line { text } => Ok(Output::Line(text)), + StoredPayload::Status { text } => Ok(Output::Status(text)), + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(tag = "record_type", rename_all = "snake_case")] +enum StoredPayload { + Message { + direction: StoredDirection, + message_type: String, + text: String, + rendered: String, + client: String, + #[serde(skip_serializing_if = "Option::is_none")] + detail: Option, + }, + Line { + text: String, + }, + Status { + text: String, + }, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +enum StoredDirection { + F2b, + B2f, +} + +impl From for StoredDirection { + fn from(value: MessageDirection) -> Self { + match value { + MessageDirection::FrontendToBackend => Self::F2b, + MessageDirection::BackendToFrontend => Self::B2f, + } + } +} + +impl From for MessageDirection { + fn from(value: StoredDirection) -> Self { + match value { + StoredDirection::F2b => Self::FrontendToBackend, + StoredDirection::B2f => Self::BackendToFrontend, + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(tag = "detail_type", rename_all = "snake_case")] +enum StoredDetail { + RowDescription { columns: Vec }, + DataRow { columns: Vec }, +} + +impl From<&EventDetail> for StoredDetail { + fn from(value: &EventDetail) -> Self { + match value { + EventDetail::RowDescription(columns) => Self::RowDescription { + columns: columns.iter().map(StoredFieldSummary::from).collect(), + }, + EventDetail::DataRow(columns) => Self::DataRow { + columns: columns.iter().map(StoredDataColumn::from).collect(), + }, + } + } +} + +impl From for EventDetail { + fn from(value: StoredDetail) -> Self { + match value { + StoredDetail::RowDescription { columns } => { + Self::RowDescription(columns.into_iter().map(Into::into).collect()) + } + StoredDetail::DataRow { columns } => { + Self::DataRow(columns.into_iter().map(Into::into).collect()) + } + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +struct StoredFieldSummary { + name: String, + type_oid: u32, + format_code: i16, +} + +impl From<&FieldSummary> for StoredFieldSummary { + fn from(value: &FieldSummary) -> Self { + Self { + name: value.name.clone(), + type_oid: value.type_oid, + format_code: value.format_code, + } + } +} + +impl From for FieldSummary { + fn from(value: StoredFieldSummary) -> Self { + Self { + name: value.name, + type_oid: value.type_oid, + format_code: value.format_code, + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +struct StoredDataColumn { + name: String, + type_oid: u32, + value: String, +} + +impl From<&DataColumn> for StoredDataColumn { + fn from(value: &DataColumn) -> Self { + Self { + name: value.name.clone(), + type_oid: value.type_oid, + value: value.value.clone(), + } + } +} + +impl From for DataColumn { + fn from(value: StoredDataColumn) -> Self { + Self { + name: value.name, + type_oid: value.type_oid, + value: value.value, + } + } +} + +fn now_timestamp() -> String { + Local::now().to_rfc3339_opts(SecondsFormat::Millis, true) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + + fn message_output() -> Output { + Output::Message { + message: DisplayMessage { + timestamp: "2026-07-17T12:34:56.789+01:00".into(), + rendered: "[12:34:56.789] [B→F] DataRow: { id='1' }".into(), + client: "127.0.0.1:40005".parse().unwrap(), + direction: MessageDirection::BackendToFrontend, + kind: "DataRow".into(), + text: "{ id='1' }".into(), + }, + detail: Some(EventDetail::DataRow(vec![DataColumn { + name: "id".into(), + type_oid: 23, + value: "'1'".into(), + }])), + } + } + + #[test] + fn jsonl_round_trip_preserves_filter_and_rich_fields() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("capture.jsonl"); + let records = vec![ + Output::Status("capture active".into()), + message_output(), + Output::Line("=== connection closed ===".into()), + ]; + { + let mut writer = SessionWriter::create(&path).unwrap(); + for record in &records { + writer.write(record).unwrap(); + } + writer.flush().unwrap(); + } + + let raw = fs::read_to_string(&path).unwrap(); + assert_eq!(raw.lines().count(), 3); + assert!(raw.contains("\"schema_version\":1")); + assert!(raw.contains("\"record_type\":\"message\"")); + assert!(raw.contains("\"detail_type\":\"data_row\"")); + + let loaded = read_all(&path).unwrap(); + assert_eq!(loaded.len(), 3); + match &loaded[1] { + Output::Message { message, detail } => { + assert_eq!(message.timestamp, "2026-07-17T12:34:56.789+01:00"); + assert_eq!(message.client.port(), 40005); + assert_eq!(message.kind, "DataRow"); + match detail { + Some(EventDetail::DataRow(columns)) => { + assert_eq!(columns[0].name, "id"); + assert_eq!(columns[0].type_oid, 23); + assert_eq!(columns[0].value, "'1'"); + } + other => panic!("expected DataRow detail, got {other:?}"), + } + } + other => panic!("expected message, got {other:?}"), + } + } + + #[test] + fn refuses_unknown_schema_without_partial_tui_load() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("future.jsonl"); + fs::write( + &path, + r#"{"schema_version":99,"timestamp":"now","record_type":"line","text":"future"} +"#, + ) + .unwrap(); + + let error = read_all(&path).unwrap_err().to_string(); + assert!(error.contains("unsupported schema version 99")); + assert!(error.contains("line 1")); + } + + #[test] + fn reports_malformed_json_with_line_number() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("broken.jsonl"); + fs::write( + &path, + concat!( + "{\"schema_version\":1,\"timestamp\":\"2026-07-17T12:34:56.789+01:00\",\"record_type\":\"line\",\"text\":\"valid\"}\n", + "not json\n" + ), + ) + .unwrap(); + + let error = read_all(&path).unwrap_err().to_string(); + assert!(error.contains("line 2")); + } + + #[test] + fn rejects_non_rfc3339_timestamps() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("bad-time.jsonl"); + fs::write( + &path, + r#"{"schema_version":1,"timestamp":"12:34","record_type":"line","text":"bad"} +"#, + ) + .unwrap(); + + let error = read_all(&path).unwrap_err().to_string(); + assert!(error.contains("timestamp must be RFC 3339")); + assert!(error.contains("line 1")); + } + + #[test] + fn tail_reader_validates_all_records_while_bounding_memory() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("many.jsonl"); + let mut writer = SessionWriter::create(&path).unwrap(); + for index in 0..5 { + writer + .write(&Output::Line(format!("line {index}"))) + .unwrap(); + } + writer.flush().unwrap(); + + let (tail, dropped) = read_tail(&path, 2).unwrap(); + assert_eq!(dropped, 3); + assert_eq!(tail.len(), 2); + assert_eq!(tail[0].rendered(), "line 3"); + assert_eq!(tail[1].rendered(), "line 4"); + } + + #[test] + fn reads_the_committed_v1_compatibility_fixture() { + let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/session-v1.jsonl"); + let records = read_all(path).unwrap(); + + assert_eq!(records.len(), 4); + assert!(matches!( + records[2].detail(), + Some(EventDetail::RowDescription(columns)) if columns[0].name == "id" + )); + assert!(matches!( + records[3].detail(), + Some(EventDetail::DataRow(columns)) if columns[0].value == "'1'" + )); + } +} diff --git a/src/tui.rs b/src/tui.rs index c1da222..6cd1197 100644 --- a/src/tui.rs +++ b/src/tui.rs @@ -13,11 +13,14 @@ //! - `r` — toggle rich message rendering //! - `c` — clear //! - `y` — edit the display filter +//! - `/` — search the message text; `n`/`N` — next/previous match +//! - `:` — open the command bar (`:save FILE`, `:open FILE`) use crossbeam_channel::Receiver; use std::error::Error; use std::io; -use std::sync::Arc; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; use std::time::Duration; use ratatui::Frame; @@ -31,10 +34,16 @@ use crate::capture::PcapOpts; use crate::decode::{self, Output}; use crate::filter::DisplayFilter; use crate::proxy::ProxyOpts; -use crate::state::Metrics; +use crate::session::{self, SessionWriter}; +use crate::state::{Metrics, MetricsSummary}; /// Cap on retained lines in the TUI's own buffer. const HISTORY_CAP: usize = 50_000; +/// Trim in chunks during fast replay so a producer cannot grow the TUI buffer +/// without bound while avoiding an O(n) front-drain for every single record. +const HISTORY_TRIM_CHUNK: usize = 1_024; +/// How many recent status/warning lines to retain for the startup splash. +const STATUS_TAIL_CAP: usize = 8; /// tapgres ASCII-art banner. Shown by the CLI (`--help` via `before_help`) and /// as the heading of the TUI startup splash. @@ -53,20 +62,21 @@ pub fn run_pcap( metrics: Arc, rich: bool, filter: DisplayFilter, + save: Option, ) -> Result<(), Box> { let splash_lines = pcap_splash_lines(&opts); let source_metrics = metrics.clone(); run( Box::new(move || { - if let Err(e) = crate::capture::run(opts, source_metrics) { - decode::status(format!("⚠ pcap source error: {e}")); - } + crate::capture::run(opts, source_metrics).map_err(|e| format!("pcap source error: {e}")) }), "pcap", metrics, rich, filter, splash_lines, + save, + None, ) } @@ -90,30 +100,59 @@ pub fn run_mitm( metrics: Arc, rich: bool, filter: DisplayFilter, + save: Option, ) -> Result<(), Box> { let splash_lines = mitm_splash_lines(&opts); let source_metrics = metrics.clone(); run( Box::new(move || { - let rt = match tokio::runtime::Builder::new_multi_thread() + let rt = tokio::runtime::Builder::new_multi_thread() .enable_all() .build() - { - Ok(rt) => rt, - Err(e) => { - decode::status(format!("⚠ failed to start mitm runtime: {e}")); - return; - } - }; - if let Err(e) = rt.block_on(crate::proxy::serve(opts, source_metrics)) { - decode::status(format!("⚠ mitm source error: {e}")); - } + .map_err(|e| format!("failed to start mitm runtime: {e}"))?; + rt.block_on(crate::proxy::serve(opts, source_metrics)) + .map_err(|e| format!("mitm source error: {e}")) }), "mitm", metrics, rich, filter, splash_lines, + save, + None, + ) +} + +/// TUI over a saved JSONL session. Records are loaded at full speed through +/// the same channel and rendering path as live capture. +pub fn run_replay( + path: PathBuf, + metrics: Arc, + rich: bool, + filter: DisplayFilter, + save: Option, +) -> Result<(), Box> { + let source_path = path.clone(); + run( + Box::new(move || { + let (outputs, dropped) = session::read_tail(&source_path, HISTORY_CAP) + .map_err(|error| format!("replay source error: {error}"))?; + if dropped > 0 { + decode::status(format!( + "replay: showing the newest {HISTORY_CAP} records; {dropped} earlier records are outside TUI history" + )); + } + outputs.into_iter().for_each(decode::replay); + decode::close_output(); + Ok(()) + }), + "replay", + metrics, + rich, + filter, + Vec::new(), + save, + Some(path), ) } @@ -138,32 +177,62 @@ fn mitm_splash_lines(opts: &ProxyOpts) -> Vec { ] } +/// The traffic source thread's body. Returns a human-readable error string on a +/// fatal source failure (capture permission, bind failure, replay read error). +type Source = Box Result<(), String> + Send + 'static>; + /// Install a shared sink, start `source` in a background thread, run the TUI on /// this (main) thread, and always restore the terminal before returning. +#[allow(clippy::too_many_arguments)] fn run( - source: Box, + source: Source, mode: &'static str, metrics: Arc, rich: bool, filter: DisplayFilter, splash_lines: Vec, + save: Option, + replay_source: Option, ) -> Result<(), Box> { // One channel: the source (background thread) produces via decode::out, - // the TUI (this thread) consumes. - let (tx, rx) = crossbeam_channel::unbounded(); + // the TUI (this thread) consumes. Bounded so a stalled UI sheds records + // rather than growing memory without limit (see decode::channel). + let (tx, rx) = decode::channel(); decode::set_output(tx); - // The source runs until the process exits; no graceful shutdown here. + // Validate/create the save destination before starting a live source. + let recorder = save.map(SessionWriter::create).transpose()?; + + // Shared slot the source thread writes a fatal error into, so the TUI can + // surface it (instead of "waiting for traffic…" forever) and exit non-zero. + let source_error: Arc>> = Arc::new(Mutex::new(None)); + let slot = source_error.clone(); + // The source runs until the process exits; no graceful shutdown here. Its + // body is caught so a panic in the capture/decode path becomes a visible + // status line rather than firing ratatui's restore hook on this background + // thread while the main thread keeps drawing. let _source_thread = std::thread::Builder::new() .name("tapgres-source".into()) - .spawn(source)?; + .spawn(move || { + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(source)); + let error = match outcome { + Ok(Ok(())) => None, + Ok(Err(e)) => Some(e), + Err(_) => Some("capture/decode thread panicked".to_string()), + }; + if let Some(e) = error { + *slot.lock().unwrap() = Some(e.clone()); + decode::status(format!("⚠ {e}")); + } + })?; let _rate_sampler = metrics.spawn_rate_sampler()?; + let mut app = App::new(rx, mode, metrics, rich, filter, splash_lines); + app.recorder = recorder; + app.source_error = source_error; + app.replay_source = replay_source; let mut terminal = ratatui::try_init()?; - let result = app_loop( - &mut terminal, - App::new(rx, mode, metrics, rich, filter, splash_lines), - ); + let result = app_loop(&mut terminal, app); // Restore the terminal even on error. try_init installs a panic hook that // also restores, so panics are covered too. let _ = ratatui::try_restore(); @@ -184,7 +253,7 @@ struct App { events: Vec, /// Indices into `events` that match the current display filter. visible: Vec, - /// Index of the top visible line into the event buffer. + /// Index into `visible` of the topmost shown row (the scroll anchor). scroll: usize, /// Auto-tail new output. follow: bool, @@ -194,7 +263,7 @@ struct App { /// `RowDescription` as a typed column list, instead of the flat line. Type /// names are shown with an icon-font (Nerd Font) glyph. rich: bool, - mode: &'static str, + mode: String, metrics: Arc, /// All-time peak messages/sec seen this session, per direction. Used as a /// fixed sparkline scale so bars don't rescale as the rate window slides; @@ -205,11 +274,38 @@ struct App { filter_text: String, filter_error: Option, filter_editing: bool, + /// Applied filter text captured when the editor opens, so Esc can cancel an + /// in-progress edit and restore it instead of wiping the filter. + filter_snapshot: String, + /// Incremental text search over the message view (`/`). Non-empty while a + /// search is active; `n`/`N` jump between matches and matches are highlighted. + search_editing: bool, + search_text: String, + command_editing: bool, + command_text: String, + command_notice: Option<(String, bool)>, + /// Recent status/warning lines, surfaced on the startup splash so a fatal + /// source error (e.g. missing capture privileges) is visible immediately. + status_tail: Vec, + /// Set by the source thread on a fatal failure; drives a non-zero exit and + /// leaves the splash so the error is shown. + source_error: Arc>>, + /// Path of the file being replayed, if any, so `:save` refuses to overwrite + /// the input the source thread is still reading. + replay_source: Option, + /// False after `:open`: the loaded replay replaces the live view for the + /// rest of this TUI session, while the source channel is drained safely. + accept_source_records: bool, + /// Number of events removed by the bounded TUI history before `:save`. + dropped_events: usize, /// Mode-specific connection/capture info lines for the startup splash. splash_lines: Vec, /// Whether the startup splash is still showing. Flips off once a real /// connection is detected (see `app_loop`). show_splash: bool, + /// Optional continuous JSONL recorder. It receives records before the TUI + /// history cap or display filter can hide them. + recorder: Option, } impl App { @@ -232,7 +328,7 @@ impl App { follow: true, wrap: false, rich, - mode, + mode: mode.to_string(), metrics, peak_msgs_in: 0, peak_msgs_out: 0, @@ -240,8 +336,20 @@ impl App { filter_text, filter_error: None, filter_editing: false, + filter_snapshot: String::new(), + search_editing: false, + search_text: String::new(), + command_editing: false, + command_text: String::new(), + command_notice: None, + status_tail: Vec::new(), + source_error: Arc::new(Mutex::new(None)), + replay_source: None, + accept_source_records: true, + dropped_events: 0, splash_lines, show_splash, + recorder: None, } } @@ -250,11 +358,50 @@ impl App { } fn push_output(&mut self, output: Output) { + let write_error = self + .recorder + .as_mut() + .and_then(|recorder| recorder.write(&output).err()); + if let Some(error) = write_error { + self.recorder = None; + self.command_notice = Some((format!("recording stopped: {error}"), true)); + } + // Keep the tail of status/warning lines so the splash can show them. + if let Output::Status(line) = &output { + self.status_tail.push(line.clone()); + let len = self.status_tail.len(); + if len > STATUS_TAIL_CAP { + self.status_tail.drain(..len - STATUS_TAIL_CAP); + } + } let index = self.events.len(); if self.matches(&output) { self.visible.push(index); } self.events.push(output); + if self.events.len() >= HISTORY_CAP + HISTORY_TRIM_CHUNK { + self.trim_history(); + } + } + + /// Evict the oldest events back down to the cap. Rebases the visible indices + /// and the scroll anchor onto the shifted buffer instead of rebuilding from + /// scratch, so scrollback doesn't jump to the top at steady state (and no + /// O(n) re-filter runs on every trim). + fn trim_history(&mut self) { + if self.events.len() <= HISTORY_CAP { + return; + } + let drop_n = self.events.len() - HISTORY_CAP; + self.events.drain(..drop_n); + self.dropped_events = self.dropped_events.saturating_add(drop_n); + // `visible` is ascending, so evicted entries are a prefix. + let removed_visible = self.visible.iter().take_while(|&&i| i < drop_n).count(); + self.visible.retain(|&i| i >= drop_n); + for index in &mut self.visible { + *index -= drop_n; + } + self.scroll = self.scroll.saturating_sub(removed_visible); } fn rebuild_visible(&mut self) { @@ -291,30 +438,234 @@ impl App { self.rebuild_visible(); } - /// Leave the splash once a real connection has been detected. Startup - /// status lines (capture/proxy banner text) arrive before any traffic but - /// do not open a connection, so they do not trigger this transition. - fn leave_splash_if_traffic(&mut self) { - if self.show_splash && self.metrics.summary().conns_opened > 0 { + /// Parse the in-progress filter text for error feedback only, without + /// re-running it over the whole history — that (expensive) reapplication + /// happens on Enter via [`App::update_filter`]. Keeps the editor responsive + /// on a full buffer. + fn parse_filter_preview(&mut self) { + self.filter_error = if self.filter_text.trim().is_empty() { + None + } else { + DisplayFilter::parse(&self.filter_text) + .err() + .map(|error| error.to_string()) + }; + } + + /// Positions into `visible` whose rendered text contains the search term + /// (case-insensitive). Empty when no search is active. + fn search_matches(&self) -> Vec { + if self.search_text.is_empty() { + return Vec::new(); + } + let needle = self.search_text.to_lowercase(); + self.visible + .iter() + .enumerate() + .filter(|(_, event_index)| { + self.events[**event_index] + .rendered() + .to_lowercase() + .contains(&needle) + }) + .map(|(position, _)| position) + .collect() + } + + /// Jump to the next (`forward`) or previous match relative to the current + /// scroll anchor, wrapping around. `include_current` lets the initial jump + /// land on a match already at the anchor. + fn jump_to_match(&mut self, forward: bool, include_current: bool) { + let matches = self.search_matches(); + let Some(&first) = matches.first() else { + return; + }; + self.follow = false; + let anchor = self.scroll; + let target = if forward { + matches + .iter() + .copied() + .find(|&p| { + if include_current { + p >= anchor + } else { + p > anchor + } + }) + .unwrap_or(first) + } else { + matches + .iter() + .rev() + .copied() + .find(|&p| p < anchor) + .unwrap_or_else(|| *matches.last().unwrap()) + }; + self.scroll = target; + } + + fn execute_command(&mut self) { + let command_text = self.command_text.clone(); + let input = command_text.trim().trim_start_matches([':', '/']).trim(); + let (name, argument) = input + .split_once(char::is_whitespace) + .map(|(name, argument)| (name, argument.trim())) + .unwrap_or((input, "")); + + let result = match name { + "w" | "write" | "save" => self.start_recording(argument), + "o" | "open" => self.open_session(argument), + "" => Err("command is empty".to_string()), + _ => Err(format!("unknown command: {name}")), + }; + self.command_notice = Some(match result { + Ok(message) => (message, false), + Err(message) => (message, true), + }); + self.command_editing = false; + } + + fn start_recording(&mut self, argument: &str) -> Result { + if argument.is_empty() { + return Err("usage: :save FILE".into()); + } + let path = expand_tilde(argument); + // Never truncate the file the replay source thread is still reading. + if let Some(source) = &self.replay_source { + if same_path(source, &path) { + return Err("refusing to overwrite the replayed input file".into()); + } + } + let mut recorder = SessionWriter::create(&path).map_err(|error| error.to_string())?; + for output in &self.events { + recorder.write(output).map_err(|error| error.to_string())?; + } + recorder.flush().map_err(|error| error.to_string())?; + let retained = self.events.len(); + self.recorder = Some(recorder); + let action = if self.accept_source_records { + format!("recording {retained} retained events + future traffic") + } else { + format!("saved {retained} retained replay events") + }; + if self.dropped_events == 0 { + Ok(format!("{action} to {}", path.display())) + } else { + Ok(format!( + "{action} to {}; {} earlier events were outside history", + path.display(), + self.dropped_events + )) + } + } + + fn open_session(&mut self, argument: &str) -> Result { + if argument.is_empty() { + return Err("usage: :open FILE".into()); + } + let path = expand_tilde(argument); + // Flush the active recorder before reading so a concurrent `:save` to the + // same file is on disk, not truncated out from under this read. Keep it + // until the read succeeds so a failed `:open` doesn't stop recording. + if let Some(recorder) = self.recorder.as_mut() { + recorder.flush().map_err(|error| error.to_string())?; + } + let (outputs, dropped) = + session::read_tail(&path, HISTORY_CAP).map_err(|error| error.to_string())?; + let count = outputs.len(); + // Switching to replay: stop recording live traffic. + self.recorder = None; + self.events = outputs; + self.visible.clear(); + self.rebuild_visible(); + self.follow = true; + self.mode = "replay".into(); + self.metrics = Arc::new(Metrics::new()); + self.peak_msgs_in = 0; + self.peak_msgs_out = 0; + self.show_splash = false; + self.accept_source_records = false; + self.dropped_events = dropped; + if dropped == 0 { + Ok(format!("opened {count} events from {}", path.display())) + } else { + Ok(format!( + "opened newest {count} events from {}; {dropped} earlier events are outside history", + path.display() + )) + } + } + + /// Leave the splash once a real connection has been detected, or a fatal + /// source error has arrived (so it isn't hidden behind "waiting for + /// traffic…"). Startup status lines arrive before any traffic but do not + /// open a connection, so they alone do not trigger this transition. + fn leave_splash_if_traffic(&mut self, conns_opened: u64) { + if self.show_splash && (conns_opened > 0 || self.source_failed()) { self.show_splash = false; } } + + /// Whether the source thread reported a fatal error. + fn source_failed(&self) -> bool { + self.source_error.lock().unwrap().is_some() + } +} + +/// Expand a leading `~/` (or bare `~`) to the user's home directory so `:save` +/// / `:open` accept the paths users naturally type. +fn expand_tilde(input: &str) -> PathBuf { + if let Some(rest) = input.strip_prefix("~/") { + if let Some(home) = std::env::var_os("HOME") { + return Path::new(&home).join(rest); + } + } else if input == "~" { + if let Some(home) = std::env::var_os("HOME") { + return PathBuf::from(home); + } + } + PathBuf::from(input) +} + +/// Whether two paths point at the same file, tolerant of one not yet existing +/// (compares canonicalized parents so a not-yet-created `:save` target is still +/// caught against the replay input). +fn same_path(a: &Path, b: &Path) -> bool { + fn resolve(p: &Path) -> Option { + if let Ok(canonical) = std::fs::canonicalize(p) { + return Some(canonical); + } + let parent = p.parent().filter(|parent| !parent.as_os_str().is_empty()); + let name = p.file_name()?; + Some( + std::fs::canonicalize(parent.unwrap_or_else(|| Path::new("."))) + .ok()? + .join(name), + ) + } + match (resolve(a), resolve(b)) { + (Some(a), Some(b)) => a == b, + _ => a == b, + } } fn app_loop(terminal: &mut ratatui::DefaultTerminal, mut app: App) -> io::Result<()> { loop { while let Ok(record) = app.rx.try_recv() { - app.push_output(record); - } - // Leave the splash once a real connection is detected. Startup status - // lines (capture/proxy banner text) arrive before any traffic but do - // not open a connection, so they don't trigger this transition. - app.leave_splash_if_traffic(); - if app.events.len() > HISTORY_CAP { - let drop_n = app.events.len() - HISTORY_CAP; - app.events.drain(..drop_n); - app.rebuild_visible(); + if app.accept_source_records { + app.push_output(record); + } } + // One metrics snapshot per frame, shared by the splash transition, the + // peak-tracking below, and the header render — instead of three clones. + let summary = app.metrics.summary(); + // Leave the splash once a real connection is detected (or the source + // failed). Startup status lines arrive before any traffic but do not + // open a connection, so they don't trigger this transition. + app.leave_splash_if_traffic(summary.conns_opened); + // History trimming happens in push_output as records arrive; no need to + // re-trim (and re-filter) every frame here. // 5 (metrics) + 3 (footer) + 2 (log block borders) rows of chrome. let term_h = terminal.size()?.height as usize; @@ -341,21 +692,24 @@ fn app_loop(terminal: &mut ratatui::DefaultTerminal, mut app: App) -> io::Result // Fixed sparkline scale: track the all-time peak messages/sec per // direction so the bars keep a stable scale instead of rescaling to // the current window's max as samples expire or arrive. - { - let summary = app.metrics.summary(); - app.peak_msgs_in = app - .peak_msgs_in - .max(summary.rates.iter().map(|r| r.msgs_in).max().unwrap_or(0)); - app.peak_msgs_out = app - .peak_msgs_out - .max(summary.rates.iter().map(|r| r.msgs_out).max().unwrap_or(0)); - } + app.peak_msgs_in = app + .peak_msgs_in + .max(summary.rates.iter().map(|r| r.msgs_in).max().unwrap_or(0)); + app.peak_msgs_out = app + .peak_msgs_out + .max(summary.rates.iter().map(|r| r.msgs_out).max().unwrap_or(0)); terminal.draw(|frame| { if app.show_splash { draw_splash(frame, &app); + if app.command_editing { + let [_, command_area] = + Layout::vertical([Constraint::Fill(1), Constraint::Length(3)]) + .areas(frame.area()); + draw_command_bar(frame, &app, command_area); + } } else { - draw(frame, &app, log_h); + draw(frame, &app, log_h, &summary); } })?; @@ -366,7 +720,13 @@ fn app_loop(terminal: &mut ratatui::DefaultTerminal, mut app: App) -> io::Result Event::Key(key) if key.kind == KeyEventKind::Press && handle_key(&mut app, log_h, key) => { - return Ok(()); + // Propagate a fatal source error as a non-zero exit, so + // `--tui` matches the documented exit status of the + // line-oriented path. + return match app.source_error.lock().unwrap().take() { + Some(e) => Err(io::Error::other(e)), + None => Ok(()), + }; } // Resize / focus / mouse etc. — just trigger a redraw next loop. _ => {} @@ -382,33 +742,102 @@ fn app_loop(terminal: &mut ratatui::DefaultTerminal, mut app: App) -> io::Result /// Handle one key. Returns `true` to quit. fn handle_key(app: &mut App, log_h: usize, key: KeyEvent) -> bool { let ctrl = key.modifiers.contains(KeyModifiers::CONTROL); - if key.code == KeyCode::Char('c') && ctrl { + // Ctrl-C quits — but while editing an input it cancels that input instead of + // killing the whole app (handled inside each editing branch below). + let editing = app.command_editing || app.filter_editing || app.search_editing; + if key.code == KeyCode::Char('c') && ctrl && !editing { return true; } - // On the splash screen only honour quit; everything else is ignored until - // traffic arrives and the main view takes over. + if app.command_editing { + match key.code { + KeyCode::Esc => { + app.command_editing = false; + app.command_text.clear(); + } + KeyCode::Char('c') if ctrl => { + app.command_editing = false; + app.command_text.clear(); + } + KeyCode::Enter => app.execute_command(), + KeyCode::Backspace => { + app.command_text.pop(); + } + KeyCode::Char(ch) if !ctrl => app.command_text.push(ch), + _ => {} + } + return false; + } + if app.search_editing { + match key.code { + // Cancel the search entirely (Esc or Ctrl-C). + KeyCode::Esc => { + app.search_editing = false; + app.search_text.clear(); + } + KeyCode::Char('c') if ctrl => { + app.search_editing = false; + app.search_text.clear(); + } + KeyCode::Enter => { + app.search_editing = false; + app.jump_to_match(true, true); + } + KeyCode::Backspace => { + app.search_text.pop(); + } + KeyCode::Char(ch) if !ctrl => app.search_text.push(ch), + _ => {} + } + return false; + } + // Keep the command bar available before the first connection so a saved + // session can be opened directly from the startup splash. if app.show_splash { - return key.code == KeyCode::Char('q'); + return match key.code { + KeyCode::Char('q') => true, + KeyCode::Char('/') | KeyCode::Char(':') => { + app.command_editing = true; + app.command_text.clear(); + app.command_notice = None; + false + } + _ => false, + }; } if app.filter_editing { match key.code { + // Cancel the edit and restore the filter that was applied when the + // editor opened, rather than wiping it. KeyCode::Esc => { - app.clear_filter(); + app.filter_text = std::mem::take(&mut app.filter_snapshot); + app.filter_error = None; + app.filter_editing = false; + } + KeyCode::Char('c') if ctrl => { + app.filter_text = std::mem::take(&mut app.filter_snapshot); + app.filter_error = None; + app.filter_editing = false; + } + // Apply on Enter (the expensive reapplication), not per keystroke. + KeyCode::Enter if app.filter_error.is_none() => { + app.update_filter(); app.filter_editing = false; } - KeyCode::Enter if app.filter_error.is_none() => app.filter_editing = false, KeyCode::Backspace => { app.filter_text.pop(); - app.update_filter(); + app.parse_filter_preview(); } KeyCode::Char(ch) if !ctrl => { app.filter_text.push(ch); - app.update_filter(); + app.parse_filter_preview(); } _ => {} } return false; } + // A key in normal mode dismisses a lingering command notice so the footer + // key hints return. + app.command_notice = None; match key.code { KeyCode::Char('q') => return true, KeyCode::Char('j') | KeyCode::Down => { @@ -435,11 +864,31 @@ fn handle_key(app: &mut App, log_h: usize, key: KeyEvent) -> bool { KeyCode::Char('f') => app.follow = !app.follow, KeyCode::Char('w') => app.wrap = !app.wrap, KeyCode::Char('r') => app.rich = !app.rich, + KeyCode::Char('n') => app.jump_to_match(true, false), + KeyCode::Char('N') => app.jump_to_match(false, false), KeyCode::Char('c') => { + // Cleared events can no longer be saved; count them so `:save`'s + // omission note stays accurate. + app.dropped_events = app.dropped_events.saturating_add(app.events.len()); app.events.clear(); app.visible.clear(); + app.scroll = 0; + } + KeyCode::Char('y') => { + app.filter_snapshot = app.filter_text.clone(); + app.filter_editing = true; + } + KeyCode::Char('/') => { + app.search_editing = true; + app.search_text.clear(); } - KeyCode::Char('y') => app.filter_editing = true, + KeyCode::Char(':') => { + app.command_editing = true; + app.command_text.clear(); + app.command_notice = None; + } + // Esc clears an active search first, then the display filter. + KeyCode::Esc if !app.search_text.is_empty() => app.search_text.clear(), KeyCode::Esc if !app.filter.is_empty() => app.clear_filter(), _ => {} } @@ -465,8 +914,33 @@ fn draw_splash(frame: &mut Frame, app: &App) { for line in &app.splash_lines { lines.push(Line::raw(format!(" {line}"))); } + // Surface recent status/warning lines so a fatal source error (e.g. missing + // capture privileges) is visible instead of an endless "waiting…". + if !app.status_tail.is_empty() { + lines.push(Line::raw("")); + for status in &app.status_tail { + let style = if status.contains('⚠') { + Style::default().fg(Color::Red) + } else { + dim + }; + lines.push(Line::styled(format!(" {status}"), style)); + } + } + if let Some((notice, is_error)) = &app.command_notice { + lines.push(Line::raw("")); + lines.push(Line::styled( + format!(" {notice}"), + Style::default().fg(if *is_error { Color::Red } else { Color::Green }), + )); + } lines.push(Line::raw("")); - lines.push(Line::styled(" waiting for traffic… press q to quit", dim)); + let waiting = if app.source_failed() { + " source failed — see above · press q to quit" + } else { + " waiting for traffic… press : for commands · q to quit" + }; + lines.push(Line::styled(waiting, dim)); // Vertically centre the splash block; horizontally centre each line. let height = lines.len() as u16; @@ -482,7 +956,7 @@ fn draw_splash(frame: &mut Frame, app: &App) { ); } -fn draw(frame: &mut Frame, app: &App, log_h: usize) { +fn draw(frame: &mut Frame, app: &App, log_h: usize, metrics: &MetricsSummary) { let [title_area, log_area, foot_area] = Layout::vertical([ Constraint::Length(5), Constraint::Fill(1), @@ -505,7 +979,6 @@ fn draw(frame: &mut Frame, app: &App, log_h: usize) { app.events.len() ) }; - let metrics = app.metrics.summary(); let current = metrics.rates.last().copied().unwrap_or_default(); // Messages-per-second series over the rate window. In is cyan and out is @@ -657,9 +1130,22 @@ fn draw(frame: &mut Frame, app: &App, log_h: usize) { let s = app.scroll; (s, (s + log_h).min(app.visible.len())) }; + let needle = app.search_text.to_lowercase(); let mut lines: Vec = Vec::new(); for &event_index in &app.visible[start..end] { - lines.extend(event_lines(&app.events[event_index], view)); + let mut event_lines = event_lines(&app.events[event_index], view); + // Highlight lines of events matching the active search. + if !needle.is_empty() + && app.events[event_index] + .rendered() + .to_lowercase() + .contains(&needle) + { + for line in &mut event_lines { + line.style = line.style.bg(Color::Rgb(80, 70, 0)); + } + } + lines.extend(event_lines); } let mut para = Paragraph::new(Text::from(lines)).block(log_block); if app.wrap { @@ -670,7 +1156,18 @@ fn draw(frame: &mut Frame, app: &App, log_h: usize) { // --- footer: follow/wrap/rich state shown by colour (green = on) --- let on = Style::default().fg(Color::Green); let off = Style::default(); - if app.filter_editing { + if app.command_editing { + draw_command_bar(frame, app, foot_area); + } else if app.search_editing { + frame.render_widget( + Paragraph::new(Line::from(vec![Span::styled( + format!(" search › {}█", app.search_text), + Style::default().fg(Color::Yellow), + )])) + .block(Block::bordered().title_top(" search · Enter next · Esc cancel ")), + foot_area, + ); + } else if app.filter_editing { let style = if app.filter_error.is_some() { Style::default().fg(Color::Red) } else { @@ -686,11 +1183,34 @@ fn draw(frame: &mut Frame, app: &App, log_h: usize) { Span::styled(format!(" display filter › {}█", app.filter_text), style), Span::styled(detail, Style::default().fg(Color::Red)), ])) - .block(Block::bordered().title_top(" display filter · Enter done · Esc clear ")), + .block(Block::bordered().title_top(" display filter · Enter apply · Esc cancel ")), + foot_area, + ); + } else if !app.search_text.is_empty() { + // An active search: show the term and match count with n/N hint. + let count = app.search_matches().len(); + frame.render_widget( + Paragraph::new(Line::from(vec![ + Span::styled( + format!(" search: {} ", app.search_text), + Style::default().fg(Color::Yellow).bold(), + ), + Span::raw(format!("· {count} matches · n/N next/prev · Esc clear")), + ])) + .block(Block::bordered()), + foot_area, + ); + } else if let Some((notice, is_error)) = &app.command_notice { + frame.render_widget( + Paragraph::new(Line::styled( + format!(" {notice}"), + Style::default().fg(if *is_error { Color::Red } else { Color::Green }), + )) + .block(Block::bordered().title_top(" status · : command ")), foot_area, ); } else { - let footer = Line::from(vec![ + let footer = vec![ Span::raw(" q quit · j/k ↑↓ · PgUp/PgDn · g/G top/bottom · f "), Span::styled("follow", if app.follow { on } else { off }), Span::raw(" · w "), @@ -702,12 +1222,27 @@ fn draw(frame: &mut Frame, app: &App, log_h: usize) { "display filter", if app.filter.is_empty() { off } else { on }, ), - Span::raw(" · c clear "), - ]); - frame.render_widget(Paragraph::new(footer).block(Block::bordered()), foot_area); + Span::raw(" · / search · : command · c clear "), + ]; + frame.render_widget( + Paragraph::new(Line::from(footer)).block(Block::bordered()), + foot_area, + ); } } +fn draw_command_bar(frame: &mut Frame, app: &App, area: ratatui::layout::Rect) { + frame.render_widget( + Paragraph::new(Line::from(vec![ + Span::styled(" :", Style::default().fg(Color::Cyan).bold()), + Span::raw(&app.command_text), + Span::styled("█", Style::default().fg(Color::Cyan)), + ])) + .block(Block::bordered().title_top(" command · :save FILE · :open FILE · Esc cancel ")), + area, + ); +} + fn human(value: u64) -> String { const UNITS: [&str; 4] = ["B", "KiB", "MiB", "GiB"]; let mut value = value as f64; @@ -792,6 +1327,7 @@ fn view_window( } (start, n) } else { + // Forward-fill from the anchor. let mut end = scroll; for (i, &event_index) in visible.iter().enumerate().skip(scroll) { let h = event_height(&events[event_index], width, view); @@ -804,7 +1340,22 @@ fn view_window( break; } } - (scroll, end) + // If the forward fill hit the end without filling the viewport, back-fill + // so the window stays full. Without this, leaving follow near the bottom + // (a single tall rich/wrapped item as the anchor) collapses the view to a + // couple of rows with blank space below. + let mut start = scroll; + if end == n && rows < log_h { + for (i, &event_index) in visible.iter().take(scroll).enumerate().rev() { + let h = event_height(&events[event_index], width, view); + if rows + h > log_h { + break; + } + rows += h; + start = i; + } + } + (start, end) } } @@ -1060,6 +1611,7 @@ mod tests { ) -> Output { Output::Message { message: DisplayMessage { + timestamp: "2026-07-17T12:34:56.789+01:00".into(), rendered: format!("[{kind}] {text}"), client: format!("127.0.0.1:{port}").parse().unwrap(), direction: MessageDirection::FrontendToBackend, @@ -1252,7 +1804,7 @@ mod tests { } #[test] - fn y_opens_editor_and_escape_clears_startup_display_filter() { + fn y_opens_editor_and_escape_cancels_edit_restoring_filter() { let (_tx, rx) = crossbeam_channel::unbounded(); let mut app = App::new( rx, @@ -1266,12 +1818,19 @@ mod tests { app.push_output(message("DataRow", "{ id=1 }", 40005)); assert_eq!(app.visible, vec![0]); + // Open the editor, type a partial edit, then Esc: the edit is abandoned + // and the previously-applied filter is restored (not wiped). handle_key( &mut app, 10, KeyEvent::new(KeyCode::Char('y'), KeyModifiers::NONE), ); assert!(app.filter_editing); + handle_key( + &mut app, + 10, + KeyEvent::new(KeyCode::Char('x'), KeyModifiers::NONE), + ); handle_key( &mut app, 10, @@ -1279,13 +1838,93 @@ mod tests { ); assert!(!app.filter_editing); + assert!( + !app.filter.is_empty(), + "Esc must not wipe the applied filter" + ); + assert_eq!(app.filter_text, "message.type == \"Query\""); + assert_eq!(app.visible, vec![0]); + + // A second Esc in normal mode clears the applied filter (documented). + handle_key( + &mut app, + 10, + KeyEvent::new(KeyCode::Esc, KeyModifiers::NONE), + ); assert!(app.filter.is_empty()); - assert!(app.filter_text.is_empty()); assert_eq!(app.visible, vec![0, 1]); } #[test] - fn slash_remains_available_for_commands() { + fn filter_applies_on_enter_not_per_keystroke() { + let mut app = app(); + app.push_output(message("Query", "SELECT 1", 40005)); + app.push_output(message("DataRow", "{ id=1 }", 40005)); + + handle_key( + &mut app, + 10, + KeyEvent::new(KeyCode::Char('y'), KeyModifiers::NONE), + ); + for ch in "message.type == \"Query\"".chars() { + handle_key( + &mut app, + 10, + KeyEvent::new(KeyCode::Char(ch), KeyModifiers::NONE), + ); + } + // Still unapplied while typing (both events visible). + assert_eq!(app.visible, vec![0, 1]); + handle_key( + &mut app, + 10, + KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE), + ); + assert!(!app.filter_editing); + assert_eq!(app.visible, vec![0]); + } + + #[test] + fn clear_counts_dropped_and_resets_scroll() { + let mut app = app(); + app.push_output(message("Query", "SELECT 1", 40005)); + app.push_output(message("Query", "SELECT 2", 40005)); + app.scroll = 1; + handle_key( + &mut app, + 10, + KeyEvent::new(KeyCode::Char('c'), KeyModifiers::NONE), + ); + assert!(app.events.is_empty()); + assert_eq!(app.scroll, 0); + assert_eq!( + app.dropped_events, 2, + "cleared events are counted as dropped" + ); + } + + #[test] + fn trim_history_preserves_scroll_position() { + let mut app = app(); + for i in 0..(HISTORY_CAP + HISTORY_TRIM_CHUNK) { + app.push_output(message("Query", &format!("q{i}"), 40005)); + } + // Sitting partway up the (now trimmed) buffer, not at the top. + app.follow = false; + app.scroll = 100; + let before = app.scroll; + app.push_output(message("Query", "trigger-trim", 40005)); + app.trim_history(); + // Scroll shifted with the evicted prefix, not reset to 0. + assert!( + app.scroll < before && app.scroll > 0, + "scroll: {}", + app.scroll + ); + } + + #[test] + fn colon_opens_command_bar_and_slash_opens_search() { let (_tx, rx) = crossbeam_channel::unbounded(); let mut app = App::new( rx, @@ -1296,13 +1935,158 @@ mod tests { Vec::new(), ); + // ':' opens the command bar (not the display filter editor). + handle_key( + &mut app, + 10, + KeyEvent::new(KeyCode::Char(':'), KeyModifiers::NONE), + ); + assert!(!app.filter_editing); + assert!(app.command_editing); + handle_key( + &mut app, + 10, + KeyEvent::new(KeyCode::Esc, KeyModifiers::NONE), + ); + + // '/' opens text search, not the command bar. handle_key( &mut app, 10, KeyEvent::new(KeyCode::Char('/'), KeyModifiers::NONE), ); + assert!(app.search_editing); + assert!(!app.command_editing); + } - assert!(!app.filter_editing); + #[test] + fn search_jumps_to_and_navigates_matches() { + let mut app = app(); + app.push_output(message("Query", "SELECT * FROM users", 40005)); // 0 + app.push_output(message("Query", "SELECT * FROM orders", 40005)); // 1 + app.push_output(message("Query", "SELECT * FROM users2", 40005)); // 2 + app.follow = false; + app.scroll = 0; + + // '/' opens search; type "orders" and Enter: jump to the match at pos 1. + handle_key( + &mut app, + 10, + KeyEvent::new(KeyCode::Char('/'), KeyModifiers::NONE), + ); + assert!(app.search_editing); + for ch in "orders".chars() { + handle_key( + &mut app, + 10, + KeyEvent::new(KeyCode::Char(ch), KeyModifiers::NONE), + ); + } + handle_key( + &mut app, + 10, + KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE), + ); + assert!(!app.search_editing); + assert_eq!(app.scroll, 1); + + // Search "users" -> matches positions 0 and 2; n/N cycle between them. + app.search_text = "users".into(); + app.scroll = 0; + handle_key( + &mut app, + 10, + KeyEvent::new(KeyCode::Char('n'), KeyModifiers::NONE), + ); + assert_eq!(app.scroll, 2, "n goes to next match after anchor 0"); + handle_key( + &mut app, + 10, + KeyEvent::new(KeyCode::Char('n'), KeyModifiers::NONE), + ); + assert_eq!(app.scroll, 0, "n wraps to the first match"); + handle_key( + &mut app, + 10, + KeyEvent::new(KeyCode::Char('N'), KeyModifiers::NONE), + ); + assert_eq!(app.scroll, 2, "N wraps back to the last match"); + + // Esc clears the active search. + handle_key( + &mut app, + 10, + KeyEvent::new(KeyCode::Esc, KeyModifiers::NONE), + ); + assert!(app.search_text.is_empty()); + } + + #[test] + fn save_command_writes_retained_and_future_events() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("saved session.jsonl"); + let mut app = app(); + app.push_output(message("Query", "SELECT 1", 40005)); + app.push_output(message("DataRow", "{ id=1 }", 40005)); + app.filter_text = "message.type == \"Query\"".into(); + app.update_filter(); + + app.command_text = format!("save {}", path.display()); + app.execute_command(); + app.push_output(message("ReadyForQuery", "txn=idle", 40005)); + app.recorder.as_mut().unwrap().flush().unwrap(); + + let saved = session::read_all(&path).unwrap(); + assert_eq!(saved.len(), 3, "display filtering must not affect saving"); + assert!( + app.command_notice + .as_ref() + .is_some_and(|(message, error)| !error && message.contains("recording")) + ); + } + + #[test] + fn open_command_atomically_replaces_view_and_preserves_rich_detail() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("replay.jsonl"); + let replayed = message_with_detail("DataRow", "{ id=1 }", 40005, Some(data_row_detail(2))); + let mut writer = SessionWriter::create(&path).unwrap(); + writer.write(&replayed).unwrap(); + writer.flush().unwrap(); + + let mut app = app(); + app.push_output(message("Query", "old", 40005)); + app.command_text = format!("open {}", path.display()); + app.execute_command(); + + assert_eq!(app.mode, "replay"); + assert!(!app.accept_source_records); + assert_eq!(app.events.len(), 1); + assert!(matches!( + app.events[0].detail(), + Some(EventDetail::DataRow(columns)) if columns.len() == 2 + )); + } + + #[test] + fn failed_open_keeps_the_existing_view() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("broken.jsonl"); + std::fs::write(&path, "not json\n").unwrap(); + let mut app = app(); + app.push_output(message("Query", "SELECT 1", 40005)); + + app.command_text = format!("open {}", path.display()); + app.execute_command(); + + assert_eq!(app.mode, "test"); + assert!(app.accept_source_records); + assert_eq!(app.events.len(), 1); + assert!( + app.command_notice + .as_ref() + .is_some_and(|(message, error)| *error && message.contains("invalid JSONL")) + ); } #[test] @@ -1332,7 +2116,7 @@ mod tests { assert!(app.show_splash); // Startup status lines do not count as traffic. app.push_output(Output::Status("tapgres: capturing on 'lo'".into())); - app.leave_splash_if_traffic(); + app.leave_splash_if_traffic(0); assert!(app.show_splash, "status lines must not leave the splash"); } @@ -1356,7 +2140,8 @@ mod tests { "127.0.0.1:5432".parse().unwrap(), false, ); - app.leave_splash_if_traffic(); + let opened = app.metrics.summary().conns_opened; + app.leave_splash_if_traffic(opened); assert!( !app.show_splash, "an opened connection must leave the splash" @@ -1364,7 +2149,7 @@ mod tests { } #[test] - fn splash_only_honours_quit() { + fn splash_honours_quit_and_command_bar() { let (_tx, rx) = crossbeam_channel::unbounded(); let mut app = App::new( rx, @@ -1381,6 +2166,15 @@ mod tests { KeyEvent::new(KeyCode::Char('y'), KeyModifiers::NONE), )); assert!(!app.filter_editing); + // Commands remain available so a replay can be opened before live + // traffic arrives. + assert!(!handle_key( + &mut app, + 10, + KeyEvent::new(KeyCode::Char('/'), KeyModifiers::NONE), + )); + assert!(app.command_editing); + app.command_editing = false; // `q` quits even from the splash. assert!(handle_key( &mut app, diff --git a/tests/fixtures/session-v1.jsonl b/tests/fixtures/session-v1.jsonl new file mode 100644 index 0000000..97e8352 --- /dev/null +++ b/tests/fixtures/session-v1.jsonl @@ -0,0 +1,4 @@ +{"schema_version":1,"timestamp":"2026-07-17T12:34:56.700+01:00","record_type":"status","text":"tapgres fixture session"} +{"schema_version":1,"timestamp":"2026-07-17T12:34:56.789+01:00","record_type":"message","direction":"f2b","message_type":"Query","text":"SELECT id FROM orders","rendered":"[12:34:56.789] [F→B] Query: SELECT id FROM orders","client":"127.0.0.1:40005"} +{"schema_version":1,"timestamp":"2026-07-17T12:34:56.800+01:00","record_type":"message","direction":"b2f","message_type":"RowDescription","text":"id(oid=23, text)","rendered":"[12:34:56.800] [B→F] RowDescription: id(oid=23, text)","client":"127.0.0.1:40005","detail":{"detail_type":"row_description","columns":[{"name":"id","type_oid":23,"format_code":0}]}} +{"schema_version":1,"timestamp":"2026-07-17T12:34:56.810+01:00","record_type":"message","direction":"b2f","message_type":"DataRow","text":"{ id='1' }","rendered":"[12:34:56.810] [B→F] DataRow: { id='1' }","client":"127.0.0.1:40005","detail":{"detail_type":"data_row","columns":[{"name":"id","type_oid":23,"value":"'1'"}]}} diff --git a/tests/session_cli.rs b/tests/session_cli.rs new file mode 100644 index 0000000..92916b4 --- /dev/null +++ b/tests/session_cli.rs @@ -0,0 +1,185 @@ +//! CLI integration coverage for the durable JSONL session source. + +use std::process::Command; + +use tapgres::decode::{EventDetail, FieldSummary, Output}; +use tapgres::filter::{DisplayMessage, MessageDirection}; +use tapgres::session::{self, SessionWriter}; + +fn message(kind: &str, text: &str, direction: MessageDirection) -> Output { + let tag = match direction { + MessageDirection::FrontendToBackend => "F→B", + MessageDirection::BackendToFrontend => "B→F", + }; + Output::Message { + message: DisplayMessage { + timestamp: "2026-07-17T12:34:56.789+01:00".into(), + rendered: format!("[12:34:56.789] [{tag}] {kind}: {text}"), + client: "127.0.0.1:40005".parse().unwrap(), + direction, + kind: kind.into(), + text: text.into(), + }, + detail: (kind == "RowDescription").then(|| { + EventDetail::RowDescription(vec![FieldSummary { + name: "id".into(), + type_oid: 23, + format_code: 0, + }]) + }), + } +} + +fn write_fixture(path: &std::path::Path) { + let mut writer = SessionWriter::create(path).unwrap(); + writer + .write(&Output::Status("saved session".into())) + .unwrap(); + writer + .write(&message( + "Query", + "SELECT * FROM orders", + MessageDirection::FrontendToBackend, + )) + .unwrap(); + writer + .write(&message( + "RowDescription", + "id(oid=23, text)", + MessageDirection::BackendToFrontend, + )) + .unwrap(); + writer.flush().unwrap(); +} + +#[test] +fn replay_uses_the_normal_stdout_filter_path() { + let dir = tempfile::tempdir().unwrap(); + let input = dir.path().join("capture.jsonl"); + write_fixture(&input); + + let result = Command::new(env!("CARGO_BIN_EXE_tapgres")) + .args([ + "--replay", + input.to_str().unwrap(), + "--display-filter", + "message.type == \"Query\"", + ]) + .output() + .unwrap(); + + assert!( + result.status.success(), + "stderr: {}", + String::from_utf8_lossy(&result.stderr) + ); + let stdout = String::from_utf8(result.stdout).unwrap(); + let stderr = String::from_utf8(result.stderr).unwrap(); + assert!(stdout.contains("Query: SELECT * FROM orders")); + assert!(!stdout.contains("RowDescription")); + assert!(stderr.contains("saved session")); +} + +#[test] +fn save_records_unfiltered_replay_stream() { + let dir = tempfile::tempdir().unwrap(); + let input = dir.path().join("capture.jsonl"); + let output = dir.path().join("copy.jsonl"); + write_fixture(&input); + + let result = Command::new(env!("CARGO_BIN_EXE_tapgres")) + .args([ + "--replay", + input.to_str().unwrap(), + "--save", + output.to_str().unwrap(), + "-Y", + "message.type == \"Query\"", + ]) + .output() + .unwrap(); + + assert!( + result.status.success(), + "stderr: {}", + String::from_utf8_lossy(&result.stderr) + ); + let saved = session::read_all(&output).unwrap(); + assert_eq!(saved.len(), 3); + assert!(saved.iter().any(|record| matches!( + record, + Output::Message { + message, + detail: Some(EventDetail::RowDescription(columns)), + } if message.kind == "RowDescription" && columns[0].type_oid == 23 + ))); +} + +#[test] +fn replay_refuses_to_overwrite_its_input() { + let dir = tempfile::tempdir().unwrap(); + let input = dir.path().join("capture.jsonl"); + write_fixture(&input); + + let result = Command::new(env!("CARGO_BIN_EXE_tapgres")) + .args([ + "--replay", + input.to_str().unwrap(), + "--save", + input.to_str().unwrap(), + ]) + .output() + .unwrap(); + + assert!(!result.status.success()); + assert!(String::from_utf8_lossy(&result.stderr).contains("must not overwrite")); + assert_eq!(session::read_all(&input).unwrap().len(), 3); +} + +#[test] +fn replay_and_save_accept_relative_paths() { + let dir = tempfile::tempdir().unwrap(); + write_fixture(&dir.path().join("capture.jsonl")); + + let result = Command::new(env!("CARGO_BIN_EXE_tapgres")) + .current_dir(dir.path()) + .args(["--replay", "capture.jsonl", "--save", "copy.jsonl"]) + .output() + .unwrap(); + + assert!( + result.status.success(), + "stderr: {}", + String::from_utf8_lossy(&result.stderr) + ); + assert_eq!( + session::read_all(dir.path().join("copy.jsonl")) + .unwrap() + .len(), + 3 + ); +} + +#[cfg(unix)] +#[test] +fn replay_refuses_hard_link_alias_as_save_target() { + let dir = tempfile::tempdir().unwrap(); + let input = dir.path().join("capture.jsonl"); + let alias = dir.path().join("same-file.jsonl"); + write_fixture(&input); + std::fs::hard_link(&input, &alias).unwrap(); + + let result = Command::new(env!("CARGO_BIN_EXE_tapgres")) + .args([ + "--replay", + input.to_str().unwrap(), + "--save", + alias.to_str().unwrap(), + ]) + .output() + .unwrap(); + + assert!(!result.status.success()); + assert!(String::from_utf8_lossy(&result.stderr).contains("must not overwrite")); + assert_eq!(session::read_all(&input).unwrap().len(), 3); +}