From 203e9b971e5d7d7d9e8fd09ae36b5f5a1ac9ced5 Mon Sep 17 00:00:00 2001 From: Roman Emreis <39233213+RomanEmreis@users.noreply.github.com> Date: Mon, 31 Aug 2026 19:04:49 +0400 Subject: [PATCH] docs: document the repository graph and project memory Adds documentation for the 0.4 repository graph release. New pages: - Repository graph: enablement (opt-in, not scaffolded by `ferrus init`), incremental indexing, the three built-in extractors with their node and edge kinds, the full `ferrus graph` command surface, orthogonal availability/build/freshness reporting, task worktree views, the read-only MCP tools and verified snippets, privacy invariants, query budgets, and the recorded benchmark and evaluation results. - Project memory: the default source allowlist and what is never indexed, `ferrus graph memory` commands, `--domain` federation semantics, repository link resolution states, and archive-driven refresh. Updates: - Configuration: `[repository_graph]` namespace reference, including which settings participate in snapshot identity, plus the two new sidecars in the runtime file table. - Quickstart: optional indexing step. - Supported agents: the six read-only retrieval tools exposed to every role. - State machine: pinned per-task repository views. - Specs & Milestones: how an approved `## Outcome` feeds project memory. - Sidebar, navbar, and footer entries; homepage section, feature cards, and version bump to 0.4.0-alpha.1. Command output and fact/edge vocabulary in the examples were captured from ferrus 0.4.0-alpha.1 running against the ferrus repository. Co-Authored-By: Claude Opus 5 --- docs/agents.md | 21 ++ docs/configuration.md | 113 ++++++++++ docs/project-memory.md | 203 +++++++++++++++++ docs/quickstart.md | 24 ++ docs/repository-graph.md | 426 ++++++++++++++++++++++++++++++++++++ docs/spec-and-milestones.md | 8 + docs/state-machine.md | 17 ++ docusaurus.config.ts | 9 +- sidebars.ts | 6 + src/pages/index.tsx | 47 +++- 10 files changed, 870 insertions(+), 4 deletions(-) create mode 100644 docs/project-memory.md create mode 100644 docs/repository-graph.md diff --git a/docs/agents.md b/docs/agents.md index 7e06d99..681bd9b 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -98,6 +98,27 @@ physically cannot call `approve`, and a supervisor physically cannot call `submit`. This is what makes the loop safe to drive from "untrusted" agents. +### Retrieval tools + +When the optional [repository graph](/docs/repository-graph) is configured, +**every** role — supervisor, executor, and the unfiltered server — additionally +gets the same six **read-only** retrieval tools: + +| Tool | Domain | +|---|---| +| `repository_graph_status` | Repository graph availability, freshness, task-view status | +| `repository_search` | Bounded ranked structural search | +| `repository_context` | Bounded context assembly, with opt-in verified snippets | +| `project_memory_status` | [Project memory](/docs/project-memory) revision, freshness, source policy | +| `project_context_search` | Bounded search across `repository`, `memory`, or `all` | +| `project_context` | Bounded federated context assembly | + +None of them resolves, claims, renews, or mutates a task lease, and none of +them builds an index. Their output is never injected into task or review +prompts — an agent has to ask. An executor working in a managed worktree is +answered from that task's own pinned baseline-plus-overlay view, never from +canonical `latest`. + ## Skill files `ferrus init` creates skill files under your `--agents-path` diff --git a/docs/configuration.md b/docs/configuration.md index 3d629ac..36599af 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -35,6 +35,9 @@ heartbeat_interval_secs = 30 # how often agents should call heartbeat [spec] directory = "docs/specs" # where /create_spec writes approved specs +[repository_graph] # optional; NOT scaffolded by `ferrus init` +enabled = true # opt into the local repository graph + project memory + [hq.supervisor] agent = "claude-code" # agent for supervisor/reviewer role: claude-code | codex | qwen-code | goose | opencode model = "" # optional override; empty = agent default @@ -108,6 +111,114 @@ in `ferrus.db`. directory = "docs/specs" # any path inside the project; created on first write ``` +## `[repository_graph]` + +The optional [repository graph](/docs/repository-graph) and +[project memory](/docs/project-memory) indexes. This namespace is **not written +by `ferrus init`** — add it yourself to opt in: + +```toml +[repository_graph] +enabled = true +``` + +Everything else has a default. Omitting the namespace entirely means the +optional indexes stay off and ordinary orchestration is unaffected. + +### Namespaces + +| Namespace | Purpose | Affects snapshot identity | +|---|---|---| +| `[repository_graph]` | `enabled`, backend selection | No — operational | +| `[repository_graph.source]` | Included content, untracked/generated/vendor policy, ordered ignore rules, sensitive paths | **Yes** | +| `[repository_graph.analyzers]` | Enabled extractors and their semantic settings | **Yes** | +| `[repository_graph.index_limits]` | Limits that can include, skip, or truncate a file during extraction | **Yes** | +| `[repository_graph.query_limits]` | Result, byte, depth, duration, snippet, and diagnostic budgets | No | +| `[repository_graph.retention]` | Snapshot and failed-build cleanup policy | No | +| `[repository_graph.memory]` | Authorized project-memory sources and extraction policy | Memory revision only | +| `[repository_graph.semantic]` | Reserved for future embedding/chunking policy | Semantic projection only | +| `[repository_graph.remote]` | Reserved endpoint, credential reference, upload policy | No | +| `[repository_graph.telemetry]` | Privacy-safe query metrics | No | + +The "affects snapshot identity" column is the one that matters day to day: +changing a query budget or retention rule **never** invalidates your index, +while changing what gets indexed does. Configuration is hashed as a canonical +semantic projection, not as raw TOML — whitespace, key ordering, and explicitly +writing out a default all produce the same digest. Credentials, tokens, and +endpoints never participate in identity and never appear in diagnostics. + +### `[repository_graph.source]` + +```toml +[repository_graph.source] +include = ["**/*"] # set-like; order is not significant +rules = [".git/**", ".ferrus/**"] # ordered ignore/negation rules; order IS significant +sensitive = [ # never read, never extracted, never uploaded + "**/.env", "**/.env.*", + "**/*.key", "**/*.pem", "**/*.p12", "**/*.pfx", + "**/id_rsa", "**/id_ed25519", +] +include_untracked = true +include_generated = false +include_vendor = false +``` + +Sensitive-path policy is applied **before** extraction, not after. + +### `[repository_graph.index_limits]` + +Bounds one indexing pass. Defaults: + +| Key | Default | +|---|---| +| `max_files` | `100000` | +| `max_directories` | `100000` | +| `max_file_bytes` | `2097152` (2 MiB) | +| `max_total_bytes` | `536870912` (512 MiB) | +| `max_facts_per_file` | `100000` | +| `max_parser_duration_ms` | `2000` | +| `max_resolved_relationships` | `1000000` | +| `max_resolver_duration_ms` | `10000` | +| `max_diagnostics` | `1000` | + +### `[repository_graph.query_limits]` + +Bounds every CLI and MCP read. The configured cap always wins over a caller's +requested budget; exceeding one returns the deterministic prefix that fits plus +an explicit truncation reason. + +| Key | Default | +|---|---| +| `max_results` | `100` | +| `max_bytes` | `262144` (256 KiB) | +| `max_snippet_bytes` | `32768` (32 KiB) | +| `max_depth` | `3` | +| `max_duration_ms` | `2000` | +| `max_diagnostics` | `50` | + +### `[repository_graph.retention]` + +```toml +[repository_graph.retention] +max_snapshots = 5 +max_failed_builds = 10 +``` + +Snapshots pinned by an active task are retained regardless. + +### `[repository_graph.telemetry]` + +```toml +[repository_graph.telemetry] +enabled = true +``` + +Emits one structured tracing metric per query: tool name, repository/task/run/ +snapshot identities, freshness, duration, result count, response bytes, +truncation reason, diagnostic count, and error category. The metric types +**cannot represent** request text, filters, repository paths, snippets, or +source bodies. Off by default. + ## `[hq.supervisor]` and `[hq.executor]` Which coding agent plays which role. Change these to swap backends without @@ -161,6 +272,8 @@ intent and run artifacts, not a mirrored state machine. |---|---| | `project.toml` | Project id, name, workspace path, `.ferrus` path, git metadata, timestamps, schema version | | `ferrus.db` | SQLite source of truth for tasks, runs, events, leases, counters, and project runtime state | +| `repo-graph.db` | Derived [repository graph](/docs/repository-graph) facts. Optional and rebuildable — safe to delete | +| `project-memory.db` | Derived [project memory](/docs/project-memory) revisions. Optional and rebuildable — safe to delete | | `archive/specs/-/` | Completed-spec archives written by `/archive-spec`: `manifest.toml`, a copy of `spec.md`, and the relocated `tasks/` and `runs/` artifacts | | `logs/` | Reserved for machine-local logs that should not be committed | diff --git a/docs/project-memory.md b/docs/project-memory.md new file mode 100644 index 0000000..53dc41b --- /dev/null +++ b/docs/project-memory.md @@ -0,0 +1,203 @@ +--- +id: project-memory +title: Project memory +sidebar_position: 7 +slug: /project-memory +--- + +# Project memory + +The [repository graph](/docs/repository-graph) knows what your code looks like +**now**. Project memory knows what your project **decided** — approved +outcomes, milestones, deviations, validation evidence, follow-up work — so a +fresh agent doesn't re-litigate a question that was settled three specs ago. + +It is a second derived, rebuildable index in its own SQLite sidecar +(`project-memory.db`). It is not orchestration state: deleting it removes no +tasks, runs, archives, specifications, or graph snapshots. Like the graph, it +works entirely offline — no network call, no LLM, no embedding model. + +## Enable and build + +Project memory shares the repository graph's configuration namespace, so enable +the graph first: + +```toml +[repository_graph] +enabled = true +``` + +Then: + +```bash +ferrus graph memory index # incremental +ferrus graph memory index --full # bypass fragment reuse; also replaces incompatible storage +ferrus graph memory status +ferrus graph memory status --json +``` + +```text +Project memory: NotBuilt +Revision: none +Freshness: Unknown +Next: ferrus graph memory index +``` + +Indexing is incremental — unchanged authorized sources reuse cached fragments, +and a no-change run does not advance the published generation. Publication is +atomic: a failed or interrupted build leaves the previous revision readable. + +## What gets indexed — and what never does + +This is the part worth reading twice. Memory ingestion runs on an **explicit +allowlist**, and the presence of a file under `.ferrus/` or in an archive is +*not* permission to read it. + +Enabled by default: + +| Source category | Content boundary | Sensitivity | +|---|---|---| +| Specification structure | Tracked headings and milestone metadata | Curated | +| Approved outcome | The approved `## Outcome` section | Curated | +| Archive manifest | Archive identity and counts | Operational metadata | +| Runtime provenance | Terminal task, run, milestone, status, and check **identities** | Operational metadata | + +Disabled and treated as sensitive — raw task descriptions, submissions, +reviews, patches, check logs, questions, answers, consultations, and +integration-error bodies. Runtime provenance opens `ferrus.db` read-only and +keeps only IDs and statuses; raw payloads, task paths, agent names, PIDs, +workspace paths, and failure reasons do not cross the adapter. + +Use JSON status to audit every category and its effective policy: + +```bash +ferrus graph memory status --json +``` + +:::warning[Curated does not mean public] +Approved `## Outcome` sections are the one thing ferrus copies verbatim into +memory. Keep them concise and free of secrets. +::: + +## Entities and relationships + +Memory covers specifications, milestones, outcomes, decisions, deviations, +validation evidence, follow-up work, and task/run references, related through +typed edges: `contains`, `implements`, `validates`, `supersedes`, `concerns`, +`touches`, `follows_up`. These are history semantics and are deliberately +independent of repository dependency edges. + +Every entity and relationship carries project scope, memory revision, source +category, a portable locator, source fingerprint, extractor identity and +version, evidence span or record ID, resolution state, confidence, and +timestamps. + +## Querying + +Repository-only is the default — nothing changes unless you ask for it: + +```bash +ferrus graph search RuntimeTaskContext # repository only +ferrus graph search "bounded retrieval" --domain memory # memory only +ferrus graph context --milestone rg4.6 --domain memory +ferrus graph context --milestone rg4.6 --domain all # federated +``` + +Memory context accepts several seed types: + +```bash +ferrus graph context --domain memory --memory-entity +ferrus graph context --domain memory --milestone rg4.6 +ferrus graph context --domain memory --task +ferrus graph context --domain memory --run +``` + +Federated responses report both identities independently: + +```text +Repository: snapshot=snapshot:7b39a635... freshness=Stale +Memory: revision=revision:... freshness=Fresh +``` + +### What `--domain all` does *not* do + +It does not merge the two stores, and it does not turn history into a claim +about current code. It resolves the repository snapshot and memory revision +separately, loads only the exact link set for those two revisions, and crosses +domains **only** through evidence-backed links. Current repository evidence +wins equal-rank tie-breaking, so historical context supplements source rather +than displacing it. `Fresh` in one domain says nothing about the other. + +## Repository links: resolved, stale, unresolved + +A memory fact can point at code, but only on explicit evidence — the tracked +specification path, the repository-relative spec path in an approved archive +manifest, curated inline references written as `` `path:src/lib.rs` `` or +`` `symbol:rust:function:src/lib.rs:run` ``, or baseline/materialized snapshot +IDs attached to terminal task and run metadata. Changed-path evidence is +computed from file identities in two authorized snapshots — raw patches and +artifact bodies are never read. + +| State | Meaning | +|---|---| +| `resolved` | Exact path or unique semantic-key match in the selected snapshot | +| `stale` | Resolved in an earlier revision, no longer matching the selected snapshot; retains the historical target | +| `unresolved` | Never matched, origin unavailable, or ambiguous semantic key | + +Stale and unresolved links stay labelled and are presented as historical +evidence only. Links are **never** promoted through text similarity or LLM +inference, and link sets are stored separately from semantic memory revisions — +so reindexing unchanged memory against a new repository snapshot does not +rewrite memory facts or change the revision ID. + +## Freshness and the archive lifecycle + +`/archive-spec` is the only workflow that writes approved `## Outcome` content. +After the archive transaction commits, ferrus attempts an incremental memory +refresh — deliberately **outside** the archive's critical path, so a refresh +failure can never undo or fail a successful archive. + +Repository and memory freshness are independent. When an authorized source +changes, queries keep the last published revision readable, report memory as +stale, and recommend: + +```bash +ferrus graph memory index +``` + +CLI status compares the current authorized source manifest exactly. +Latency-bounded MCP retrieval may report freshness as `unknown` when it cannot +perform that comparison safely — `unknown` is not a claim that the revision is +fresh. + +## What agents get + +Supervisor and Executor see the same read-only surface: + +| Tool | Purpose | +|---|---| +| `project_memory_status` | Availability, revision, freshness, source policy, counts | +| `project_context_search` | Bounded ranked search across the selected domain | +| `project_context` | Bounded evidence-backed context assembly | + +Every MCP request must state its domain — `repository`, `memory`, or `all` — +explicitly. The tagged shape makes it impossible to silently broaden a +repository-only request. None of these tools builds an index, authors an +`## Outcome`, or touches task lifecycle state. + +## Operational notes + +- Treat stale or unresolved repository links as historical evidence, never as a + current-code relationship. +- Do **not** use `project-memory.db` as a backup. Rebuild it from tracked specs + and registered runtime metadata. +- Status reports retention diagnostics (total and historical revisions, + terminal builds not backing the current publication, link-set counts). They + are diagnostics only — ferrus does not silently delete history. If the + sidecar grows unexpectedly, `--full` is the explicit cleanup path. + +## See also + +- [Repository graph](/docs/repository-graph) — structural indexing and retrieval +- [Specs & Milestones](/docs/spec-and-milestones#closing-out-a-spec-archive-spec) — where `## Outcome` comes from +- [Configuration](/docs/configuration#repository_graph) diff --git a/docs/quickstart.md b/docs/quickstart.md index 9da310a..a2ece33 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -102,6 +102,30 @@ summary and files the spent task/run artifacts away — see the Press **Ctrl+C** twice within 2 seconds to exit HQ. ::: +## 5. (Optional) Index your repository + +ferrus can build a local structural index of your codebase so agents navigate +it by symbol and relationship instead of re-grepping it on every spawn. It is +opt-in — add the namespace to `ferrus.toml`: + +```toml +[repository_graph] +enabled = true +``` + +Then build it: + +```bash +ferrus graph index +ferrus graph status +ferrus graph search MyType --limit 10 +``` + +The index lives in its own machine-local SQLite sidecar, is rebuildable, and is +safe to delete at any time — an absent index never blocks the task loop. See +[Repository graph](/docs/repository-graph) and +[Project memory](/docs/project-memory). + ## What happens next ```text diff --git a/docs/repository-graph.md b/docs/repository-graph.md new file mode 100644 index 0000000..349894a --- /dev/null +++ b/docs/repository-graph.md @@ -0,0 +1,426 @@ +--- +id: repository-graph +title: Repository graph +sidebar_position: 6 +slug: /repository-graph +--- + +# Repository graph + +Every time an agent spawns, it starts blind. It greps, it reads directory +listings, it opens the same five files it opened on the last run — and it pays +tokens for all of it. The **repository graph** is ferrus's answer: an optional, +machine-local structural index that agents can query instead of rediscovering +your codebase from scratch. + +It is a **rebuildable materialized view of one explicitly identified source +state**. Your source and Git remain authoritative. The graph may be missing, +stale, incompatible, or deleted at any moment without changing task state — an +absent index never blocks the Supervisor → Executor → Reviewer loop. + +:::info[Two sidecars, one feature] +The repository graph indexes **current structure**. Its sibling, +[project memory](/docs/project-memory), indexes **curated history** — approved +outcomes, decisions, milestones. They live in separate SQLite files, advance on +independent revisions, and are queried through the same commands and tools. +::: + +## Enable it + +The graph is **opt-in and not scaffolded** by `ferrus init`. Add the namespace +to `ferrus.toml` yourself: + +```toml +[repository_graph] +enabled = true +``` + +That single line is enough — every other setting has a default. See +[Configuration](/docs/configuration#repository_graph) for the full surface. + +## Build the index + +```bash +ferrus graph index +``` + +```text +Indexed repository graph +Snapshot: snapshot:7b39a63599ec38388b417269b82f330051507ca17b71ae25af95787b64130791 +Freshness: fresh +Files: 265 discovered, 0 reused, 265 parsed, 0 skipped, 0 failed +Facts: 6913 nodes, 8200 edges, 700 diagnostics +Work: 2054529 bytes in 880 ms +``` + +Re-running it is **incremental**: unchanged files are reused rather than +re-parsed, and a no-op build publishes the same snapshot identity. + +```bash +ferrus graph index # incremental — reuses unchanged fragments +ferrus graph index --full # ignore the fragment cache and re-run every extractor +``` + +Facts land in `repo-graph.db` next to `ferrus.db`: + +| Path | Contents | +|---|---| +| `~/.ferrus/projects//ferrus.db` | Runtime source of truth (tasks, runs, leases) | +| `~/.ferrus/projects//repo-graph.db` | Derived structural facts — **safe to delete** | +| `~/.ferrus/projects//project-memory.db` | Derived curated history — **safe to delete** | + +Deleting a sidecar loses nothing but build time. `ferrus graph index` rebuilds +it. + +:::tip[Git is not required] +Indexing reads the filesystem through a confined source adapter. A clean Git +tree lets ferrus reuse Git content identities, but a non-Git directory indexes +fine — it just gets content digests instead. +::: + +## What gets indexed + +Three built-in extractors run today: + +| Extractor | Covers | Emits | +|---|---|---| +| `builtin.generic-structure` | Every included file, any language | `repository`, `directory`, `file`, `document`, `manifest`, `configuration`, `entry_point` | +| `builtin.rust-syntax` | Rust sources, via tree-sitter | `module`, `mod_declaration`, `struct`, `enum`, `trait`, `function`, `impl`, `constant`, `type_alias`, `import`, `re_export` | +| `builtin.cargo-manifest` | `Cargo.toml` | `cargo_workspace`, `cargo_package`, `cargo_target`, `declared_dependency` | + +Relationships between them: + +| Edge | Meaning | +|---|---| +| `contains` | Structural containment (directory → file, module → symbol) | +| `classified_as` | A file's inferred role (document, manifest, configuration, entry point) | +| `declares_module` | A `mod x;` declaration pointing at its module | +| `imports` / `re_exports` | `use` declarations, split by visibility | +| `declares_target` | A Cargo package declaring a lib/bin/bench/test target | +| `depends_on` | A declared Cargo dependency, classified internal / workspace / external | + +So a Rust project gets real symbol-level navigation, and **any** project gets +structure, documentation, configuration, and entry-point mapping. Additional +languages are extractor work, not architecture work — the identity, snapshot, +and query contracts are language-neutral. + +### Semantic keys + +Beyond opaque node IDs, most facts carry a human-typeable **semantic key**: + +```text +rust:struct:src/project.rs:project::RuntimeTaskContext +rust:file-module:src/project.rs +cargo:package:Cargo.toml:ferrus +cargo:target:Cargo.toml:bin:ferrus +file:README.md +configuration:11:ferrus.toml:toml +``` + +A semantic key is a best-effort lookup convenience, **not** a stable primary +key. Renames, refactors, and parser changes may change it; opaque node IDs are +scoped to one snapshot and are not promised to survive the next one either. + +## Query it from the CLI + +### Status first + +```bash +ferrus graph status +``` + +```text +Repository graph: Available +Snapshot: snapshot:7b39a63599ec38388b417269b82f330051507ca17b71ae25af95787b64130791 +Freshness: Stale +Facts: 265 files, 6913 nodes, 8200 edges +Diagnostics: 0 info, 700 warnings, 0 errors +``` + +`status` is the only graph command that works while the graph is disabled or +unbuilt — everything else requires `enabled = true`. When there is nothing to +query it tells you what to run next (`Next: ferrus graph index`). + +### Search + +```bash +ferrus graph search RuntimeTaskContext --limit 5 +``` + +```text +Snapshot: snapshot:7b39a635... +Freshness: Stale +Diagnostics: 0 info, 700 warnings, 0 errors +Truncated: no +0.98 struct rust:struct:src/project.rs:project::RuntimeTaskContext + id=node:df6810fd... evidence=src/project.rs:361:1 resolution=Resolved confidence=Exact +``` + +Matching is ranked deterministically, best class first: exact semantic key → +exact path → exact name → name prefix → name substring → semantic-key substring +→ path substring. Filters narrow it further: + +```bash +ferrus graph search Task --kind struct --kind enum --path src --limit 20 +``` + +### Show + +Look up everything evidenced by one node, symbol, or path: + +```bash +ferrus graph show --node node:df6810fd... +ferrus graph show --symbol "rust:struct:src/project.rs:project::RuntimeTaskContext" +ferrus graph show --path src/project.rs +``` + +### Context + +Assemble a bounded, evidence-backed context packet around one or more seeds: + +```bash +ferrus graph context --symbol "rust:struct:src/project.rs:project::RuntimeTaskContext" --depth 1 +``` + +```text +Snapshot: snapshot:7b39a635... +Freshness: Stale +Diagnostics: 0 info, 700 warnings, 0 errors +Truncated: Depth (2 results, 1974 bytes, depth 1) +struct rust:struct:src/project.rs:project::RuntimeTaskContext src/project.rs:361:1 + selected=ExactSeed resolution=Resolved confidence=Exact extractor=builtin.rust-syntax@1.0.0+tree-sitter-0.26.11.rust-0.24.2 +module rust:file-module:src/project.rs src/project.rs:1:1 + selected=Containment resolution=Resolved confidence=Exact extractor=builtin.rust-syntax@1.0.0+tree-sitter-0.26.11.rust-0.24.2 +``` + +Every item states **why it was selected** (`ExactSeed`, `Containment`, +`Declaration`, resolved dependency, documentation, configuration, …) and which +extractor produced it. Ranking is deterministic: selection reason, then depth, +then evidence location, then kind, then key, then node ID. The same request +against the same snapshot returns the same ordered results. + +### Neighbors + +Walk a bounded neighborhood around one node: + +```bash +ferrus graph neighbors node:599f0a1f... --direction both --depth 2 --limit 50 +``` + +```text +Nodes: + module node:599f0a1f... evidence=src/project.rs:1:1 resolution=Resolved confidence=Exact + struct node:df6810fd... evidence=src/project.rs:361:1 resolution=Resolved confidence=Exact +Edges: + contains node:599f0a1f... -> node:df6810fd... resolution=Resolved confidence=Exact +``` + +### Machine-readable output + +Every graph command accepts `--json` and emits one document containing the wire +version, repository and snapshot identities, freshness, diagnostics, +pagination/truncation state, and the operation payload. + +```bash +ferrus graph status --json +ferrus graph search RuntimeTaskContext --json +``` + +## Snapshots, builds, and freshness + +The single most important thing to understand about the graph is that **three +states are orthogonal** and never collapse into one "is it OK?" flag: + +| Axis | Values | Question it answers | +|---|---|---| +| Availability | `not_built`, `available`, `incompatible` | Is there a published snapshot I can read? | +| Build | `building`, `complete`, `published`, `failed`, `superseded` | What happened to the most recent attempt? | +| Freshness | `fresh`, `stale`, `unknown`, `not_applicable` | Does the published snapshot still match the source? | + +This is why status can report *"an available snapshot, plus a newer refresh +that failed"* without either fact erasing the other. A failed build never +replaces the last published view, and partial facts are never visible to +ordinary queries — publication is a compare-and-set against the expected +current pointer, so an older build cannot overwrite a newer one. + +Status also emits a machine-readable recommended action: `index`, +`wait_for_build`, `retry_index`, `refresh_index`, or `rebuild`. + +Freshness is computed by **actually comparing the current source manifest** +against the snapshot's identity inputs — not inferred from events. Edit a file +outside ferrus entirely and the next `ferrus graph status` still reports +`Stale`. + +A snapshot's identity is derived from the repository, its source manifest +digest, the graph model version, the effective semantic configuration digest, +and the extractor-set digest. Operational settings — query budgets, retention, +telemetry, endpoints, credentials — deliberately **do not** participate. Two +configurations that mean the same thing produce the same snapshot; changing a +timeout does not invalidate your index. + +## Task views: what the executor sees + +Executors work in isolated Git worktrees, so pointing them at a canonical +`latest` index would be wrong. Instead, each task gets its own **task +repository view**: + +```text +pinned baseline snapshot (the task's Ferrus baseline Git tree) + + overlay (that worktree's changed / added / deleted files) + = task view +``` + +- A newer canonical snapshot **does not** make a pinned baseline stale — other + tasks advancing does not disturb this one. +- Worktree edits make only *that* task's overlay stale. +- Overlay deletions hide baseline nodes, edges, snippets, and search hits for + the deleted path. +- On submission, the view is **frozen** together with its Git tree, and review + reopens it by identity. The reviewer reads exactly what was submitted. +- If a task view is unavailable, the tool says so (`not_built`, `stale`, + `unavailable`, `failed`) and directs the agent to inspect source directly. It + **never silently falls back** to canonical data. + +Canonical integration after approval marks the canonical snapshot stale and +refreshes it best-effort. Graph refresh failure can never undo or fail a +successful approval. + +## What agents actually get + +When the graph is configured, `ferrus serve` exposes three **read-only** tools +to Supervisor, Executor, and unfiltered servers alike: + +| Tool | Purpose | +|---|---| +| `repository_graph_status` | Availability, build state, freshness, task-view status, recommended action | +| `repository_search` | Bounded ranked search — `query`, `kinds`, `paths`, budget caps, `cursor` | +| `repository_context` | Bounded context assembly — `seeds`, `direction`, `edge_kinds`, `include_unresolved`, `include_external`, `include_snippets` | + +None of them resolves, claims, renews, or mutates a task lease. Graph output is +**never injected into task or review prompts** — an agent must ask for it, and +tool descriptions steer it to check status first and keep requests bounded. + +### Verified snippets + +Structural results never embed file bodies. Source text is opt-in per request +via `include_snippets`, and every excerpt is resolved through `SnapshotContent`: +the reader holds a canonical source-root handle, rejects symlink traversal, +reapplies the sensitive-path policy, and verifies size, mode, and SHA-256 +**before** slicing the recorded byte span. An independent aggregate snippet-byte +cap applies on top. + +Content that changed, became unavailable, was excluded, or is not UTF-8 is +omitted and reported as a bounded diagnostic (`content.changed`, +`content.unavailable`, `content.non_utf8`). Unverified bytes are never +returned. + +## Absence is not proof + +The contract every consumer must respect: **a missing node or edge means "not +known by this snapshot and its capabilities" — not "does not exist."** + +Every fact carries its extractor ID and version, evidence path and span, +resolution state (`resolved` / `unresolved` / `external`), and a confidence +classification. Paths that cannot be represented as valid UTF-8 are skipped +with a bounded diagnostic — an explicit capability limitation, not evidence +that the file has no relationships. + +That's also what the diagnostic counts in every response are for. The ferrus +dogfood snapshot above carries 700 bounded warnings: places the extractors +declined to guess. + +## Privacy and safety + +The graph treats your repository as untrusted input: + +- Indexing **never executes** repository code, hooks, build scripts, compilers, + or macros. It parses. +- Source adapters confine paths to the selected root and do not follow symlinks + outside it. +- Sensitive-path policy is applied **before** extraction. The default excluded + set covers `**/.env`, `**/.env.*`, `**/*.key`, `**/*.pem`, `**/*.p12`, + `**/*.pfx`, `**/id_rsa`, `**/id_ed25519`. +- Full source bodies are **not stored** in the sidecar — only identities, + spans, and structure. +- Logs, events, metrics, errors, and diagnostics carry IDs, counts, timings, + and bounded codes — never source bodies, secrets, or absolute workspace + paths. +- Credentials, tokens, and endpoints never participate in snapshot identity and + never appear in diagnostics. +- Everything is local. Indexing and retrieval make no network call, load no + model, and contact no service. + +## Budgets + +Every query is bounded, and the **configured service cap always wins** over a +client's request. When a budget runs out you get the deterministic prefix that +fits plus an explicit truncation reason (`Results`, `Bytes`, `Depth`, +`Duration`, `Capability`) — never a silent drop and never a generic backend +error. A continuation cursor is issued only after at least one result was +returned, and it is bound to the exact operation, snapshot, and normalized +request shape. + +Defaults, all tunable under `[repository_graph.query_limits]`: + +| Key | Default | +|---|---| +| `max_results` | `100` | +| `max_bytes` | `262144` (256 KiB) | +| `max_snippet_bytes` | `32768` (32 KiB) | +| `max_depth` | `3` | +| `max_duration_ms` | `2000` | +| `max_diagnostics` | `50` | + +## Measured behaviour + +From the Criterion benchmark over a generated 300-module Rust fixture +(`cargo bench --bench repository_graph`, macOS arm64, release profile): + +| Operation | Time | Parsed | Reused | +|---|---:|---:|---:| +| Cold build (302 files) | 97.0 ms | 302 | 0 | +| No-op build | 49.1 ms | 0 | 302 | +| One-file change | 84.0 ms | 1 | 301 | +| Indexed symbol search | 238 µs | — | — | + +These are development baselines for catching accidental full re-extraction or +unbounded queries — not release performance guarantees. Compare relative +behaviour and fact counts before comparing wall-clock across machines. + +A 26-case deterministic navigation evaluation gates retrieval quality on every +change: + +| Gate | Threshold | Observed | +|---|---:|---:| +| Exact path Recall@1 | 100% | 100% | +| Exact unique-symbol Recall@1 | 100% | 100% | +| Supported discovery Recall@10 | ≥ 90% | 93.75% | +| Repeated same-snapshot determinism | 100% | 100% | +| No regression vs. navigation baseline | 100% | 100% | +| Median files-read **or** context-byte reduction | ≥ 20% | 100% files-read reduction | + +The honest caveat: on that same corpus, broad `direction=both` expansion +currently returns *more* serialized graph evidence than a baseline source scan +(median context-byte change ≈ `-772%`), even though it eliminates source-file +reads entirely. That is a real limitation and it is why the graph stays a +**query tool** rather than something ferrus injects into prompts for you. + +## Deliberately not here yet + +- **Automatic context injection.** Agents ask; ferrus does not push. Context-byte + volume needs to come down first. +- **Embeddings and semantic search.** Structural retrieval already covers the + high-confidence paths, and a mandatory model runtime or vector extension is + not worth adding before a measured recall gap exists. The architecture + reserves a separate semantic revision so an embedding model change can never + rebuild or rename a structural snapshot. +- **Remote / distributed indexing.** The identity, snapshot, evidence, and + bounded-query contracts are deliberately backend-neutral so a future service + can partition storage and workers by tenant — but local ferrus never + initializes a network client, and remote indexing would be strictly opt-in. + +## See also + +- [Project memory](/docs/project-memory) — curated history and cross-domain queries +- [Configuration](/docs/configuration#repository_graph) — the full `[repository_graph]` surface +- [Supported agents](/docs/agents#tools-exposed-per-role) — which roles see the retrieval tools diff --git a/docs/spec-and-milestones.md b/docs/spec-and-milestones.md index 567c9d2..309a3b8 100644 --- a/docs/spec-and-milestones.md +++ b/docs/spec-and-milestones.md @@ -170,6 +170,14 @@ The spec file itself stays in your repository, now carrying its `## Outcome` summary; only the machine-local task/run scratch artifacts are relocated, so `.ferrus/tasks/` and `.ferrus/runs/` stay focused on active work. +That approved `## Outcome` is also the one thing +[project memory](/docs/project-memory) indexes verbatim. If the optional index +is enabled, ferrus refreshes it right after the archive transaction commits — +outside the archive's critical path, so a refresh failure can never undo a +successful archive — and future agents can then query the decision with +`ferrus graph search ... --domain memory` instead of re-reading raw task +artifacts that no longer exist in the checkout. + ### Preconditions `/archive-spec` refuses to run — with a specific message — unless all of these diff --git a/docs/state-machine.md b/docs/state-machine.md index f215524..a8662e8 100644 --- a/docs/state-machine.md +++ b/docs/state-machine.md @@ -67,6 +67,23 @@ up; `/run [--limit N]` queues every ready milestone in the selected spec (or up to `--limit`) in one shot so the scheduler can work through several tasks in parallel. +## Pinned repository views + +If the optional [repository graph](/docs/repository-graph) is enabled, each task +also carries a **repository view reference**: the immutable baseline snapshot +matching the task's pinned Ferrus baseline Git tree, plus an overlay derived +from that worktree's changed, added, and deleted files. + +This keeps structural context as isolated as the worktree itself. A newer +canonical snapshot does not make a running task's baseline stale; only that +task's own edits invalidate its overlay. On submission the view is frozen +alongside its Git tree, so the reviewer reads exactly what was submitted. + +Indexing is never a correctness dependency of this state machine. If a view is +missing, stale, or failed, the tools say so and the agent falls back to reading +source — dispatch, checks, submission, review, and approval all proceed +regardless. + ## Retry and review budgets Three counters guard the loop, all configured in `ferrus.toml` under diff --git a/docusaurus.config.ts b/docusaurus.config.ts index e60bc07..74e8865 100644 --- a/docusaurus.config.ts +++ b/docusaurus.config.ts @@ -81,11 +81,16 @@ const config: Config = { label: 'Quickstart', position: 'left', }, + { + to: '/docs/repository-graph', + label: 'Repository graph', + position: 'left', + className: 'navbar__link--accent', + }, { to: '/docs/migration', label: 'Migrating from 0.2.x', position: 'left', - className: 'navbar__link--accent', }, { href: 'https://github.com/ferrus-dev/ferrus', @@ -104,6 +109,8 @@ const config: Config = { {label: 'Configuration', to: '/docs/configuration'}, {label: 'HQ Commands', to: '/docs/hq'}, {label: 'State Machine', to: '/docs/state-machine'}, + {label: 'Repository Graph', to: '/docs/repository-graph'}, + {label: 'Project Memory', to: '/docs/project-memory'}, {label: 'Migrating from 0.2.x', to: '/docs/migration'}, {label: 'Local Models', to: '/docs/local-models'}, ], diff --git a/sidebars.ts b/sidebars.ts index 4ed29dc..643a0cc 100644 --- a/sidebars.ts +++ b/sidebars.ts @@ -7,6 +7,12 @@ const sidebars: SidebarsConfig = { 'hq', 'spec-and-milestones', 'state-machine', + { + type: 'category', + label: 'Repository intelligence', + collapsed: false, + items: ['repository-graph', 'project-memory'], + }, 'agents', 'migration', 'local-models', diff --git a/src/pages/index.tsx b/src/pages/index.tsx index 49e85cc..eba8ce7 100644 --- a/src/pages/index.tsx +++ b/src/pages/index.tsx @@ -39,7 +39,7 @@ function Hero() {
- alpha · v0.3.1 + alpha · v0.4.0 Apache-2.0 Rust 1.95+
@@ -71,7 +71,7 @@ function Install() { {`# stable — published on crates.io cargo install ferrus # or pin an exact version: -cargo install --locked ferrus@0.3.1-alpha.1`} +cargo install --locked ferrus@0.4.0-alpha.1`}
@@ -112,6 +112,38 @@ ferrus # enter HQ`} +
+ + $ Repository graph + +

+ New in 0.4: an optional, machine-local structural index of your + repository. Agents query it by symbol, path, and relationship instead + of re-grepping the codebase on every spawn — and every fact comes back + with its evidence location, extractor, and confidence. +

+ {`ferrus graph index # build the snapshot (incremental) +ferrus graph status # availability, freshness, fact counts +ferrus graph search RuntimeTaskContext # ranked, evidence-backed lookup +ferrus graph context --symbol # bounded context packet`} +

+ Snapshots are immutable and atomically published. Executors query + their own worktree view — a pinned baseline plus a changed-file + overlay — so parallel tasks never see each other's work in progress. + The index is opt-in, entirely local, rebuildable, and safe to delete: + an absent graph never blocks the task loop. +

+

+ Read the repository graph docs → +

+
+ + ); +} + function Features() { const features = [ { @@ -130,9 +162,17 @@ function Features() { title: 'No hidden context', body: 'Everything the agents see lives in plain files you can read, scoped per task under .ferrus/tasks/ and .ferrus/runs/.', }, + { + title: 'Repository-aware', + body: 'An optional local graph indexes structure, symbols, and relationships so agents navigate instead of rediscovering — every fact evidence-backed and snapshot-pinned.', + }, + { + title: 'Remembers decisions', + body: 'Project memory keeps approved outcomes, milestones, and deviations queryable, so settled questions stay settled across specs.', + }, ]; return ( -
+
$ Why ferrus @@ -160,6 +200,7 @@ export default function Home(): ReactNode {
+