feat(mcp): add codebase exploration tools

README.md

@@ -247,9 +247,9 @@ Once configured, the agent automatically decides when semantic code search is he
 <details>
 <summary>MCP Tool Reference</summary>
 
-When running as an MCP server (`ccc mcp`), the following tool is exposed:
+When running as an MCP server (`ccc mcp`), the following tools are exposed:
 
-**`search`** — Search the codebase using semantic similarity.
+**`search`** - Search the codebase using semantic similarity.
 
 ```
 search(
@@ -262,7 +262,32 @@ search(
 )
 ```
 
-Returns matching code chunks with file path, language, code content, line numbers, and similarity score.
+Returns matching code chunks with file path, language, code content, line numbers, similarity score, and `freshness` (`stale` plus the time of the latest successful index). When `stale` is true, search again with `refresh_index=true` before relying on the result.
+
+**`grep`** - Structural code search by syntax example. It runs locally and needs no index or embedding model.
+
+```
+grep(
+    pattern: str,                        # e.g. r"def \NAME(\(ARGS*\)):"
+    path: str = ".",                     # File or directory relative to the project root
+    languages: list[str] | None = None,  # Optional language filter
+    path_glob: str | None = None,        # Optional project-relative glob
+    limit: int = 100,                    # Maximum matches returned (1-1000)
+)
+```
+
+Returns exact matched source spans with file paths and line ranges. Paths outside the configured project are rejected.
+
+**`status`** - Report index health and freshness: whether indexing is active, index/file/chunk counts, indexed-language breakdown, and `freshness`.
+
+**`codebase_overview`** - Return a compact tree of indexable source files, top-level directory file counts, and language file counts without returning source text. Use it to orient in an unfamiliar repository.
+
+```
+codebase_overview(
+    max_depth: int = 2,                  # Tree depth (1-8)
+    max_entries: int = 80,               # Maximum tree entries (1-500)
+)
+```
 </details>
 
 ## Manual CLI Usage

scripts/find_best_models.py

@@ -45,7 +45,7 @@
 import pandas as pd
 from datasets import load_dataset
 from huggingface_hub import HfApi, ModelInfo, dataset_info
-from tabulate import tabulate
+from tabulate import tabulate  # type: ignore[import-untyped]
 
 CACHE_DIR = Path(".cocoindex_code/cache")
 CACHE_FILE = CACHE_DIR / "mteb_results.parquet"
@@ -168,10 +168,11 @@ def main() -> None:
     # Fetch dataset info for freshness (always live check for the date)
     try:
         info = dataset_info("mteb/results")
-        last_update = info.lastModified.strftime("%Y-%m-%d")
-        report.append(
-            f"\n> **Data Freshness**: MTEB results dataset last updated on `{last_update}`."
-        )
+        if info.lastModified is not None:
+            last_update = info.lastModified.strftime("%Y-%m-%d")
+            report.append(
+                f"\n> **Data Freshness**: MTEB results dataset last updated on `{last_update}`."
+            )
     except Exception:
         pass
 

src/cocoindex_code/cli.py

@@ -7,6 +7,7 @@
 import sys
 import threading
 from collections.abc import Callable
+from datetime import datetime
 from pathlib import Path
 from typing import TYPE_CHECKING, TypeVar
 
@@ -32,6 +33,7 @@
     find_parent_with_marker,
     find_project_root,
     format_path_for_display,
+    index_state_path,
     normalize_input_path,
     project_settings_path,
     resolve_db_dir,
@@ -210,6 +212,12 @@ def print_index_stats(status: ProjectStatusResponse) -> None:
     _typer.echo("\nIndex stats:")
     _typer.echo(f"  Chunks: {status.total_chunks}")
     _typer.echo(f"  Files:  {status.total_files}")
+    if not status.indexing:
+        if status.stale:
+            _typer.echo("  Freshness: stale (run `ccc index`)")
+        elif status.indexed_at_us is not None:
+            indexed_at = datetime.fromtimestamp(status.indexed_at_us / 1_000_000).astimezone()
+            _typer.echo(f"  Freshness: up to date (indexed {indexed_at:%Y-%m-%d %H:%M:%S %Z})")
     if status.languages:
         _typer.echo("  Languages:")
         for lang, count in sorted(status.languages.items(), key=lambda x: -x[1]):
@@ -824,6 +832,7 @@ def reset(
     db_files = [
         cocoindex_db_path(project_root),
         target_sqlite_db_path(project_root),
+        index_state_path(project_root),
     ]
     settings_file = project_settings_path(project_root)
 

src/cocoindex_code/daemon.py

@@ -584,7 +584,8 @@ async def _dispatch(
         if isinstance(req, ProjectStatusRequest):
             project = await registry.get_project(req.project_root)
             await project.ensure_indexing_started()
-            return project.get_status()
+            # Status checks the source tree on disk, which is blocking I/O.
+            return await asyncio.to_thread(project.get_status)
 
         if isinstance(req, DaemonStatusRequest):
             now = time.monotonic()

src/cocoindex_code/grep.py

@@ -261,7 +261,7 @@ def _match_file(target: _Target) -> FileMatches | GrepWarning | None:
     if fm is None:
         return None  # binary, prefilter-rejected, or no matches
     matches = sorted(fm.matches, key=lambda m: m.chunks[0].start.byte_offset if m.chunks else 0)
-    return FileMatches(path=target.display, source=fm.ast.source, matches=matches)
+    return FileMatches(path=target.display, source=fm.source.text, matches=matches)
 
 
 class Grep:

src/cocoindex_code/project.py

@@ -4,8 +4,11 @@
 
 import asyncio
 import contextvars
+import hashlib
+import json
 import logging
 import sqlite3
+import time
 import weakref
 from collections.abc import AsyncIterator, Callable, Mapping
 from concurrent.futures import Future, ThreadPoolExecutor
@@ -16,6 +19,7 @@
 from cocoindex.connectors import sqlite as coco_sqlite
 
 from .chunking import CHUNKER_REGISTRY, LoadedChunker
+from .file_walk import build_matcher, iter_included_files
 from .indexer import indexer_main
 from .protocol import (
     IndexingProgress,
@@ -31,6 +35,8 @@
     cocoindex_db_path as _cocoindex_db_path,
 )
 from .settings import (
+    index_state_path,
+    load_project_settings,
     resolve_db_dir,
 )
 from .settings import (
@@ -52,6 +58,54 @@
 _T = TypeVar("_T")
 
 
+def _source_fingerprint(project_root: Path) -> str | None:
+    """Return a stable fingerprint of the indexable source tree and settings.
+
+    The fingerprint deliberately uses paths, mtimes, and sizes rather than file
+    contents: checking freshness must remain much cheaper than embedding every
+    file. It catches additions, removals, renames, and ordinary edits, including
+    changes to the project settings that control which files are indexed.
+    """
+    try:
+        settings = load_project_settings(project_root)
+        matcher = build_matcher(
+            project_root,
+            settings.include_patterns,
+            settings.exclude_patterns,
+            settings.max_file_size,
+        )
+        digest = hashlib.sha256()
+        settings_path = project_root / ".cocoindex_code" / "settings.yml"
+        digest.update(b"settings\x00")
+        digest.update(settings_path.read_bytes())
+        for path, relative_path in iter_included_files(project_root, project_root, matcher):
+            stat = path.stat()
+            digest.update(relative_path.as_posix().encode())
+            digest.update(b"\x00")
+            digest.update(str(stat.st_mtime_ns).encode())
+            digest.update(b"\x00")
+            digest.update(str(stat.st_size).encode())
+            digest.update(b"\x00")
+        return digest.hexdigest()
+    except OSError:
+        # A concurrent file change or unreadable path means freshness is unknown,
+        # never that an index is fresh.
+        return None
+
+
+def _load_index_state(path: Path) -> tuple[str | None, int | None]:
+    """Read a prior successful index fingerprint, tolerating old/corrupt state."""
+    try:
+        value = json.loads(path.read_text())
+        fingerprint = value.get("source_fingerprint")
+        indexed_at_us = value.get("indexed_at_us")
+        if isinstance(fingerprint, str) and isinstance(indexed_at_us, int):
+            return fingerprint, indexed_at_us
+    except (OSError, ValueError, TypeError):
+        pass
+    return None, None
+
+
 class _EmptyContextThreadPoolExecutor(ThreadPoolExecutor):
     """A ``ThreadPoolExecutor`` whose worker threads start with an empty context.
 
@@ -89,6 +143,9 @@ class Project:
     _initial_index_task: asyncio.Task[None] | None
     _initial_index_started: asyncio.Event | None
     _indexing_stats: IndexingProgress | None = None
+    _index_state_path: Path
+    _source_tree_fingerprint: str | None
+    _indexed_at_us: int | None
 
     def close(self) -> None:
         """Close project resources to release file handles (LMDB, SQLite)."""
@@ -131,6 +188,7 @@ async def _run_index_inner(
         on_progress: Callable[[IndexingProgress], None] | None = None,
     ) -> None:
         """Run indexing (lock must already be held)."""
+        completed = False
         try:
             handle = self._app.update()
             async for snapshot in handle.watch():
@@ -148,8 +206,23 @@ async def _run_index_inner(
                     if on_progress is not None:
                         on_progress(progress)
                     await asyncio.sleep(0.1)
+            completed = True
         finally:
             try:
+                # An index that encountered file-processing errors cannot be called
+                # fresh: clear any older state rather than presenting stale data as
+                # current. The filesystem walk belongs off the daemon event loop.
+                no_index_errors = (
+                    self._indexing_stats is None or self._indexing_stats.num_errors == 0
+                )
+                if completed and no_index_errors:
+                    try:
+                        await asyncio.to_thread(self._record_index_state)
+                    except Exception:
+                        logger.exception("Unable to record index freshness state")
+                        self._clear_index_state()
+                else:
+                    self._clear_index_state()
                 if self._clear_mps_cache_after_index:
                     try:
                         await clear_mps_allocator_cache()
@@ -159,6 +232,33 @@ async def _run_index_inner(
                 self._initial_index_done.set()
                 self._indexing_stats = None
 
+    def _record_index_state(self) -> None:
+        """Persist the source-tree fingerprint after a successful index update."""
+        fingerprint = _source_fingerprint(self._project_root)
+        if fingerprint is None:
+            self._clear_index_state()
+            return
+        indexed_at_us = time.time_ns() // 1_000
+        payload = json.dumps(
+            {"source_fingerprint": fingerprint, "indexed_at_us": indexed_at_us},
+            sort_keys=True,
+        )
+        self._index_state_path.parent.mkdir(parents=True, exist_ok=True)
+        tmp_path = self._index_state_path.with_suffix(".tmp")
+        tmp_path.write_text(payload)
+        tmp_path.replace(self._index_state_path)
+        self._source_tree_fingerprint = fingerprint
+        self._indexed_at_us = indexed_at_us
+
+    def _clear_index_state(self) -> None:
+        """Forget freshness state after an incomplete or failed index update."""
+        self._source_tree_fingerprint = None
+        self._indexed_at_us = None
+        try:
+            self._index_state_path.unlink(missing_ok=True)
+        except OSError:
+            logger.warning("Unable to remove stale index freshness state", exc_info=True)
+
     async def ensure_indexing_started(self) -> None:
         """Kick off background indexing and wait until it is active or queued.
 
@@ -283,13 +383,18 @@ def get_status(self) -> ProjectStatusResponse:
 
         is_indexing = self._index_lock.locked()
         progress = self._indexing_stats if is_indexing else None
+        stale = True
+        if index_exists and not is_indexing and self._source_tree_fingerprint is not None:
+            stale = _source_fingerprint(self._project_root) != self._source_tree_fingerprint
         return ProjectStatusResponse(
             indexing=is_indexing,
             total_chunks=total_chunks,
             total_files=total_files,
             languages={lang: cnt for lang, cnt in lang_rows},
             progress=progress,
             index_exists=index_exists,
+            stale=stale,
+            indexed_at_us=self._indexed_at_us,
         )
 
     # ------------------------------------------------------------------
@@ -378,4 +483,8 @@ async def create(
         result._initial_index_done = asyncio.Event()
         result._initial_index_task = None
         result._initial_index_started = None
+        result._index_state_path = index_state_path(project_root)
+        result._source_tree_fingerprint, result._indexed_at_us = _load_index_state(
+            result._index_state_path
+        )
         return result

src/cocoindex_code/protocol.py

@@ -152,6 +152,13 @@ class ProjectStatusResponse(_msgspec.Struct, tag="project_status"):
     languages: dict[str, int]
     progress: IndexingProgress | None = None
     index_exists: bool = True
+    # A status is stale while an update is running, when no successful index
+    # fingerprint exists, or when the current source-tree fingerprint differs
+    # from the one recorded at the end of the last successful update.
+    stale: bool = True
+    # Unix timestamp in microseconds of the successful index update that
+    # produced the fingerprint. Optional for indexes created by older versions.
+    indexed_at_us: int | None = None
 
 
 class DaemonProjectInfo(_msgspec.Struct):