From 512bd3f19d67a265fb4f7a0d2632cc9976ca8417 Mon Sep 17 00:00:00 2001 From: Ethan Date: Tue, 13 Oct 2026 21:20:15 +0700 Subject: [PATCH] feat(mcp): add codebase exploration tools --- README.md | 31 ++- scripts/find_best_models.py | 11 +- src/cocoindex_code/cli.py | 9 + src/cocoindex_code/daemon.py | 3 +- src/cocoindex_code/grep.py | 2 +- src/cocoindex_code/project.py | 109 ++++++++++ src/cocoindex_code/protocol.py | 7 + src/cocoindex_code/server.py | 377 ++++++++++++++++++++++++++++++++- src/cocoindex_code/settings.py | 6 + tests/test_e2e.py | 1 + tests/test_index_freshness.py | 143 +++++++++++++ tests/test_project_indexing.py | 6 + tests/test_protocol.py | 4 + tests/test_server.py | 103 ++++++++- 14 files changed, 798 insertions(+), 14 deletions(-) create mode 100644 tests/test_index_freshness.py diff --git a/README.md b/README.md index 77bae52..6f9abfc 100644 --- a/README.md +++ b/README.md @@ -247,9 +247,9 @@ Once configured, the agent automatically decides when semantic code search is he
MCP Tool Reference -When running as an MCP server (`ccc mcp`), the following tool is exposed: +When running as an MCP server (`ccc mcp`), the following tools are exposed: -**`search`** — Search the codebase using semantic similarity. +**`search`** - Search the codebase using semantic similarity. ``` search( @@ -262,7 +262,32 @@ search( ) ``` -Returns matching code chunks with file path, language, code content, line numbers, and similarity score. +Returns matching code chunks with file path, language, code content, line numbers, similarity score, and `freshness` (`stale` plus the time of the latest successful index). When `stale` is true, search again with `refresh_index=true` before relying on the result. + +**`grep`** - Structural code search by syntax example. It runs locally and needs no index or embedding model. + +``` +grep( + pattern: str, # e.g. r"def \NAME(\(ARGS*\)):" + path: str = ".", # File or directory relative to the project root + languages: list[str] | None = None, # Optional language filter + path_glob: str | None = None, # Optional project-relative glob + limit: int = 100, # Maximum matches returned (1-1000) +) +``` + +Returns exact matched source spans with file paths and line ranges. Paths outside the configured project are rejected. + +**`status`** - Report index health and freshness: whether indexing is active, index/file/chunk counts, indexed-language breakdown, and `freshness`. + +**`codebase_overview`** - Return a compact tree of indexable source files, top-level directory file counts, and language file counts without returning source text. Use it to orient in an unfamiliar repository. + +``` +codebase_overview( + max_depth: int = 2, # Tree depth (1-8) + max_entries: int = 80, # Maximum tree entries (1-500) +) +```
## Manual CLI Usage diff --git a/scripts/find_best_models.py b/scripts/find_best_models.py index 724061d..57a7a12 100644 --- a/scripts/find_best_models.py +++ b/scripts/find_best_models.py @@ -45,7 +45,7 @@ import pandas as pd from datasets import load_dataset from huggingface_hub import HfApi, ModelInfo, dataset_info -from tabulate import tabulate +from tabulate import tabulate # type: ignore[import-untyped] CACHE_DIR = Path(".cocoindex_code/cache") CACHE_FILE = CACHE_DIR / "mteb_results.parquet" @@ -168,10 +168,11 @@ def main() -> None: # Fetch dataset info for freshness (always live check for the date) try: info = dataset_info("mteb/results") - last_update = info.lastModified.strftime("%Y-%m-%d") - report.append( - f"\n> **Data Freshness**: MTEB results dataset last updated on `{last_update}`." - ) + if info.lastModified is not None: + last_update = info.lastModified.strftime("%Y-%m-%d") + report.append( + f"\n> **Data Freshness**: MTEB results dataset last updated on `{last_update}`." + ) except Exception: pass diff --git a/src/cocoindex_code/cli.py b/src/cocoindex_code/cli.py index 7128c47..5a43a38 100644 --- a/src/cocoindex_code/cli.py +++ b/src/cocoindex_code/cli.py @@ -7,6 +7,7 @@ import sys import threading from collections.abc import Callable +from datetime import datetime from pathlib import Path from typing import TYPE_CHECKING, TypeVar @@ -32,6 +33,7 @@ find_parent_with_marker, find_project_root, format_path_for_display, + index_state_path, normalize_input_path, project_settings_path, resolve_db_dir, @@ -210,6 +212,12 @@ def print_index_stats(status: ProjectStatusResponse) -> None: _typer.echo("\nIndex stats:") _typer.echo(f" Chunks: {status.total_chunks}") _typer.echo(f" Files: {status.total_files}") + if not status.indexing: + if status.stale: + _typer.echo(" Freshness: stale (run `ccc index`)") + elif status.indexed_at_us is not None: + indexed_at = datetime.fromtimestamp(status.indexed_at_us / 1_000_000).astimezone() + _typer.echo(f" Freshness: up to date (indexed {indexed_at:%Y-%m-%d %H:%M:%S %Z})") if status.languages: _typer.echo(" Languages:") for lang, count in sorted(status.languages.items(), key=lambda x: -x[1]): @@ -824,6 +832,7 @@ def reset( db_files = [ cocoindex_db_path(project_root), target_sqlite_db_path(project_root), + index_state_path(project_root), ] settings_file = project_settings_path(project_root) diff --git a/src/cocoindex_code/daemon.py b/src/cocoindex_code/daemon.py index 088adee..076885e 100644 --- a/src/cocoindex_code/daemon.py +++ b/src/cocoindex_code/daemon.py @@ -584,7 +584,8 @@ async def _dispatch( if isinstance(req, ProjectStatusRequest): project = await registry.get_project(req.project_root) await project.ensure_indexing_started() - return project.get_status() + # Status checks the source tree on disk, which is blocking I/O. + return await asyncio.to_thread(project.get_status) if isinstance(req, DaemonStatusRequest): now = time.monotonic() diff --git a/src/cocoindex_code/grep.py b/src/cocoindex_code/grep.py index 7a7d416..ae41e29 100644 --- a/src/cocoindex_code/grep.py +++ b/src/cocoindex_code/grep.py @@ -261,7 +261,7 @@ def _match_file(target: _Target) -> FileMatches | GrepWarning | None: if fm is None: return None # binary, prefilter-rejected, or no matches matches = sorted(fm.matches, key=lambda m: m.chunks[0].start.byte_offset if m.chunks else 0) - return FileMatches(path=target.display, source=fm.ast.source, matches=matches) + return FileMatches(path=target.display, source=fm.source.text, matches=matches) class Grep: diff --git a/src/cocoindex_code/project.py b/src/cocoindex_code/project.py index e0c8bf0..a734f04 100644 --- a/src/cocoindex_code/project.py +++ b/src/cocoindex_code/project.py @@ -4,8 +4,11 @@ import asyncio import contextvars +import hashlib +import json import logging import sqlite3 +import time import weakref from collections.abc import AsyncIterator, Callable, Mapping from concurrent.futures import Future, ThreadPoolExecutor @@ -16,6 +19,7 @@ from cocoindex.connectors import sqlite as coco_sqlite from .chunking import CHUNKER_REGISTRY, LoadedChunker +from .file_walk import build_matcher, iter_included_files from .indexer import indexer_main from .protocol import ( IndexingProgress, @@ -31,6 +35,8 @@ cocoindex_db_path as _cocoindex_db_path, ) from .settings import ( + index_state_path, + load_project_settings, resolve_db_dir, ) from .settings import ( @@ -52,6 +58,54 @@ _T = TypeVar("_T") +def _source_fingerprint(project_root: Path) -> str | None: + """Return a stable fingerprint of the indexable source tree and settings. + + The fingerprint deliberately uses paths, mtimes, and sizes rather than file + contents: checking freshness must remain much cheaper than embedding every + file. It catches additions, removals, renames, and ordinary edits, including + changes to the project settings that control which files are indexed. + """ + try: + settings = load_project_settings(project_root) + matcher = build_matcher( + project_root, + settings.include_patterns, + settings.exclude_patterns, + settings.max_file_size, + ) + digest = hashlib.sha256() + settings_path = project_root / ".cocoindex_code" / "settings.yml" + digest.update(b"settings\x00") + digest.update(settings_path.read_bytes()) + for path, relative_path in iter_included_files(project_root, project_root, matcher): + stat = path.stat() + digest.update(relative_path.as_posix().encode()) + digest.update(b"\x00") + digest.update(str(stat.st_mtime_ns).encode()) + digest.update(b"\x00") + digest.update(str(stat.st_size).encode()) + digest.update(b"\x00") + return digest.hexdigest() + except OSError: + # A concurrent file change or unreadable path means freshness is unknown, + # never that an index is fresh. + return None + + +def _load_index_state(path: Path) -> tuple[str | None, int | None]: + """Read a prior successful index fingerprint, tolerating old/corrupt state.""" + try: + value = json.loads(path.read_text()) + fingerprint = value.get("source_fingerprint") + indexed_at_us = value.get("indexed_at_us") + if isinstance(fingerprint, str) and isinstance(indexed_at_us, int): + return fingerprint, indexed_at_us + except (OSError, ValueError, TypeError): + pass + return None, None + + class _EmptyContextThreadPoolExecutor(ThreadPoolExecutor): """A ``ThreadPoolExecutor`` whose worker threads start with an empty context. @@ -89,6 +143,9 @@ class Project: _initial_index_task: asyncio.Task[None] | None _initial_index_started: asyncio.Event | None _indexing_stats: IndexingProgress | None = None + _index_state_path: Path + _source_tree_fingerprint: str | None + _indexed_at_us: int | None def close(self) -> None: """Close project resources to release file handles (LMDB, SQLite).""" @@ -131,6 +188,7 @@ async def _run_index_inner( on_progress: Callable[[IndexingProgress], None] | None = None, ) -> None: """Run indexing (lock must already be held).""" + completed = False try: handle = self._app.update() async for snapshot in handle.watch(): @@ -148,8 +206,23 @@ async def _run_index_inner( if on_progress is not None: on_progress(progress) await asyncio.sleep(0.1) + completed = True finally: try: + # An index that encountered file-processing errors cannot be called + # fresh: clear any older state rather than presenting stale data as + # current. The filesystem walk belongs off the daemon event loop. + no_index_errors = ( + self._indexing_stats is None or self._indexing_stats.num_errors == 0 + ) + if completed and no_index_errors: + try: + await asyncio.to_thread(self._record_index_state) + except Exception: + logger.exception("Unable to record index freshness state") + self._clear_index_state() + else: + self._clear_index_state() if self._clear_mps_cache_after_index: try: await clear_mps_allocator_cache() @@ -159,6 +232,33 @@ async def _run_index_inner( self._initial_index_done.set() self._indexing_stats = None + def _record_index_state(self) -> None: + """Persist the source-tree fingerprint after a successful index update.""" + fingerprint = _source_fingerprint(self._project_root) + if fingerprint is None: + self._clear_index_state() + return + indexed_at_us = time.time_ns() // 1_000 + payload = json.dumps( + {"source_fingerprint": fingerprint, "indexed_at_us": indexed_at_us}, + sort_keys=True, + ) + self._index_state_path.parent.mkdir(parents=True, exist_ok=True) + tmp_path = self._index_state_path.with_suffix(".tmp") + tmp_path.write_text(payload) + tmp_path.replace(self._index_state_path) + self._source_tree_fingerprint = fingerprint + self._indexed_at_us = indexed_at_us + + def _clear_index_state(self) -> None: + """Forget freshness state after an incomplete or failed index update.""" + self._source_tree_fingerprint = None + self._indexed_at_us = None + try: + self._index_state_path.unlink(missing_ok=True) + except OSError: + logger.warning("Unable to remove stale index freshness state", exc_info=True) + async def ensure_indexing_started(self) -> None: """Kick off background indexing and wait until it is active or queued. @@ -283,6 +383,9 @@ def get_status(self) -> ProjectStatusResponse: is_indexing = self._index_lock.locked() progress = self._indexing_stats if is_indexing else None + stale = True + if index_exists and not is_indexing and self._source_tree_fingerprint is not None: + stale = _source_fingerprint(self._project_root) != self._source_tree_fingerprint return ProjectStatusResponse( indexing=is_indexing, total_chunks=total_chunks, @@ -290,6 +393,8 @@ def get_status(self) -> ProjectStatusResponse: languages={lang: cnt for lang, cnt in lang_rows}, progress=progress, index_exists=index_exists, + stale=stale, + indexed_at_us=self._indexed_at_us, ) # ------------------------------------------------------------------ @@ -378,4 +483,8 @@ async def create( result._initial_index_done = asyncio.Event() result._initial_index_task = None result._initial_index_started = None + result._index_state_path = index_state_path(project_root) + result._source_tree_fingerprint, result._indexed_at_us = _load_index_state( + result._index_state_path + ) return result diff --git a/src/cocoindex_code/protocol.py b/src/cocoindex_code/protocol.py index 77625bd..ac727ca 100644 --- a/src/cocoindex_code/protocol.py +++ b/src/cocoindex_code/protocol.py @@ -152,6 +152,13 @@ class ProjectStatusResponse(_msgspec.Struct, tag="project_status"): languages: dict[str, int] progress: IndexingProgress | None = None index_exists: bool = True + # A status is stale while an update is running, when no successful index + # fingerprint exists, or when the current source-tree fingerprint differs + # from the one recorded at the end of the last successful update. + stale: bool = True + # Unix timestamp in microseconds of the successful index update that + # produced the fingerprint. Optional for indexes created by older versions. + indexed_at_us: int | None = None class DaemonProjectInfo(_msgspec.Struct): diff --git a/src/cocoindex_code/server.py b/src/cocoindex_code/server.py index 0f04b8a..20935fe 100644 --- a/src/cocoindex_code/server.py +++ b/src/cocoindex_code/server.py @@ -12,13 +12,24 @@ import asyncio import json import os +import threading +from collections import Counter +from datetime import UTC, datetime from pathlib import Path +from cocoindex.ops.text import detect_code_language from mcp.server.mcpserver import MCPServer from pydantic import BaseModel, Field from ._version import __version__ -from .settings import DaemonSettings, load_user_settings, user_settings_root_note +from .file_walk import build_matcher, iter_included_files +from .protocol import ProjectStatusResponse +from .settings import ( + DaemonSettings, + load_project_settings, + load_user_settings, + user_settings_root_note, +) _MCP_INSTRUCTIONS = ( "Code search and codebase understanding tools." @@ -46,6 +57,16 @@ class CodeChunkResult(BaseModel): score: float = Field(description="Similarity score (0-1, higher is better)") +class IndexFreshnessModel(BaseModel): + """Whether the source tree still matches the latest successful index.""" + + stale: bool = Field(description="True when a refresh is needed or freshness is unknown") + indexed_at: str | None = Field( + default=None, + description="UTC time of the successful index update, or null when unavailable", + ) + + class SearchResultModel(BaseModel): """Result from search tool.""" @@ -53,6 +74,66 @@ class SearchResultModel(BaseModel): results: list[CodeChunkResult] = Field(default_factory=list) total_returned: int = Field(default=0) offset: int = Field(default=0) + freshness: IndexFreshnessModel = Field(default_factory=lambda: IndexFreshnessModel(stale=True)) + message: str | None = None + + +class IndexStatusModel(BaseModel): + """Index health, size, and freshness for the current project.""" + + indexing: bool + index_exists: bool + total_chunks: int + total_files: int + languages: dict[str, int] = Field( + default_factory=dict, + description="Indexed chunk counts by programming language", + ) + freshness: IndexFreshnessModel + message: str | None = None + + +class StructuralMatchResult(BaseModel): + """One structural-code match.""" + + file_path: str = Field(description="Path relative to the project root") + language: str | None = Field(description="Detected language, if available") + content: str = Field(description="The exact source span matching the structural pattern") + start_line: int = Field(description="Starting line number (1-indexed)") + end_line: int = Field(description="Ending line number (1-indexed)") + + +class GrepResultModel(BaseModel): + """Result from structural grep.""" + + success: bool + matches: list[StructuralMatchResult] = Field(default_factory=list) + total_matches: int = Field(default=0) + warnings: list[str] = Field(default_factory=list) + truncated: bool = False + message: str | None = None + + +class DirectorySummary(BaseModel): + """Indexed-source file count for a top-level directory.""" + + path: str + files: int + + +class CodebaseOverviewResult(BaseModel): + """A compact source-tree overview for agent orientation.""" + + success: bool + total_source_files: int = 0 + languages: dict[str, int] = Field( + default_factory=dict, + description="Current source file counts by detected language", + ) + top_level_directories: list[DirectorySummary] = Field(default_factory=list) + tree: list[str] = Field(default_factory=list) + truncated: bool = False + freshness: IndexFreshnessModel = Field(default_factory=lambda: IndexFreshnessModel(stale=True)) message: str | None = None @@ -63,6 +144,145 @@ def _with_note(message: str | None, note: str | None) -> str | None: return f"{message}\nNote: {note}" if message else f"Note: {note}" +def _freshness_from_status(status: ProjectStatusResponse) -> IndexFreshnessModel: + """Translate daemon timestamp fields into a stable MCP-facing shape.""" + indexed_at_us = getattr(status, "indexed_at_us", None) + indexed_at = None + if indexed_at_us is not None: + indexed_at = datetime.fromtimestamp(indexed_at_us / 1_000_000, UTC).isoformat() + return IndexFreshnessModel(stale=getattr(status, "stale", True), indexed_at=indexed_at) + + +def _status_model(status: ProjectStatusResponse, note: str | None = None) -> IndexStatusModel: + """Build the public MCP status model from the daemon protocol response.""" + return IndexStatusModel( + indexing=status.indexing, + index_exists=status.index_exists, + total_chunks=status.total_chunks, + total_files=status.total_files, + languages=status.languages, + freshness=_freshness_from_status(status), + message=_with_note(None, note), + ) + + +def _resolve_project_path(project_root: Path, path: str) -> Path: + """Resolve an MCP path and reject paths outside the configured project.""" + root = project_root.resolve() + target = (root / path).resolve() + if not target.is_relative_to(root): + raise ValueError("path must be within the project root") + if not target.exists(): + raise ValueError(f"path not found: {path}") + return target + + +def _language_for_path(path: Path, overrides: dict[str, str]) -> str: + """Match the indexer's language-detection behavior for MCP-only tools.""" + return overrides.get(path.suffix) or detect_code_language(filename=path.name) or "text" + + +def _run_structural_grep( + project_root: Path, + pattern: str, + path: str, + languages: list[str] | None, + path_glob: str | None, + limit: int, +) -> tuple[list[StructuralMatchResult], int, list[str], bool]: + """Run local structural grep and retain only the requested number of matches.""" + from . import grep as _grep + + target = _resolve_project_path(project_root, path) + settings = load_project_settings(project_root) + overrides = {f".{entry.ext}": entry.lang for entry in settings.language_overrides} + grep_run = _grep.Grep( + _grep.GrepRequest( + pattern=pattern, + root=target, + languages=frozenset(language.lower() for language in languages) if languages else None, + path_glob=path_glob, + ) + ) + matches: list[StructuralMatchResult] = [] + warnings: list[str] = [] + total_matches = 0 + lock = threading.Lock() + + def emit(item: _grep.FileMatches | _grep.GrepWarning) -> None: + nonlocal total_matches + if isinstance(item, _grep.GrepWarning): + with lock: + warnings.append(item.message) + return + relative_path = Path(item.path).resolve().relative_to(project_root.resolve()).as_posix() + language = _language_for_path(Path(relative_path), overrides) + file_matches: list[StructuralMatchResult] = [] + for match in item.matches: + if not match.chunks: + continue + chunk = match.chunks[0] + file_matches.append( + StructuralMatchResult( + file_path=relative_path, + language=language, + content=item.source[chunk.start.char_offset : chunk.end.char_offset], + start_line=chunk.start.line, + end_line=chunk.end.line, + ) + ) + with lock: + total_matches += len(file_matches) + remaining = limit - len(matches) + if remaining > 0: + matches.extend(file_matches[:remaining]) + + grep_run.run(emit) + matches.sort(key=lambda match: (match.file_path, match.start_line, match.end_line)) + return matches, total_matches, sorted(warnings), grep_run.unusable + + +def _build_codebase_overview( + project_root: Path, max_depth: int, max_entries: int +) -> tuple[int, dict[str, int], list[DirectorySummary], list[str], bool]: + """Summarize the currently indexable source tree without reading file content.""" + settings = load_project_settings(project_root) + matcher = build_matcher( + project_root, + settings.include_patterns, + settings.exclude_patterns, + settings.max_file_size, + ) + overrides = {f".{entry.ext}": entry.lang for entry in settings.language_overrides} + languages: Counter[str] = Counter() + top_level: Counter[str] = Counter() + tree_entries: set[str] = set() + total_files = 0 + + for path, relative_path in iter_included_files(project_root, project_root, matcher): + total_files += 1 + languages[_language_for_path(path, overrides)] += 1 + parts = relative_path.parts + if len(parts) > 1: + top_level[parts[0]] += 1 + for depth in range(1, min(len(parts), max_depth) + 1): + is_file = depth == len(parts) + entry = "/".join(parts[:depth]) + tree_entries.add(entry if is_file else f"{entry}/") + if len(parts) > max_depth: + tree_entries.add(f"{'/'.join(parts[:max_depth])}/…") + + tree = sorted(tree_entries) + truncated = len(tree) > max_entries + return ( + total_files, + dict(sorted(languages.items())), + [DirectorySummary(path=path, files=count) for path, count in sorted(top_level.items())], + tree[:max_entries], + truncated, + ) + + # === Daemon-backed MCP server factory === @@ -136,7 +356,17 @@ async def search( try: note = user_settings_root_note(Path(project_root), Path.cwd()) if refresh_index: - await loop.run_in_executor(None, lambda: _client.index(project_root)) + index_response = await loop.run_in_executor( + None, lambda: _client.index(project_root) + ) + if not index_response.success: + return SearchResultModel( + success=False, + message=_with_note( + f"Index refresh failed: {index_response.message or 'unknown error'}", + note, + ), + ) resp = await loop.run_in_executor( None, lambda: _client.search( @@ -148,6 +378,18 @@ async def search( offset=offset, ), ) + # A separate status request adds a source-tree freshness check without + # changing the search protocol or forcing a refresh on every query. + try: + status = await loop.run_in_executor( + None, lambda: _client.project_status(project_root) + ) + freshness = _freshness_from_status(status) + except Exception: + # Never discard useful search hits merely because observability + # failed. Conservatively tell the agent it must refresh to trust + # the result. + freshness = IndexFreshnessModel(stale=True) return SearchResultModel( success=resp.success, results=[ @@ -163,6 +405,7 @@ async def search( ], total_returned=resp.total_returned, offset=resp.offset, + freshness=freshness, message=_with_note(resp.message, note), ) except Exception as e: @@ -170,6 +413,136 @@ async def search( success=False, message=_with_note(f"Query failed: {e!s}", note) ) + @mcp.tool( + name="grep", + description=( + "Structural code search by syntax example. Use this when you know the code shape " + "or an exact API call/definition to find. It does not require an index or embeddings. " + "Use \\ metavariables, for example `def \\NAME(\\(ARGS*\\)):` or `foo(\\(ARGS*\\))`." + ), + ) + async def grep( + pattern: str = Field(description="Structural pattern to match, using \\ metavariables."), + path: str = Field( + default=".", + description=( + "Project-relative file or directory to search. It must stay inside the project." + ), + ), + languages: list[str] | None = Field( + default=None, + description="Optional programming-language filter, for example ['python', 'rust'].", + ), + path_glob: str | None = Field( + default=None, + description="Optional project-relative glob filter, for example 'src/**'.", + ), + limit: int = Field(default=100, ge=1, le=1000, description="Maximum matches to return."), + ) -> GrepResultModel: + """Run local, AST-aware structural search without using the vector index.""" + loop = asyncio.get_event_loop() + note = user_settings_root_note(Path(project_root), Path.cwd()) + try: + matches, total_matches, warnings, unusable = await loop.run_in_executor( + None, + lambda: _run_structural_grep( + Path(project_root), pattern, path, languages, path_glob, limit + ), + ) + message = None + if unusable: + message = "Pattern did not compile for any of the matching languages." + return GrepResultModel( + success=not unusable, + matches=matches, + total_matches=total_matches, + warnings=warnings, + truncated=total_matches > len(matches), + message=_with_note(message, note), + ) + except Exception as e: + return GrepResultModel( + success=False, message=_with_note(f"Structural search failed: {e!s}", note) + ) + + @mcp.tool( + name="status", + description=( + "Show index size, language breakdown, and freshness. Check this before trusting " + "a search run with refresh_index=false." + ), + ) + async def status() -> IndexStatusModel: + """Get index status and whether the source tree has changed since indexing.""" + from . import client as _client + + loop = asyncio.get_event_loop() + note = user_settings_root_note(Path(project_root), Path.cwd()) + try: + daemon_status = await loop.run_in_executor( + None, lambda: _client.project_status(project_root) + ) + return _status_model(daemon_status, note) + except Exception as e: + return IndexStatusModel( + indexing=False, + index_exists=False, + total_chunks=0, + total_files=0, + freshness=IndexFreshnessModel(stale=True), + message=_with_note(f"Status failed: {e!s}", note), + ) + + @mcp.tool( + name="codebase_overview", + description=( + "Return a compact source-tree and language overview for orienting in an unfamiliar " + "codebase. It lists no file contents and does not require a completed index." + ), + ) + async def codebase_overview( + max_depth: int = Field( + default=2, + ge=1, + le=8, + description="Maximum directory depth represented in the tree.", + ), + max_entries: int = Field( + default=80, + ge=1, + le=500, + description="Maximum tree entries to return.", + ), + ) -> CodebaseOverviewResult: + """Summarize indexable files and languages without spending tokens on source text.""" + from . import client as _client + + loop = asyncio.get_event_loop() + note = user_settings_root_note(Path(project_root), Path.cwd()) + try: + daemon_status = await loop.run_in_executor( + None, lambda: _client.project_status(project_root) + ) + total_files, languages, directories, tree, truncated = await loop.run_in_executor( + None, lambda: _build_codebase_overview(Path(project_root), max_depth, max_entries) + ) + return CodebaseOverviewResult( + success=True, + total_source_files=total_files, + languages=languages, + top_level_directories=directories, + tree=tree, + truncated=truncated, + freshness=_freshness_from_status(daemon_status), + message=_with_note(None, note), + ) + except Exception as e: + return CodebaseOverviewResult( + success=False, + freshness=IndexFreshnessModel(stale=True), + message=_with_note(f"Codebase overview failed: {e!s}", note), + ) + return mcp diff --git a/src/cocoindex_code/settings.py b/src/cocoindex_code/settings.py index 0aa1faf..d082f84 100644 --- a/src/cocoindex_code/settings.py +++ b/src/cocoindex_code/settings.py @@ -321,6 +321,12 @@ def _reset_host_path_mapping_cache() -> None: _TARGET_SQLITE_DB_NAME = "target_sqlite.db" _COCOINDEX_DB_NAME = "cocoindex.db" +_INDEX_STATE_NAME = "index_state.json" + + +def index_state_path(project_root: Path) -> Path: + """Return the path to the derived index-freshness state for a project.""" + return resolve_db_dir(project_root) / _INDEX_STATE_NAME def target_sqlite_db_path(project_root: Path) -> Path: diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 8d6d4f8..0b9de8b 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -244,6 +244,7 @@ def test_session_reset_databases(e2e_project: Path) -> None: # DB files should be gone assert not (e2e_project / ".cocoindex_code" / "cocoindex.db").exists() assert not (e2e_project / ".cocoindex_code" / "target_sqlite.db").exists() + assert not (e2e_project / ".cocoindex_code" / "index_state.json").exists() # Re-index — project is still initialized, just databases gone. Same daemon, so # the reset must have released the project's LMDB env. diff --git a/tests/test_index_freshness.py b/tests/test_index_freshness.py new file mode 100644 index 0000000..a84e591 --- /dev/null +++ b/tests/test_index_freshness.py @@ -0,0 +1,143 @@ +"""Index-freshness tests. + +A project reports stale only when its source tree no longer matches the +fingerprint recorded by the last successful index pass. These use +``Project.create`` with a stub embedder so no embedding model is loaded, and +drive a real index run. +""" + +from __future__ import annotations + +import gc +from pathlib import Path + +from conftest import StubEmbedder + +from cocoindex_code.project import Project +from cocoindex_code.settings import ( + ProjectSettings, + index_state_path, + save_project_settings, +) + + +def _settings() -> ProjectSettings: + return ProjectSettings(include_patterns=["**/*.py"], exclude_patterns=["**/.cocoindex_code"]) + + +async def _create_project(project_root: Path) -> Project: + save_project_settings(project_root, _settings()) + return await Project.create(project_root, StubEmbedder(), indexing_params={}, query_params={}) + + +async def _reopen(project_root: Path) -> Project: + """Load a project again, as a daemon restart does.""" + return await _create_project(project_root) + + +def _close_and_release(project: Project) -> None: + """Release a project's handles. The core CocoIndex environment is process + global and only freed once the owning Project is garbage collected, so the + caller must not keep a reference.""" + project.close() + + +async def test_status_is_fresh_after_a_successful_index(tmp_path: Path) -> None: + (tmp_path / "main.py").write_text("def one():\n return 1\n") + project = await _create_project(tmp_path) + + await project.run_index() + + status = project.get_status() + assert status.index_exists is True + assert status.stale is False + assert status.indexed_at_us is not None + assert index_state_path(tmp_path).is_file() + + +async def test_editing_a_file_marks_the_index_stale(tmp_path: Path) -> None: + source = tmp_path / "main.py" + source.write_text("def one():\n return 1\n") + project = await _create_project(tmp_path) + await project.run_index() + assert project.get_status().stale is False + + source.write_text("def one():\n return 1\n\n\ndef two():\n return 2\n") + + assert project.get_status().stale is True + + await project.run_index() + assert project.get_status().stale is False + + +async def test_adding_an_indexable_file_marks_the_index_stale(tmp_path: Path) -> None: + (tmp_path / "main.py").write_text("def one():\n return 1\n") + project = await _create_project(tmp_path) + await project.run_index() + + (tmp_path / "extra.py").write_text("def extra():\n return 0\n") + + assert project.get_status().stale is True + + +async def test_editing_project_settings_marks_the_index_stale(tmp_path: Path) -> None: + """The include patterns decide what the index covers, so they are fingerprinted.""" + (tmp_path / "main.py").write_text("def one():\n return 1\n") + (tmp_path / "notes.md").write_text("# notes\n") + project = await _create_project(tmp_path) + await project.run_index() + + save_project_settings( + tmp_path, + ProjectSettings( + include_patterns=["**/*.py", "**/*.md"], exclude_patterns=["**/.cocoindex_code"] + ), + ) + + assert project.get_status().stale is True + + +async def test_freshness_survives_reloading_the_project(tmp_path: Path) -> None: + """Freshness lives on disk, so a daemon restart does not report a false stale.""" + (tmp_path / "main.py").write_text("def one():\n return 1\n") + project = await _create_project(tmp_path) + await project.run_index() + indexed_at_us = project.get_status().indexed_at_us + _close_and_release(project) + del project + gc.collect() + + reopened = await _reopen(tmp_path) + + status = reopened.get_status() + assert status.stale is False + assert status.indexed_at_us == indexed_at_us + + +async def test_missing_state_reports_stale(tmp_path: Path) -> None: + """An index built by an older version has no state file: freshness is unknown.""" + (tmp_path / "main.py").write_text("def one():\n return 1\n") + project = await _create_project(tmp_path) + await project.run_index() + index_state_path(tmp_path).unlink() + _close_and_release(project) + del project + gc.collect() + + reopened = await _reopen(tmp_path) + + assert reopened.get_status().stale is True + + +async def test_unreadable_state_reports_stale(tmp_path: Path) -> None: + (tmp_path / "main.py").write_text("def one():\n return 1\n") + project = await _create_project(tmp_path) + await project.run_index() + index_state_path(tmp_path).write_text("not json") + _close_and_release(project) + del project + gc.collect() + + reopened = await _reopen(tmp_path) + + assert reopened.get_status().stale is True diff --git a/tests/test_project_indexing.py b/tests/test_project_indexing.py index 0db593a..5ae7908 100644 --- a/tests/test_project_indexing.py +++ b/tests/test_project_indexing.py @@ -63,6 +63,12 @@ def __init__( self._initial_index_started = None self._indexing_stats = None + def _record_index_state(self) -> None: + """Controlled lifecycle tests have no filesystem-backed project.""" + + def _clear_index_state(self) -> None: + """Controlled lifecycle tests have no filesystem-backed project.""" + async def test_projects_can_prepare_indexes_concurrently() -> None: release = asyncio.Event() diff --git a/tests/test_protocol.py b/tests/test_protocol.py index 923cea2..fdb3bac 100644 --- a/tests/test_protocol.py +++ b/tests/test_protocol.py @@ -318,11 +318,15 @@ def test_encode_decode_project_status_without_progress() -> None: total_chunks=50, total_files=10, languages={"python": 50}, + stale=False, + indexed_at_us=1_700_000_000_000_000, ) data = encode_response(resp) decoded = decode_response(data) assert isinstance(decoded, ProjectStatusResponse) assert decoded.progress is None + assert decoded.stale is False + assert decoded.indexed_at_us == 1_700_000_000_000_000 def test_all_response_types_round_trip() -> None: diff --git a/tests/test_server.py b/tests/test_server.py index 84e4f53..2f9e013 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -5,8 +5,9 @@ from cocoindex_code import client as daemon_client from cocoindex_code._version import __version__ -from cocoindex_code.protocol import SearchResponse +from cocoindex_code.protocol import ProjectStatusResponse, SearchResponse from cocoindex_code.server import create_mcp_server +from cocoindex_code.settings import default_project_settings, save_project_settings async def test_mcp_server_uses_v2_protocol(monkeypatch: pytest.MonkeyPatch) -> None: @@ -15,6 +16,18 @@ async def test_mcp_server_uses_v2_protocol(monkeypatch: pytest.MonkeyPatch) -> N "search", lambda **kwargs: SearchResponse(success=True, offset=kwargs["offset"]), ) + monkeypatch.setattr( + daemon_client, + "project_status", + lambda _root: ProjectStatusResponse( + indexing=False, + total_chunks=0, + total_files=0, + languages={}, + stale=False, + indexed_at_us=1_700_000_000_000_000, + ), + ) server = create_mcp_server(".") async with Client(server, raise_exceptions=True) as client: @@ -24,14 +37,100 @@ async def test_mcp_server_uses_v2_protocol(monkeypatch: pytest.MonkeyPatch) -> N {"query": "authentication", "refresh_index": False}, ) - assert [tool.name for tool in tools.tools] == ["search"] + assert [tool.name for tool in tools.tools] == [ + "search", + "grep", + "status", + "codebase_overview", + ] assert result.structured_content == { "success": True, "results": [], "total_returned": 0, "offset": 0, + "freshness": {"stale": False, "indexed_at": "2023-11-14T22:13:20+00:00"}, + "message": None, + } + + +async def test_mcp_grep_searches_project_files_without_daemon(tmp_path: Path) -> None: + save_project_settings(tmp_path, default_project_settings()) + (tmp_path / "src").mkdir() + (tmp_path / "src" / "example.py").write_text("def greet(name):\n return name\n") + + async with Client(create_mcp_server(str(tmp_path)), raise_exceptions=True) as client: + result = await client.call_tool("grep", {"pattern": r"def \NAME(\(ARGS*\)):"}) + + assert result.structured_content == { + "success": True, + "matches": [ + { + "file_path": "src/example.py", + "language": "python", + "content": "def greet(name):", + "start_line": 1, + "end_line": 1, + } + ], + "total_matches": 1, + "warnings": [], + "truncated": False, + "message": None, + } + + +async def test_mcp_grep_rejects_a_path_outside_the_project(tmp_path: Path) -> None: + save_project_settings(tmp_path, default_project_settings()) + + async with Client(create_mcp_server(str(tmp_path)), raise_exceptions=True) as client: + result = await client.call_tool("grep", {"pattern": "x", "path": ".."}) + + assert result.structured_content is not None + assert result.structured_content["success"] is False + assert "within the project root" in result.structured_content["message"] + + +async def test_mcp_status_and_overview_report_freshness( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + save_project_settings(tmp_path, default_project_settings()) + (tmp_path / "src").mkdir() + (tmp_path / "src" / "api.py").write_text("def endpoint():\n pass\n") + (tmp_path / "README.md").write_text("# Example\n") + monkeypatch.setattr( + daemon_client, + "project_status", + lambda _root: ProjectStatusResponse( + indexing=False, + index_exists=True, + total_chunks=12, + total_files=2, + languages={"python": 9, "markdown": 3}, + stale=True, + indexed_at_us=1_700_000_000_000_000, + ), + ) + + async with Client(create_mcp_server(str(tmp_path)), raise_exceptions=True) as client: + status = await client.call_tool("status", {}) + overview = await client.call_tool("codebase_overview", {"max_depth": 2}) + + assert status.structured_content == { + "indexing": False, + "index_exists": True, + "total_chunks": 12, + "total_files": 2, + "languages": {"python": 9, "markdown": 3}, + "freshness": {"stale": True, "indexed_at": "2023-11-14T22:13:20+00:00"}, "message": None, } + assert overview.structured_content is not None + assert overview.structured_content["success"] is True + assert overview.structured_content["total_source_files"] == 2 + assert overview.structured_content["languages"] == {"markdown": 1, "python": 1} + assert overview.structured_content["top_level_directories"] == [{"path": "src", "files": 1}] + assert overview.structured_content["tree"] == ["README.md", "src/", "src/api.py"] + assert overview.structured_content["freshness"]["stale"] is True async def test_mcp_server_reports_own_version() -> None: