diff --git a/docs/architecture/code-index-architecture.md b/docs/architecture/code-index-architecture.md index ae3159398b..6835c267d1 100644 --- a/docs/architecture/code-index-architecture.md +++ b/docs/architecture/code-index-architecture.md @@ -346,6 +346,11 @@ the C++ tier; `loom code scip-delta` runs one delta pass. Where both tiers hold the same row, reads prefer the compiler row, and every answer names the tier it read (`scip` or `treesitter`). +Exact code grep answers can carry the caller's readable code-linked lessons for +the matched files. Lessons are read after cached match data, within the request's +remaining deadline. Presence queries and uncovered paths carry none; +`include_lessons=false` skips the read. + Python compiler conversion keeps callable and class descriptors distinct from parameters, type parameters and fields. Call edges require a containing function span; a reference at module or class scope does not belong to the preceding function. diff --git a/docs/reference/operations/code.md b/docs/reference/operations/code.md index 93fd7f90d3..7f1fdc1cc4 100644 --- a/docs/reference/operations/code.md +++ b/docs/reference/operations/code.md @@ -710,6 +710,8 @@ Exact-match lookup over the indexed corpus. | Input field | Type | Default | Meaning | | --- | --- | --- | --- | +| `include_lessons` | `bool` | `True` | Attach `lessons`: the memories about the answer's top files (for code.context_pack, a `task` briefing's only). | +| `lesson_namespaces` | `list[str]` | *(computed)* | The memory namespaces `lessons` reads besides the repository's team rules (over MCP the caller's memory home, filled in); one the caller cannot read is skipped. | | `org` | `str` | `''` | The tenant (first element of the identity tuple). | | `namespace` | `str` | `''` | The namespace the op reads or writes. | | `agent` | `str` | `'-'` | The writing or reading agent; `'-'` or `'*'` means any agent where the op allows it. | @@ -733,6 +735,8 @@ Exact-match lookup over the indexed corpus. | Output field | Type | Meaning | | --- | --- | --- | +| `lessons` | `list[dict[str, Any]]` | The memories about the answer's top files, best first, capped at `code_result_lessons_tokens`; each row {path, memory_key, namespace, excerpt, code_state}, with `changed_since` and `supersedes_key` when they apply. | +| `lessons_left_out` | `int` | How many rows about those files the lessons' token cap or a cut left out, with the rows another row read for the answer supersedes. | | `matches` | `list[dict[str, Any]]` | {path, line, text, scope, scope_start_line, scope_end_line} per matching LINE, path/line ordered. scope_* is the span of the enclosing unit named by `scope` (0/0 when the line sits in no unit) — enough to decide whether to fetch the body… | | `next_cursor` | `str` | Continuation: cursor for the next page of matching lines, "" when the extracted set is exhausted. has_more mirrors it. | | `has_more` | `bool` | True when `next_cursor` is set. | diff --git a/sdk/python/loom/ops.py b/sdk/python/loom/ops.py index 9606382061..036811d7d9 100644 --- a/sdk/python/loom/ops.py +++ b/sdk/python/loom/ops.py @@ -6537,6 +6537,8 @@ class CodeFirstBuildOutput(TypedDict, total=False): class CodeGrepInput(TypedDict, total=False): "Exact-match lookup over the indexed corpus. Unlike shell grep, the" + include_lessons: NotRequired[bool] + lesson_namespaces: NotRequired[list[str]] org: NotRequired[str] namespace: NotRequired[str] agent: NotRequired[str] @@ -6562,6 +6564,8 @@ class CodeGrepInput(TypedDict, total=False): class CodeGrepBody(TypedDict, total=False): "Exact-match lookup over the indexed corpus. Unlike shell grep, the" + include_lessons: NotRequired[bool] + lesson_namespaces: NotRequired[list[str]] org: NotRequired[str] namespace: NotRequired[str] agent: NotRequired[str] @@ -6585,6 +6589,8 @@ class CodeGrepBody(TypedDict, total=False): class CodeGrepOutput(TypedDict, total=False): + lessons: NotRequired[list[dict[str, Any]]] + lessons_left_out: NotRequired[int] matches: Required[list[dict[str, Any]]] next_cursor: NotRequired[str] has_more: NotRequired[bool] diff --git a/sdk/typescript/src/generated/ops.ts b/sdk/typescript/src/generated/ops.ts index 8949c864b3..c88b34f554 100644 --- a/sdk/typescript/src/generated/ops.ts +++ b/sdk/typescript/src/generated/ops.ts @@ -864,6 +864,8 @@ export interface CodeFirstBuildOutput { } export interface CodeGrepInput { + include_lessons?: boolean; + lesson_namespaces?: string[]; org?: string; namespace?: string; agent?: string; @@ -887,6 +889,8 @@ export interface CodeGrepInput { } export interface CodeGrepOutput { + lessons?: Array>; + lessons_left_out?: number; matches: Array>; next_cursor?: string; has_more?: boolean; diff --git a/server/code/api/models_grep.py b/server/code/api/models_grep.py index cbde25fac9..e152d79b6e 100644 --- a/server/code/api/models_grep.py +++ b/server/code/api/models_grep.py @@ -17,9 +17,12 @@ from server.code.api.models_common import ( _CodeIdentity, + _drop_empty_lessons, _drop_empty_presence_keys, _drop_unfired_symbol_blocks, _lead_namespaces, + _LessonsOut, + _ResultLessons, _trim_namespaces, ) from server.operations.base import MCP_HIDDEN @@ -29,7 +32,7 @@ # ============================================================ -class CodeGrepInput(_CodeIdentity): +class CodeGrepInput(_CodeIdentity, _ResultLessons): """Exact-match lookup over the indexed corpus. Unlike shell grep, the answer arrives WITH what raw grep can't attach: the enclosing symbol per hit, one indexed round-trip, and a hit list that @@ -129,7 +132,7 @@ def effective_limit(self) -> int: return min(self.limit or 20, 200) -class CodeGrepOutput(BaseModel): +class CodeGrepOutput(_LessonsOut): @model_serializer(mode="wrap") def _drop_unfired_mode_fields(self, handler: SerializerFunctionWrapHandler) -> dict[str, Any]: # the presence block rides only when it fired, the neighbours block @@ -142,7 +145,7 @@ def _drop_unfired_mode_fields(self, handler: SerializerFunctionWrapHandler) -> d for k in ("partial", "partial_reason", "fallback"): if not data.get(k): data.pop(k, None) - return data + return _drop_empty_lessons(data) # {path, line, text, scope, scope_start_line, scope_end_line} per matching # LINE, path/line ordered. scope_* is the span of the enclosing unit named diff --git a/server/code/api/ops.py b/server/code/api/ops.py index 5b2af306c7..c99098b611 100644 --- a/server/code/api/ops.py +++ b/server/code/api/ops.py @@ -358,7 +358,7 @@ name="code.grep", input_model=CodeGrepInput, output_model=CodeGrepOutput, - handler=handle_code_grep, + handler=result_lessons.WithLessons(handle_code_grep, result_lessons.grep_paths), idempotent=True, required_role=Role.VIEWER, ), diff --git a/server/code/app/result_lessons.py b/server/code/app/result_lessons.py index f420af7919..037a168dd2 100644 --- a/server/code/app/result_lessons.py +++ b/server/code/app/result_lessons.py @@ -18,6 +18,7 @@ from collections.abc import Awaitable, Callable, Iterable from typing import Any +from server.code.api.models_grep import CodeGrepInput from server.code.api.models_search import CodeSearchInput from server.code.app import common, enrich, memory_links from server.code.app.search_legs import resolve_deadline_ms @@ -48,12 +49,16 @@ def _searched_namespace(input: Any) -> str: def _bound_s(input: Any, started: float) -> float: """Seconds the read may take: `code_enrich_timeout_s`, and no more than what is left - of code.search's own `deadline_ms`, read as the search reads it.""" + of the operation's own `deadline_ms`.""" bound = settings.code_enrich_timeout_s - if isinstance(input, CodeSearchInput): # the one wrapped op with a deadline + if isinstance(input, CodeSearchInput): budget = code_deadline.start(resolve_deadline_ms(input.deadline_ms), started=started) if budget is not None: bound = min(bound, budget.left_ms() / 1000) + elif isinstance(input, CodeGrepInput): + budget = code_deadline.start(input.deadline_ms, started=started) + if budget is not None: + bound = min(bound, budget.left_ms() / 1000) return bound @@ -119,6 +124,13 @@ async def __call__(self, input: Any) -> Any: # The files each code answer returns, best first. +def grep_paths(input: Any, out: Any) -> list[str]: + if input.mode == "presence" or out.not_covered: + return [] + rows = out.files if input.mode == "files" else out.matches + return [str(row.get("path") or "") for row in rows] + + def search_paths(input: Any, out: Any) -> list[str]: # An abstained search's results are not stood behind, and a search over several # repositories has no one team namespace: no lessons on either. diff --git a/tests/integration/test_code_result_lessons.py b/tests/integration/test_code_result_lessons.py index 6e20050d66..10ebd155a9 100644 --- a/tests/integration/test_code_result_lessons.py +++ b/tests/integration/test_code_result_lessons.py @@ -5,6 +5,7 @@ from __future__ import annotations import asyncio +import uuid from typing import Any import httpx @@ -35,6 +36,64 @@ def _set( assert r.status_code == 200, r.text +async def test_grep_attaches_requested_file_lessons_and_respects_opt_out( + control_plane_url: str, mint_test_token: Any, ch_client: Any +) -> None: + from server.settings import settings + + org = unique_org("grep-lessons") + writer = mint_test_token(org=org, namespaces=["*"], roles=["writer"]) + _set(org, control_plane_url, writer, "notes/allowed", "Keep callback order.", [PATH]) + ch_client.insert( + "loom.code_chunks", + [ + [ + org, + "code", + "m", + PATH, + uuid.uuid4(), + "function", + "run", + 1, + 2, + "def run():\n return callback()\n", + [0.0] * settings.embedding_dim, + 1, + ] + ], + column_names=[ + "org", + "namespace", + "embedding_model", + "path", + "chunk_id", + "kind", + "scope", + "start_line", + "end_line", + "content", + "embedding", + "version", + ], + ) + reader = mint_test_token(org=org, namespaces=["code", "dev"], roles=["viewer"]) + body = dict(org=org, namespace="code", pattern="callback", lesson_namespaces=["dev"]) + async with httpx.AsyncClient( + base_url=control_plane_url, headers={"Authorization": f"Bearer {reader}"}, timeout=10 + ) as client: + response = await client.post("/v1/code.grep", json=body) + assert response.status_code == 200, response.text + payload = response.json() + data = payload.get("data", payload) + assert data["matches"][0]["path"] == PATH + assert {row["memory_key"] for row in data["lessons"]} == {"notes/allowed"} + response = await client.post("/v1/code.grep", json={**body, "include_lessons": False}) + assert response.status_code == 200, response.text + payload = response.json() + assert "lessons" not in payload.get("data", payload) + + async def test_lessons_match_the_result_file_by_path( control_plane_url: str, mint_test_token: Any, ch_client: Any ) -> None: diff --git a/tests/unit/test_code_result_lessons.py b/tests/unit/test_code_result_lessons.py index 53205d2399..d0b0e903ca 100644 --- a/tests/unit/test_code_result_lessons.py +++ b/tests/unit/test_code_result_lessons.py @@ -11,6 +11,7 @@ import pytest +from server.code.api.models_grep import CodeGrepInput, CodeGrepOutput from server.code.api.models_navigation import CodeOutlineInput, CodeOutlineOutput from server.code.api.models_search import CodeSearchInput, CodeSearchOutput from server.code.app import result_lessons, search_memo @@ -535,6 +536,7 @@ async def _outline(inp: CodeOutlineInput) -> CodeOutlineOutput: @pytest.mark.parametrize( "name", [ + "code.grep", "code.search", "code.callers", "code.callees", @@ -615,12 +617,14 @@ async def insert_search_memo_row(self, row: dict[str, Any]) -> None: def test_every_code_result_op_is_wrapped_with_its_files() -> None: from server.code.app.context import handle_code_context_pack + from server.code.app.grep import handle_code_grep from server.code.app.search.handler import handle_code_search from server.code.app.symbol.graph import handle_code_callees, handle_code_callers from server.code.app.symbol.impact import handle_code_impact from server.code.app.symbol.outline import handle_code_outline wanted = { + "code.grep": (handle_code_grep, result_lessons.grep_paths), "code.search": (handle_code_search, result_lessons.search_paths), "code.callers": (handle_code_callers, result_lessons.edge_paths), "code.callees": (handle_code_callees, result_lessons.edge_paths), @@ -636,6 +640,57 @@ def test_every_code_result_op_is_wrapped_with_its_files() -> None: assert "lessons" in op.output_model.model_fields +@pytest.mark.asyncio +async def test_grep_lessons_use_matched_files_and_respect_opt_out( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[dict[str, Any]] = [] + _stub_reads(monkeypatch, ["code-repo"], calls) + + async def grep(inp: CodeGrepInput) -> CodeGrepOutput: + return CodeGrepOutput( + matches=[{"path": "src/a.py", "line": 1}], chunks_matched=1, truncated=False + ) + + wrapped = result_lessons.WithLessons(grep, result_lessons.grep_paths) + inp = CodeGrepInput(org="acme", namespace="code-repo", pattern="ab") + out = await wrapped(inp) + assert out.lessons[0]["memory_key"] == "k" + assert calls[0]["paths"] == ["src/a.py"] + calls.clear() + out = await wrapped(inp.model_copy(update={"include_lessons": False})) + assert calls == [] and "lessons" not in out.model_dump() + + +def test_grep_presence_and_uncovered_paths_have_no_lessons(): + inp = CodeGrepInput(pattern="ab") + out = CodeGrepOutput(matches=[{"path": "src/a.py"}], chunks_matched=1, truncated=False) + assert result_lessons.grep_paths(inp, out) == ["src/a.py"] + assert result_lessons.grep_paths(inp.model_copy(update={"mode": "presence"}), out) == [] + assert result_lessons.grep_paths(inp, out.model_copy(update={"not_covered": True})) == [] + files = out.model_copy(update={"files": [{"path": "src/b.py"}]}) + assert result_lessons.grep_paths(inp.model_copy(update={"mode": "files"}), files) == [ + "src/b.py" + ] + + +def test_grep_enrichment_uses_only_the_remaining_deadline(monkeypatch: pytest.MonkeyPatch): + from types import SimpleNamespace + + from tests.unit.clock_fake import ManualClock + + clock = ManualClock() + clock_time = SimpleNamespace(monotonic=clock.monotonic) + monkeypatch.setattr(result_lessons, "time", clock_time) + monkeypatch.setattr(result_lessons.code_deadline, "time", clock_time) + inp = CodeGrepInput(pattern="ab", deadline_ms=200) + started = clock.monotonic() + clock.advance(0.15) + assert result_lessons._bound_s(inp, started) == pytest.approx(0.05) + clock.advance(0.10) + assert result_lessons._bound_s(inp, started) <= 0 + + def test_each_answer_names_its_files_best_first() -> None: from types import SimpleNamespace as NS