Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/architecture/code-index-architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -346,6 +346,11 @@ the C++ tier; `loom code scip-delta` runs one delta pass.
Where both tiers hold the same row, reads prefer the compiler row, and every answer
names the tier it read (`scip` or `treesitter`).

Exact code grep answers can carry the caller's readable code-linked lessons for
the matched files. Lessons are read after cached match data, within the request's
remaining deadline. Presence queries and uncovered paths carry none;
`include_lessons=false` skips the read.

Python compiler conversion keeps callable and class descriptors distinct from
parameters, type parameters and fields. Call edges require a containing function
span; a reference at module or class scope does not belong to the preceding function.
Expand Down
4 changes: 4 additions & 0 deletions docs/reference/operations/code.md
Original file line number Diff line number Diff line change
Expand Up @@ -710,6 +710,8 @@ Exact-match lookup over the indexed corpus.

| Input field | Type | Default | Meaning |
| --- | --- | --- | --- |
| `include_lessons` | `bool` | `True` | Attach `lessons`: the memories about the answer's top files (for code.context_pack, a `task` briefing's only). |
| `lesson_namespaces` | `list[str]` | *(computed)* | The memory namespaces `lessons` reads besides the repository's team rules (over MCP the caller's memory home, filled in); one the caller cannot read is skipped. |
| `org` | `str` | `''` | The tenant (first element of the identity tuple). |
| `namespace` | `str` | `''` | The namespace the op reads or writes. |
| `agent` | `str` | `'-'` | The writing or reading agent; `'-'` or `'*'` means any agent where the op allows it. |
Expand All @@ -733,6 +735,8 @@ Exact-match lookup over the indexed corpus.

| Output field | Type | Meaning |
| --- | --- | --- |
| `lessons` | `list[dict[str, Any]]` | The memories about the answer's top files, best first, capped at `code_result_lessons_tokens`; each row {path, memory_key, namespace, excerpt, code_state}, with `changed_since` and `supersedes_key` when they apply. |
| `lessons_left_out` | `int` | How many rows about those files the lessons' token cap or a cut left out, with the rows another row read for the answer supersedes. |
| `matches` | `list[dict[str, Any]]` | {path, line, text, scope, scope_start_line, scope_end_line} per matching LINE, path/line ordered. scope_* is the span of the enclosing unit named by `scope` (0/0 when the line sits in no unit) — enough to decide whether to fetch the body… |
| `next_cursor` | `str` | Continuation: cursor for the next page of matching lines, "" when the extracted set is exhausted. has_more mirrors it. |
| `has_more` | `bool` | True when `next_cursor` is set. |
Expand Down
6 changes: 6 additions & 0 deletions sdk/python/loom/ops.py
Original file line number Diff line number Diff line change
Expand Up @@ -6537,6 +6537,8 @@ class CodeFirstBuildOutput(TypedDict, total=False):
class CodeGrepInput(TypedDict, total=False):
"Exact-match lookup over the indexed corpus. Unlike shell grep, the"

include_lessons: NotRequired[bool]
lesson_namespaces: NotRequired[list[str]]
org: NotRequired[str]
namespace: NotRequired[str]
agent: NotRequired[str]
Expand All @@ -6562,6 +6564,8 @@ class CodeGrepInput(TypedDict, total=False):
class CodeGrepBody(TypedDict, total=False):
"Exact-match lookup over the indexed corpus. Unlike shell grep, the"

include_lessons: NotRequired[bool]
lesson_namespaces: NotRequired[list[str]]
org: NotRequired[str]
namespace: NotRequired[str]
agent: NotRequired[str]
Expand All @@ -6585,6 +6589,8 @@ class CodeGrepBody(TypedDict, total=False):


class CodeGrepOutput(TypedDict, total=False):
lessons: NotRequired[list[dict[str, Any]]]
lessons_left_out: NotRequired[int]
matches: Required[list[dict[str, Any]]]
next_cursor: NotRequired[str]
has_more: NotRequired[bool]
Expand Down
4 changes: 4 additions & 0 deletions sdk/typescript/src/generated/ops.ts
Original file line number Diff line number Diff line change
Expand Up @@ -864,6 +864,8 @@ export interface CodeFirstBuildOutput {
}

export interface CodeGrepInput {
include_lessons?: boolean;
lesson_namespaces?: string[];
org?: string;
namespace?: string;
agent?: string;
Expand All @@ -887,6 +889,8 @@ export interface CodeGrepInput {
}

export interface CodeGrepOutput {
lessons?: Array<Record<string, unknown>>;
lessons_left_out?: number;
matches: Array<Record<string, unknown>>;
next_cursor?: string;
has_more?: boolean;
Expand Down
9 changes: 6 additions & 3 deletions server/code/api/models_grep.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,12 @@

from server.code.api.models_common import (
_CodeIdentity,
_drop_empty_lessons,
_drop_empty_presence_keys,
_drop_unfired_symbol_blocks,
_lead_namespaces,
_LessonsOut,
_ResultLessons,
_trim_namespaces,
)
from server.operations.base import MCP_HIDDEN
Expand All @@ -29,7 +32,7 @@
# ============================================================


class CodeGrepInput(_CodeIdentity):
class CodeGrepInput(_CodeIdentity, _ResultLessons):
"""Exact-match lookup over the indexed corpus. Unlike shell grep, the
answer arrives WITH what raw grep can't attach: the
enclosing symbol per hit, one indexed round-trip, and a hit list that
Expand Down Expand Up @@ -129,7 +132,7 @@ def effective_limit(self) -> int:
return min(self.limit or 20, 200)


class CodeGrepOutput(BaseModel):
class CodeGrepOutput(_LessonsOut):
@model_serializer(mode="wrap")
def _drop_unfired_mode_fields(self, handler: SerializerFunctionWrapHandler) -> dict[str, Any]:
# the presence block rides only when it fired, the neighbours block
Expand All @@ -142,7 +145,7 @@ def _drop_unfired_mode_fields(self, handler: SerializerFunctionWrapHandler) -> d
for k in ("partial", "partial_reason", "fallback"):
if not data.get(k):
data.pop(k, None)
return data
return _drop_empty_lessons(data)

# {path, line, text, scope, scope_start_line, scope_end_line} per matching
# LINE, path/line ordered. scope_* is the span of the enclosing unit named
Expand Down
2 changes: 1 addition & 1 deletion server/code/api/ops.py
Original file line number Diff line number Diff line change
Expand Up @@ -358,7 +358,7 @@
name="code.grep",
input_model=CodeGrepInput,
output_model=CodeGrepOutput,
handler=handle_code_grep,
handler=result_lessons.WithLessons(handle_code_grep, result_lessons.grep_paths),
idempotent=True,
required_role=Role.VIEWER,
),
Expand Down
16 changes: 14 additions & 2 deletions server/code/app/result_lessons.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
from collections.abc import Awaitable, Callable, Iterable
from typing import Any

from server.code.api.models_grep import CodeGrepInput
from server.code.api.models_search import CodeSearchInput
from server.code.app import common, enrich, memory_links
from server.code.app.search_legs import resolve_deadline_ms
Expand Down Expand Up @@ -48,12 +49,16 @@ def _searched_namespace(input: Any) -> str:

def _bound_s(input: Any, started: float) -> float:
"""Seconds the read may take: `code_enrich_timeout_s`, and no more than what is left
of code.search's own `deadline_ms`, read as the search reads it."""
of the operation's own `deadline_ms`."""
bound = settings.code_enrich_timeout_s
if isinstance(input, CodeSearchInput): # the one wrapped op with a deadline
if isinstance(input, CodeSearchInput):
budget = code_deadline.start(resolve_deadline_ms(input.deadline_ms), started=started)
if budget is not None:
bound = min(bound, budget.left_ms() / 1000)
elif isinstance(input, CodeGrepInput):
budget = code_deadline.start(input.deadline_ms, started=started)
if budget is not None:
bound = min(bound, budget.left_ms() / 1000)
return bound


Expand Down Expand Up @@ -119,6 +124,13 @@ async def __call__(self, input: Any) -> Any:
# The files each code answer returns, best first.


def grep_paths(input: Any, out: Any) -> list[str]:
if input.mode == "presence" or out.not_covered:
return []
rows = out.files if input.mode == "files" else out.matches
return [str(row.get("path") or "") for row in rows]


def search_paths(input: Any, out: Any) -> list[str]:
# An abstained search's results are not stood behind, and a search over several
# repositories has no one team namespace: no lessons on either.
Expand Down
59 changes: 59 additions & 0 deletions tests/integration/test_code_result_lessons.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from __future__ import annotations

import asyncio
import uuid
from typing import Any

import httpx
Expand Down Expand Up @@ -35,6 +36,64 @@ def _set(
assert r.status_code == 200, r.text


async def test_grep_attaches_requested_file_lessons_and_respects_opt_out(
control_plane_url: str, mint_test_token: Any, ch_client: Any
) -> None:
from server.settings import settings

org = unique_org("grep-lessons")
writer = mint_test_token(org=org, namespaces=["*"], roles=["writer"])
_set(org, control_plane_url, writer, "notes/allowed", "Keep callback order.", [PATH])
ch_client.insert(
"loom.code_chunks",
[
[
org,
"code",
"m",
PATH,
uuid.uuid4(),
"function",
"run",
1,
2,
"def run():\n return callback()\n",
[0.0] * settings.embedding_dim,
1,
]
],
column_names=[
"org",
"namespace",
"embedding_model",
"path",
"chunk_id",
"kind",
"scope",
"start_line",
"end_line",
"content",
"embedding",
"version",
],
)
reader = mint_test_token(org=org, namespaces=["code", "dev"], roles=["viewer"])
body = dict(org=org, namespace="code", pattern="callback", lesson_namespaces=["dev"])
async with httpx.AsyncClient(
base_url=control_plane_url, headers={"Authorization": f"Bearer {reader}"}, timeout=10
) as client:
response = await client.post("/v1/code.grep", json=body)
assert response.status_code == 200, response.text
payload = response.json()
data = payload.get("data", payload)
assert data["matches"][0]["path"] == PATH
assert {row["memory_key"] for row in data["lessons"]} == {"notes/allowed"}
response = await client.post("/v1/code.grep", json={**body, "include_lessons": False})
assert response.status_code == 200, response.text
payload = response.json()
assert "lessons" not in payload.get("data", payload)


async def test_lessons_match_the_result_file_by_path(
control_plane_url: str, mint_test_token: Any, ch_client: Any
) -> None:
Expand Down
55 changes: 55 additions & 0 deletions tests/unit/test_code_result_lessons.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

import pytest

from server.code.api.models_grep import CodeGrepInput, CodeGrepOutput
from server.code.api.models_navigation import CodeOutlineInput, CodeOutlineOutput
from server.code.api.models_search import CodeSearchInput, CodeSearchOutput
from server.code.app import result_lessons, search_memo
Expand Down Expand Up @@ -535,6 +536,7 @@ async def _outline(inp: CodeOutlineInput) -> CodeOutlineOutput:
@pytest.mark.parametrize(
"name",
[
"code.grep",
"code.search",
"code.callers",
"code.callees",
Expand Down Expand Up @@ -615,12 +617,14 @@ async def insert_search_memo_row(self, row: dict[str, Any]) -> None:

def test_every_code_result_op_is_wrapped_with_its_files() -> None:
from server.code.app.context import handle_code_context_pack
from server.code.app.grep import handle_code_grep
from server.code.app.search.handler import handle_code_search
from server.code.app.symbol.graph import handle_code_callees, handle_code_callers
from server.code.app.symbol.impact import handle_code_impact
from server.code.app.symbol.outline import handle_code_outline

wanted = {
"code.grep": (handle_code_grep, result_lessons.grep_paths),
"code.search": (handle_code_search, result_lessons.search_paths),
"code.callers": (handle_code_callers, result_lessons.edge_paths),
"code.callees": (handle_code_callees, result_lessons.edge_paths),
Expand All @@ -636,6 +640,57 @@ def test_every_code_result_op_is_wrapped_with_its_files() -> None:
assert "lessons" in op.output_model.model_fields


@pytest.mark.asyncio
async def test_grep_lessons_use_matched_files_and_respect_opt_out(
monkeypatch: pytest.MonkeyPatch,
) -> None:
calls: list[dict[str, Any]] = []
_stub_reads(monkeypatch, ["code-repo"], calls)

async def grep(inp: CodeGrepInput) -> CodeGrepOutput:
return CodeGrepOutput(
matches=[{"path": "src/a.py", "line": 1}], chunks_matched=1, truncated=False
)

wrapped = result_lessons.WithLessons(grep, result_lessons.grep_paths)
inp = CodeGrepInput(org="acme", namespace="code-repo", pattern="ab")
out = await wrapped(inp)
assert out.lessons[0]["memory_key"] == "k"
assert calls[0]["paths"] == ["src/a.py"]
calls.clear()
out = await wrapped(inp.model_copy(update={"include_lessons": False}))
assert calls == [] and "lessons" not in out.model_dump()


def test_grep_presence_and_uncovered_paths_have_no_lessons():
inp = CodeGrepInput(pattern="ab")
out = CodeGrepOutput(matches=[{"path": "src/a.py"}], chunks_matched=1, truncated=False)
assert result_lessons.grep_paths(inp, out) == ["src/a.py"]
assert result_lessons.grep_paths(inp.model_copy(update={"mode": "presence"}), out) == []
assert result_lessons.grep_paths(inp, out.model_copy(update={"not_covered": True})) == []
files = out.model_copy(update={"files": [{"path": "src/b.py"}]})
assert result_lessons.grep_paths(inp.model_copy(update={"mode": "files"}), files) == [
"src/b.py"
]


def test_grep_enrichment_uses_only_the_remaining_deadline(monkeypatch: pytest.MonkeyPatch):
from types import SimpleNamespace

from tests.unit.clock_fake import ManualClock

clock = ManualClock()
clock_time = SimpleNamespace(monotonic=clock.monotonic)
monkeypatch.setattr(result_lessons, "time", clock_time)
monkeypatch.setattr(result_lessons.code_deadline, "time", clock_time)
inp = CodeGrepInput(pattern="ab", deadline_ms=200)
started = clock.monotonic()
clock.advance(0.15)
assert result_lessons._bound_s(inp, started) == pytest.approx(0.05)
clock.advance(0.10)
assert result_lessons._bound_s(inp, started) <= 0


def test_each_answer_names_its_files_best_first() -> None:
from types import SimpleNamespace as NS

Expand Down
Loading