Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions docs/architecture/code-index-architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -481,6 +481,19 @@ Every column is in the generated [schema reference](../reference/schema.md).

### Symbol bodies

Literal hits, rendered-message matches and symbol mentions use the same retained-source
line mapping as definition bodies. Copied doc comments and continuation headers do not
become file lines. Compacted class skeletons and incomplete source fragments cannot
establish exact hit coordinates: grep omits those lines and marks its response partial,
and symbol mentions disclose truncation. A missing hit in that response is not evidence
that the file lacks the text.

Grep continuation pools retain source-mapping and partial-coverage metadata. A
pool with another mapping triggers a disclosed rerun; restart pagination for
complete results because its offset can count synthetic hits. Signed cursor
formats and stored match shapes are unchanged across rolling builds. Readers
that do not use the additive pool metadata can still consume its matches.

A definition body covers every declared source line without conflicting overlaps,
missing lines or skeleton substitutions. Doc-comment copies and continuation headers
are removed from chunk prefixes when placing the retained lines. The read uses live
Expand Down
85 changes: 68 additions & 17 deletions server/code/app/grep.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
start_stage_wall,
)
from server.code.domain import deadline as code_deadline
from server.code.domain.symbol_body import retained_source_lines
from server.code.store.index import CodeRepo
from server.core.caller_regex import compile_caller_regex
from server.core.ch import caller_shaped, class_timeout_s
Expand Down Expand Up @@ -266,19 +267,23 @@ async def _grep_lines(

truncated = scan_capped or len(matches) > limit or len(chunks) >= min(200, limit * 3)
trace_run_id = new_trace_run_id()
pool_matches = matches
matches = matches[:limit]
neighbours, anchors_capped, neighbour_tokens, route = await _grep_neighbours(
repo, org, ns, input, matches, route, budget=budget
)
coverage = await asyncio.to_thread(_line_coverage_fields, chunks, budget)
next_cursor = _issue_grep_cursor(
input,
org,
ns,
matches,
pool_matches,
served=limit,
run_id=trace_run_id,
chunks_matched=len(chunks),
truncated=truncated,
)
matches = matches[:limit]
neighbours, anchors_capped, neighbour_tokens, route = await _grep_neighbours(
repo, org, ns, input, matches, route, budget=budget
partial=bool(coverage.get("partial")),
partial_reason=str(coverage.get("partial_reason") or ""),
)
# one entry per matched FILE (ids and paths alike — grep has no unit ids,
# so the outcome fold skips this op; the paths still feed opened_file)
Expand Down Expand Up @@ -330,7 +335,7 @@ async def _grep_lines(
anchors=len(neighbours),
anchors_capped=anchors_capped,
neighbour_tokens=neighbour_tokens,
**_partial_fields(budget),
**coverage,
)


Expand Down Expand Up @@ -398,12 +403,13 @@ async def _grep_neighbours(

# server/code/domain/extract/chunker.py::_split_oversized prefixes every
# window after the first with this marker + the unit's header line.
_CONTINUED_PREFIX = "// … continued from "
_CONTINUED_PREFIXES = ("// … continued from ", "# … continued from ", "/* … continued from ")

# Most chunk text the line step scans per call: 200 chunks of at most
# LOOM_CODE_CHUNK_MAX_CHARS (6000) is 1.2 MB, so 2 MB leaves room for older, larger
# chunks and still bounds the work one request can ask of a worker.
_LINE_SCAN_MAX_CHARS = 2_000_000
_GREP_LINE_MAPPING_VERSION = 1


def _matching_lines(
Expand All @@ -427,15 +433,13 @@ def _matching_lines(
scanned += len(content)
if scanned > _LINE_SCAN_MAX_CHARS:
return matches, True
lines = content.splitlines()
# a continuation window of an oversized unit opens with a synthetic
# "// … continued from <header>" line that is not in the file: drop
# it so line numbers stay exact and the header cannot re-match
if lines and lines[0].startswith(_CONTINUED_PREFIX):
lines = lines[1:]
lines = retained_source_lines(c)
if lines is None:
continue
scope = str(c["scope"] or "")
span = spans.get((str(c["path"]), scope), (0, 0)) if scope else (0, 0)
for off, line in enumerate(lines):
line = line.removesuffix("\n").removesuffix("\r")
hay = line if input.case_sensitive else line.lower()
hit = line_re.search(line) if line_re is not None else (needle in hay)
if not hit:
Expand Down Expand Up @@ -504,6 +508,8 @@ def _issue_grep_cursor(
run_id: str,
chunks_matched: int,
truncated: bool,
partial: bool = False,
partial_reason: str = "",
) -> str:
"""Store every extracted line (served ones included, so offsets are
pool positions) and return the cursor for the page after `served`;
Expand All @@ -524,6 +530,9 @@ def _issue_grep_cursor(
"matches": matches,
"chunks_matched": chunks_matched,
"truncated": truncated,
"line_mapping": _GREP_LINE_MAPPING_VERSION,
"partial": partial,
"partial_reason": partial_reason,
},
indexed_commit="",
route="grep",
Expand All @@ -550,6 +559,8 @@ async def _serve_grep_page(
if row is None:
return None, "pool_not_found", offset
payload = row["payload"]
if payload.get("line_mapping") != _GREP_LINE_MAPPING_VERSION:
return None, "source_line_mapping_changed", offset
pool = [dict(m) for m in (payload.get("matches") or [])]
limit = input.effective_limit
excluded = set(input.exclude_unit_ids)
Expand Down Expand Up @@ -620,6 +631,8 @@ async def _serve_grep_page(
await index_as_of(repo, org, ns), page_commit="", offset=offset, fallback=""
),
trace_run_id=trace_run_id,
partial=bool(payload.get("partial")),
partial_reason=str(payload.get("partial_reason") or ""),
),
"",
offset,
Expand All @@ -636,6 +649,10 @@ async def _grep_page_by_rerun(input: CodeGrepInput, *, offset: int, reason: str)
out = await handle_code_grep(fresh)
out.matches = out.matches[offset : offset + limit]
out.as_of = paging.disclose(out.as_of, page_commit="", offset=offset, fallback=reason)
if reason == "source_line_mapping_changed":
out.partial = True
out.partial_reason = _notes(out.partial_reason, "cursor offset used another source mapping")
out.note = _notes(out.note, "restart pagination for complete results")
set_code_span_attributes(page_offset=offset, page_fallback=reason)
return out

Expand Down Expand Up @@ -668,7 +685,7 @@ async def _unit_spans(
else:
spans[key] = (min(prev[0], start) if start else prev[0], max(prev[1], end))
windowed.add(key)
if str(c.get("content") or "").startswith(_CONTINUED_PREFIX):
if str(c.get("content") or "").startswith(_CONTINUED_PREFIXES):
windowed.add(key)
sym_ids.setdefault(key, []).extend(str(s) for s in (c.get("symbol_ids") or []) if s)
ids = list(dict.fromkeys(s for key in windowed for s in sym_ids.get(key, [])))
Expand Down Expand Up @@ -828,9 +845,14 @@ async def _grep_message(
literal = await _message_literal_fallback(
repo, org, ns, input, route=route, started=started, scope=scope, budget=budget
)
coverage = await asyncio.to_thread(_line_coverage_fields, chunks, budget, previous=literal)
if literal is not None and literal.matches:
return literal.model_copy(
update={**verdict_fields, "note": _notes(note, _MESSAGE_FALLBACK_NOTE, literal.note)}
update={
**verdict_fields,
"note": _notes(note, _MESSAGE_FALLBACK_NOTE, literal.note),
**coverage,
}
)
unit_spans = await _unit_spans(repo, org, ns, chunks) if ranked else {}
matches = _statement_matches(ranked[:limit], unit_spans)
Expand All @@ -845,7 +867,7 @@ async def _grep_message(
trace_run_id=literal.trace_run_id,
fallback="literal",
**verdict_fields,
**_partial_fields(budget),
**coverage,
)
files = list(dict.fromkeys(m["path"] for m in matches))
trace_run_id = new_trace_run_id()
Expand Down Expand Up @@ -892,7 +914,7 @@ async def _grep_message(
note=note.strip().rstrip(";"),
trace_run_id=trace_run_id,
**verdict_fields,
**_partial_fields(budget),
**coverage,
)


Expand Down Expand Up @@ -954,6 +976,35 @@ def _notes(*parts: str) -> str:
return "; ".join(p for p in (str(x).strip().rstrip(";").strip() for x in parts) if p)


def _line_coverage_fields(
chunks: list[dict[str, Any]],
budget: code_deadline.Budget | None,
*,
previous: CodeGrepOutput | None = None,
) -> dict[str, Any]:
fields = _partial_fields(budget)
reasons = [str(fields.get("partial_reason") or "")]
scanned = 0
for chunk in chunks:
scanned += len(str(chunk.get("content") or ""))
if scanned > _LINE_SCAN_MAX_CHARS:
fields["partial"] = True
reasons.append("source line coverage validation stopped at the scan limit")
break
if retained_source_lines(chunk) is None:
fields["partial"] = True
reasons.append(
"some indexed chunks have no exact source line mapping; inspect the source file",
)
break
if previous is not None and previous.partial:
fields["partial"] = True
reasons.append(previous.partial_reason)
if fields.get("partial"):
fields["partial_reason"] = _notes(*dict.fromkeys(reasons))
return fields


_MESSAGE_FALLBACK_NOTE = (
"no throw / LOG_ statement template matches this text, so it was grepped as a literal: "
"mode 'message' is for rendered error or log text; send a code line with mode 'matches' "
Expand Down
10 changes: 6 additions & 4 deletions server/code/app/message_grep.py
Original file line number Diff line number Diff line change
Expand Up @@ -388,17 +388,19 @@ def rank_statements(
chunks: list[dict[str, Any]],
*,
error_name: str = "",
continued_prefix: str = "// … continued from ",
) -> list[Statement]:
"""Every raising/logging statement in the chunks, scored against the
message; best first, then path/line. One entry per (path, line)."""
from server.code.domain.symbol_body import retained_source_lines

seen: set[tuple[str, int]] = set()
out: list[Statement] = []
for c in chunks:
path = str(c.get("path") or "")
lines = str(c.get("content") or "").splitlines()
if lines and lines[0].startswith(continued_prefix):
lines = lines[1:]
lines = retained_source_lines(c)
if lines is None:
continue
lines = [line.removesuffix("\n").removesuffix("\r") for line in lines]
start = int(c.get("start_line") or 1)
scope = str(c.get("scope") or "")
for line_no, text, window in statement_windows(lines, start):
Expand Down
15 changes: 9 additions & 6 deletions server/code/app/symbol/uses.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
from server.code.app.spans import code_span, span_attrs
from server.code.domain.extract.treesitter import lang_for_path
from server.code.domain.search_scope import LANG_EXTENSIONS, build_scope
from server.code.domain.symbol_body import retained_source_lines
from server.code.store.index import CodeRepo

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -62,14 +63,12 @@ def _uses_lines(
handler's extraction so a line grep returns is a line this returns:
case-sensitive literal, (path, line) deduped across windows of one
oversized unit, the synthetic continuation header dropped."""
from server.code.app.grep import _CONTINUED_PREFIX

out: list[dict[str, Any]] = []
seen: set[tuple[str, int]] = set()
for c in chunks:
lines = str(c["content"]).splitlines()
if lines and lines[0].startswith(_CONTINUED_PREFIX):
lines = lines[1:]
lines = retained_source_lines(c)
if lines is None:
continue
scope = str(c.get("scope") or "")
span = spans.get((str(c["path"]), scope), (0, 0)) if scope else (0, 0)
for off, line in enumerate(lines):
Expand Down Expand Up @@ -139,7 +138,11 @@ async def _symbol_uses(
if not any(p == u["path"] and lo <= u["line"] <= hi for p, lo, hi in definition_spans)
]
lines.sort(key=lambda u: (u["path"], u["line"]))
truncated = len(lines) > limit or any(len(part) >= chunk_limit for part in fetched)
truncated = (
len(lines) > limit
or any(len(part) >= chunk_limit for part in fetched)
or any(retained_source_lines(chunk) is None for chunk in chunks)
)
return lines[:limit], truncated


Expand Down
8 changes: 4 additions & 4 deletions server/code/domain/index/grep_sql.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,11 +58,11 @@ def grep_chunks_sql(cond: str, prefix_clause: str, path_sql: str, extra_settings
"""mode=matches: whole chunk rows in (path, start_line) order, chunk_id
breaking ties so the page edge is deterministic."""
inner = newest_live(
"path, scope, symbol_ids, start_line, end_line, content, chunk_id",
"path, scope, symbol_ids, kind, start_line, end_line, content, chunk_id",
_where(cond, prefix_clause, path_sql),
)
return f"""
SELECT path, scope, symbol_ids, start_line, end_line, content
SELECT path, scope, symbol_ids, kind, start_line, end_line, content
FROM {inner}
ORDER BY path, start_line, chunk_id
LIMIT {{limit:UInt32}}
Expand Down Expand Up @@ -94,7 +94,7 @@ def grep_message_sql(prefix_clause: str, path_sql: str, extra_settings: str = ""
" AND multiSearchAny(content, {keywords:Array(String)})"
)
inner = newest_live(
"path, scope, symbol_ids, start_line, end_line, content, chunk_id,\n"
"path, scope, symbol_ids, kind, start_line, end_line, content, chunk_id,\n"
" arrayCount(\n"
" x -> x > 0,\n"
" multiSearchAllPositionsCaseInsensitive(\n"
Expand All @@ -103,7 +103,7 @@ def grep_message_sql(prefix_clause: str, path_sql: str, extra_settings: str = ""
_where(cond, prefix_clause, path_sql),
)
return f"""
SELECT path, scope, symbol_ids, start_line, end_line, content, shingles,
SELECT path, scope, symbol_ids, kind, start_line, end_line, content, shingles,
uniqExact(path) OVER () AS admitted_files
FROM {inner}
ORDER BY shingles DESC, path, start_line, chunk_id
Expand Down
5 changes: 3 additions & 2 deletions server/code/domain/symbol_body.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@
from typing import Any


def _source_lines(row: dict[str, Any]) -> list[str] | None:
def retained_source_lines(row: dict[str, Any]) -> list[str] | None:
"""Physical source lines, excluding presentation prefixes and unmappable skeletons."""
if row.get("kind") == "class_skeleton":
return None
text = str(row.get("content") or "")
Expand Down Expand Up @@ -48,7 +49,7 @@ def complete_source_span(
return ""
by_line: dict[int, str] = {}
for row in rows:
lines = _source_lines(row)
lines = retained_source_lines(row)
if lines is None:
return ""
for line_no, text in enumerate(lines, int(row["start_line"])):
Expand Down
Loading
Loading