Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions docs/architecture/code-index-architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -242,8 +242,10 @@ Identical chunks in one file get an ordinal suffix.

Class skeletons carry compacted presentation text, not physical source coordinates.
The chunker retains class source outside method chunks as `other` chunks, including
documentation, declarations and data members. These source runs keep their original
line spans even when they are shorter than the general leftover-chunk threshold.
documentation, declarations, data members and blank separators. These source runs
keep their original line spans even when they are shorter than the general
leftover-chunk threshold. Body reconstruction requires every source line; missing
separators cause an incomplete read even when all member bodies are present.
Skeletons remain available for class summaries, and unmappable chunks still disclose
partial grep coverage. Retaining source adds chunk rows and can add embedding work.
The shared chunker's stamp moves only the chunk stage, across language families;
Expand Down
6 changes: 3 additions & 3 deletions server/code/domain/extract/chunker.py
Original file line number Diff line number Diff line change
Expand Up @@ -621,12 +621,12 @@ def chunk_file(
if header_chunk is not None:
chunks.append(header_chunk)

# leftovers: uncovered source runs (blank lines and covered lines end
# a run), optionally merged across blank lines up to merge_target_chars
# Container separators are source too; a missing blank line prevents
# complete body reads. Blank lines outside containers still end a run.
runs: list[tuple[int, int]] = []
run_start = None
for i in range(1, len(src_lines) + 1):
uncovered = not covered[i] and src_lines[i - 1].strip() != ""
uncovered = not covered[i] and (container_source[i] or src_lines[i - 1].strip() != "")
if uncovered and run_start is None:
run_start = i
boundary = (not uncovered) or i == len(src_lines)
Expand Down
46 changes: 46 additions & 0 deletions tests/unit/test_code_symbol_source_lines.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,52 @@
from server.code.domain.symbol_body import complete_source_span


@pytest.mark.parametrize("newline", ["\n", "\r\n"], ids=["lf", "crlf"])
@pytest.mark.parametrize("merge_target", [0, 3000])
@pytest.mark.parametrize(
("path", "text", "class_name"),
[
(
"src/example.py",
'class Group:\n """A group.\n\n Details.\n """\n\n'
" def first(self):\n return 1\n\n \n"
" def second(self):\n return 2\n",
"src.example::Group",
),
(
"src/example.cpp",
"struct Group {\n\n int first() { return 1; }\n\n \n"
" int second() { return 2; }\n\n};\n",
"Group",
),
],
ids=["python", "cpp"],
)
def test_container_bodies_retain_blank_source_between_members(
newline: str, merge_target: int, path: str, text: str, class_name: str
) -> None:
from server.code.domain.extract.chunker import chunk_file

text = text.replace("\n", newline)
parsed, error = parse_for_ingest(path, text.encode())
assert parsed is not None and not error
symbol = next(s for s in parsed.symbols if s.qualified_name == class_name)
chunks = chunk_file(parsed, merge_target_chars=merge_target)
rows = [
{
"kind": c.kind,
"content": c.content,
"start_line": c.start_line,
"end_line": c.end_line,
}
for c in chunks
if c.kind != "class_skeleton"
and c.start_line <= symbol.end_line
and c.end_line >= symbol.start_line
]
assert complete_source_span(rows, symbol.start_line, symbol.end_line, max_chars=0) == text


@pytest.mark.parametrize("separator", ["", "\u0085", "\u2028", "\u2029"])
@pytest.mark.parametrize("newline", ["\n", "\r\n"], ids=["lf", "crlf"])
@pytest.mark.parametrize("name", ["f", "g"])
Expand Down
Loading