diff --git a/docs/architecture/code-index-architecture.md b/docs/architecture/code-index-architecture.md index 8d767d5137..2e2952a804 100644 --- a/docs/architecture/code-index-architecture.md +++ b/docs/architecture/code-index-architecture.md @@ -242,8 +242,10 @@ Identical chunks in one file get an ordinal suffix. Class skeletons carry compacted presentation text, not physical source coordinates. The chunker retains class source outside method chunks as `other` chunks, including -documentation, declarations and data members. These source runs keep their original -line spans even when they are shorter than the general leftover-chunk threshold. +documentation, declarations, data members and blank separators. These source runs +keep their original line spans even when they are shorter than the general +leftover-chunk threshold. Body reconstruction requires every source line; missing +separators cause an incomplete read even when all member bodies are present. Skeletons remain available for class summaries, and unmappable chunks still disclose partial grep coverage. Retaining source adds chunk rows and can add embedding work. The shared chunker's stamp moves only the chunk stage, across language families; diff --git a/server/code/domain/extract/chunker.py b/server/code/domain/extract/chunker.py index 9732b21a88..d7a3865322 100644 --- a/server/code/domain/extract/chunker.py +++ b/server/code/domain/extract/chunker.py @@ -621,12 +621,12 @@ def chunk_file( if header_chunk is not None: chunks.append(header_chunk) - # leftovers: uncovered source runs (blank lines and covered lines end - # a run), optionally merged across blank lines up to merge_target_chars + # Container separators are source too; a missing blank line prevents + # complete body reads. Blank lines outside containers still end a run. runs: list[tuple[int, int]] = [] run_start = None for i in range(1, len(src_lines) + 1): - uncovered = not covered[i] and src_lines[i - 1].strip() != "" + uncovered = not covered[i] and (container_source[i] or src_lines[i - 1].strip() != "") if uncovered and run_start is None: run_start = i boundary = (not uncovered) or i == len(src_lines) diff --git a/tests/unit/test_code_symbol_source_lines.py b/tests/unit/test_code_symbol_source_lines.py index 47c4632ce4..77b8c6216d 100644 --- a/tests/unit/test_code_symbol_source_lines.py +++ b/tests/unit/test_code_symbol_source_lines.py @@ -9,6 +9,52 @@ from server.code.domain.symbol_body import complete_source_span +@pytest.mark.parametrize("newline", ["\n", "\r\n"], ids=["lf", "crlf"]) +@pytest.mark.parametrize("merge_target", [0, 3000]) +@pytest.mark.parametrize( + ("path", "text", "class_name"), + [ + ( + "src/example.py", + 'class Group:\n """A group.\n\n Details.\n """\n\n' + " def first(self):\n return 1\n\n \n" + " def second(self):\n return 2\n", + "src.example::Group", + ), + ( + "src/example.cpp", + "struct Group {\n\n int first() { return 1; }\n\n \n" + " int second() { return 2; }\n\n};\n", + "Group", + ), + ], + ids=["python", "cpp"], +) +def test_container_bodies_retain_blank_source_between_members( + newline: str, merge_target: int, path: str, text: str, class_name: str +) -> None: + from server.code.domain.extract.chunker import chunk_file + + text = text.replace("\n", newline) + parsed, error = parse_for_ingest(path, text.encode()) + assert parsed is not None and not error + symbol = next(s for s in parsed.symbols if s.qualified_name == class_name) + chunks = chunk_file(parsed, merge_target_chars=merge_target) + rows = [ + { + "kind": c.kind, + "content": c.content, + "start_line": c.start_line, + "end_line": c.end_line, + } + for c in chunks + if c.kind != "class_skeleton" + and c.start_line <= symbol.end_line + and c.end_line >= symbol.start_line + ] + assert complete_source_span(rows, symbol.start_line, symbol.end_line, max_chars=0) == text + + @pytest.mark.parametrize("separator", ["", "\u0085", "\u2028", "\u2029"]) @pytest.mark.parametrize("newline", ["\n", "\r\n"], ids=["lf", "crlf"]) @pytest.mark.parametrize("name", ["f", "g"])