From d777f5ce4b27d69f696cc0ead77a5b5a4e394bca Mon Sep 17 00:00:00 2001 From: Lareb Zafar Date: Sat, 21 Nov 2026 19:39:55 +0530 Subject: [PATCH] Code lookup preserves source coverage and identifier resolution Retain physical class source outside compacted summaries, bind Python import spellings to stored definitions, and admit exact underscore lexemes through the lexical gate without filtering on a SELECT alias. Existing identities and scope predicates remain intact. Retaining class source refreshes the chunk stage. Signed-off-by: Lareb Zafar --- docs/architecture/code-index-architecture.md | 20 ++++ docs/features/code-index/overview.md | 6 ++ docs/reference/operations/code.md | 4 +- server/code/api/models_symbol.py | 13 +-- server/code/app/index/symbol_forms.py | 61 ++++++++++-- server/code/domain/extract/chunker.py | 16 ++- server/code/store/index/search.py | 38 ++++--- tests/analyzer/cases/code.py | 6 +- tests/analyzer/shapes.txt | 18 ++-- .../test_code_lexical_token_gate_live_ch.py | 54 ++++++++++ tests/unit/test_code_grep_container_source.py | 43 ++++++++ tests/unit/test_code_lexical_statement.py | 13 +-- .../unit/test_code_lexical_token_gate_sql.py | 69 +++++++++++++ tests/unit/test_code_symbol_python_forms.py | 98 +++++++++++++++++++ 14 files changed, 405 insertions(+), 54 deletions(-) create mode 100644 tests/integration/test_code_lexical_token_gate_live_ch.py create mode 100644 tests/unit/test_code_grep_container_source.py create mode 100644 tests/unit/test_code_lexical_token_gate_sql.py create mode 100644 tests/unit/test_code_symbol_python_forms.py diff --git a/docs/architecture/code-index-architecture.md b/docs/architecture/code-index-architecture.md index 4174dd9dc3..4a3f4b9723 100644 --- a/docs/architecture/code-index-architecture.md +++ b/docs/architecture/code-index-architecture.md @@ -240,6 +240,15 @@ their symbol spans (`lane_units.span_chunks`). A chunk id is an unchanged chunk keeps its id and hash and the embed step skips it without a lookup. Identical chunks in one file get an ordinal suffix. +Class skeletons carry compacted presentation text, not physical source coordinates. +The chunker retains class source outside method chunks as `other` chunks, including +documentation, declarations and data members. These source runs keep their original +line spans even when they are shorter than the general leftover-chunk threshold. +Skeletons remain available for class summaries, and unmappable chunks still disclose +partial grep coverage. Retaining source adds chunk rows and can add embedding work. +The shared chunker's stamp moves only the chunk stage, across language families; +existing indexes need that stage re-derived before missing source becomes readable. + ### Symbols and edges A symbol id is `treesitter.symbol_id(qualified_name, signature)`: the first 128 bits of @@ -471,6 +480,12 @@ Every column is in the generated [schema reference](../reference/schema.md). ## Query patterns +Python import spellings use the bounded short-name candidate read after an exact +symbol miss. Read-side matching preserves dotted module scopes and treats a package's +`__init__.py` as implicit. Only Python source rows participate; ambiguous or incomplete +candidate sets decline. The response discloses `resolved_by: python_module` while +retaining stored names and ids. Extraction and its stamp do not change. + | Op | Path | How | | -------------------------------- | ----------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `code.search` | `code/app/search/pipeline.py` | Routes first (pattern, path, history question), then runs legs under the deadline (`gather.py`): lexical (idf-weighted token count), vector (a scan with no vector index, or twin shortlist), PR history, fuzzy symbol vote. `fusion.py` fuses by reciprocal rank (k = 60); `serve.py` pages into the token budget or abstains. A repeat within the memo window is one read of `code_search_memo`. | @@ -479,6 +494,11 @@ Every column is in the generated [schema reference](../reference/schema.md). | `code.callers`, `code.impact` | `code/store/index/edges.py` | Callers are the `to_symbol_id` range of `code_edges`, compiler rows preferred, with the unresolved count. `code.impact` maps diff hunks to seed symbols and walks outward; an empty walk answers `no_indexed_paths`, `no_seeds_in_hunks` or `no_static_callers`. | | `code.outline`, `code.enclosing` | — | Per-path reads of `code_symbols` and `code_chunks` spans. | +The lexical text-index gate tokenizes the query with `splitByNonAlpha`, matching the +index tokenizer. The score counts only whole app lexemes and rejects partial matches +from the gate. Underscore-only lexemes use the exact score within the same tenant, +model and path scope because the text index has no token for them. + ### Symbol bodies Literal hits, rendered-message matches and symbol mentions use the same retained-source diff --git a/docs/features/code-index/overview.md b/docs/features/code-index/overview.md index 2b0db21d15..fda7829084 100644 --- a/docs/features/code-index/overview.md +++ b/docs/features/code-index/overview.md @@ -32,6 +32,12 @@ unless `LOOM_MCP_CODE_TOOLS=true`; `?toolset=coding` always does. Every op, grouped by task, is on [operations.md](operations.md). +Python symbol lookups accept dotted import names such as `package.encoder.dumps` +and package names such as `package::dumps` for a definition in +`package/__init__.py`. A unique match reports `resolved_by: python_module` and +retains the stored qualified name and symbol id. Ambiguous matches remain unresolved. +This name matching reads the existing index and requires no reindexing. + ## One namespace is one branch A code namespace tracks one repository branch. Each index tier advances its own commit diff --git a/docs/reference/operations/code.md b/docs/reference/operations/code.md index 97032eba01..93fd7f90d3 100644 --- a/docs/reference/operations/code.md +++ b/docs/reference/operations/code.md @@ -835,7 +835,7 @@ Symbol-shaped (name/symbol_id) or diff-shaped (files/diff) blast radius. | `causes` | `list[str]` | Why `epistemic` is `lower_bound`, from the set above; empty when it is `exact`. | | `by_hop_counts` | `dict[int, int]` | Distinct symbols reached per hop BEFORE any slicing: {1: n, 2: m}. | | `truncated` | `bool` | True when `impacted_symbols` is a page of the radius (per_hop_limit / offset sliced it, or the per-hop edge read hit its cap). | -| `resolved_by` | `str` | `anonymous_namespace` \| `tail` when the symbol shape's name bound through symbol-form tolerance; "" and off the wire otherwise. | +| `resolved_by` | `str` | `anonymous_namespace` \| `tail` \| `python_module` \| `registration` when a read-side name form bound; "" and off the wire for an exact hit. | | `verdict` | `str` | Why the radius is empty, '' when it is not. | | `paths_seen` | `list[dict[str, str]]` | Diff shape: every path the request named, with its class — `test` \| `doc` \| `config` \| `unindexed` (outside the indexed prefixes, or a source path the index holds no file row for) \| `src` — so the verdict is actionable without re-deriving… | | `overlay` | `bool` | On the wire only when `pr_number` was given — see CodeCallersOutput. | @@ -1579,7 +1579,7 @@ Exact symbol lookup: definition, signature, doc comment, and body of a function/ | `uses_truncated` | `bool` | True when the `uses` list stopped at `uses_limit`. | | `uses_tokens` | `int` | Estimated tokens the compact `uses` block cost out of `token_budget`. | | `picked_by` | `str` | Which context key ranked the definitions when `near_paths` / `pr_number` was given — same_file \| same_dir \| co_change \| path_distance \| none. "" and off the wire when no context was given. | -| `resolved_by` | `str` | `anonymous_namespace` \| `tail` when an over-qualified name bound through symbol-form tolerance; "" and off the wire otherwise. | +| `resolved_by` | `str` | `anonymous_namespace` \| `tail` \| `python_module` \| `registration` when a read-side name form bound; "" and off the wire for an exact hit. | | `documented_in` | `list[dict[str, Any]]` | The document sections that cite the definition (the Markdown lane's `reads` edges) — section qualified name, path, line; at most five distinct sections, first by confidence. | | `neighbours` | `list[dict[str, Any]]` | The neighbours graph block on the definitions (one entry per definition anchor); empty unless `neighbours` ran. | | `anchors` | `int` | How many definitions the `neighbours` block was built for; 0 when it is off. | diff --git a/server/code/api/models_symbol.py b/server/code/api/models_symbol.py index 5aa4b2f8f0..36c1c6461d 100644 --- a/server/code/api/models_symbol.py +++ b/server/code/api/models_symbol.py @@ -180,8 +180,8 @@ def _drop_empty_presence(self, handler: SerializerFunctionWrapHandler) -> dict[s # — same_file | same_dir | co_change | path_distance | none. "" and off the wire # when no context was given. picked_by: str = "" - # `anonymous_namespace` | `tail` when an over-qualified name bound through - # symbol-form tolerance; "" and off the wire otherwise. + # `anonymous_namespace` | `tail` | `python_module` | `registration` when a + # read-side name form bound; "" and off the wire for an exact hit. resolved_by: str = "" # The document sections that cite the definition (the Markdown lane's `reads` edges) # — section qualified name, path, line; at most five distinct sections, first by @@ -372,8 +372,9 @@ def _drop_unrequested_candidates( # `anonymous_namespace` (`(anonymous namespace)::` segments stripped) or `tail` (a # namespace the definition does not carry dropped; one definition fit), or # `registration` when a SQL-level name (`sumMap`, `arrayFold::executeImpl`) bound to - # the implementing class's methods through its registration statement. "" and off - # the wire for an exact hit. + # the implementing class's methods through its registration statement, or + # `python_module` for an import spelling of a retained Python name. "" and + # off the wire for an exact hit. resolved_by: str = "" # A bare short name that resolves to many unrelated definitions (setColumns # on 60 classes) is not walked: `ambiguous` is set, `candidates` lists the @@ -550,8 +551,8 @@ def _drop_unset(self, handler: SerializerFunctionWrapHandler) -> dict[str, Any]: # True when `impacted_symbols` is a page of the radius (per_hop_limit / # offset sliced it, or the per-hop edge read hit its cap). truncated: bool = False - # `anonymous_namespace` | `tail` when the symbol shape's name bound through - # symbol-form tolerance; "" and off the wire otherwise. + # `anonymous_namespace` | `tail` | `python_module` | `registration` when a + # read-side name form bound; "" and off the wire for an exact hit. resolved_by: str = "" # Why the radius is empty, '' when it is not. Answers: # `no_indexed_paths` (the change names no path the symbol table holds diff --git a/server/code/app/index/symbol_forms.py b/server/code/app/index/symbol_forms.py index 8f901188c1..035edd4897 100644 --- a/server/code/app/index/symbol_forms.py +++ b/server/code/app/index/symbol_forms.py @@ -35,6 +35,11 @@ after rules 1 and 2, for bare names too: a bare SQL name is not an over-qualification but a different vocabulary, and the class it names is one class — the ambiguity guard is skipped for these rows. + +Python import spellings bind through the same bounded bare-name read. A dotted +module remains one stored scope segment, and a package's `__init__.py` is implicit +in its import name. Only Python rows participate; ties and incomplete candidate +sets decline (`resolved_by: python_module`). Stored names and ids stay intact. """ from __future__ import annotations @@ -47,6 +52,7 @@ RESOLVED_ANON = "anonymous_namespace" RESOLVED_TAIL = "tail" RESOLVED_REGISTRATION = "registration" +RESOLVED_PYTHON = "python_module" # Bare-name definitions read for the tail rule; a hub name past this many # rows is not bound (the rule needs the whole candidate set to call a match unique). TAIL_READ_LIMIT = 400 @@ -167,39 +173,74 @@ def dropped_segments(written: str, rows: list[dict[str, Any]]) -> set[str]: return out +def python_bind(written: str, rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Bind an import spelling to one retained Python definition name.""" + spelling = ".".join(segments(written)) + if "." not in spelling: + return [] + best: dict[str, list[dict[str, Any]]] = {} + best_rank = 0 + for row in rows: + path = str(row.get("path") or "") + if not path.endswith(".py"): + continue + qualified = str(row.get("qualified_name") or "") + parts = segments(qualified) + if len(parts) < 2: + continue + forms = [".".join(parts)] + # Only the file's module scope is implicit in a package import. A class's + # constructor and a directory named __init__ keep their names. + if path.endswith("/__init__.py") and parts[0].endswith(".__init__"): + forms.append(".".join([parts[0].removesuffix(".__init__"), *parts[1:]])) + rank = max( + (2 if form == spelling else int(form.endswith("." + spelling))) for form in forms + ) + if rank > best_rank: + best_rank, best = rank, {} + if rank and rank == best_rank: + best.setdefault(qualified, []).append(row) + return next(iter(best.values())) if len(best) == 1 else [] + + async def resolve_tolerant( repo: Any, org: str, ns: str, name: str, ) -> tuple[list[dict[str, Any]], str]: - """The two rules over the repository, after an exact miss: (rows, - resolved_by). ([], "") when neither binds. Rule 1 runs for any name with - an anonymous segment (the rest may be bare and is looked up as such); - rule 2 only for a qualified name.""" + """Read-side name forms after an exact miss: (rows, resolved_by). + Anonymous scopes, Python imports and namespace tails precede registration; + every binding retains the stored identity.""" stripped = strip_anonymous(name) if stripped: rows = await repo.symbols_by_name(org, ns, stripped) if rows: return rows, RESOLVED_ANON - if is_qualified(name): - bound = await _tail(repo, org, ns, name) + if is_qualified(name) or "." in name: + bound, resolved_by = await _qualified(repo, org, ns, name) if bound: - return bound, RESOLVED_TAIL + return bound, resolved_by return await _registration(repo, org, ns, name) -async def _tail(repo: Any, org: str, ns: str, name: str) -> list[dict[str, Any]]: +async def _qualified(repo: Any, org: str, ns: str, name: str) -> tuple[list[dict[str, Any]], str]: bare = _named(name)[-1] + if "::" not in name: + bare = bare.rsplit(".", 1)[-1] rows = await repo.symbols_by_name(org, ns, bare, limit=TAIL_READ_LIMIT) if not rows or len(rows) >= TAIL_READ_LIMIT: - return [] + return [], "" + bound = python_bind(name, rows) + if bound: + return bound, RESOLVED_PYTHON classes: set[str] = set() for seg in sorted(dropped_segments(name, rows)): hits = await repo.symbols_by_name(org, ns, seg, limit=5) if any(str(h.get("kind") or "") in {"class", "struct"} for h in hits): classes.add(seg) - return tail_bind(name, rows, class_names=classes) + bound = tail_bind(name, rows, class_names=classes) + return bound, RESOLVED_TAIL if bound else "" async def _registration( diff --git a/server/code/domain/extract/chunker.py b/server/code/domain/extract/chunker.py index ae96b23028..9732b21a88 100644 --- a/server/code/domain/extract/chunker.py +++ b/server/code/domain/extract/chunker.py @@ -7,7 +7,7 @@ * every class/struct -> one `class_skeleton` chunk: the class text with method BODIES elided to `{ ... }` (methods get their own chunks) * leading comments+includes -> one `file_header` chunk - * anything left at top level -> grouped `other` chunks + * uncovered source, including class documentation and data -> `other` chunks * a function larger than max_chars splits at top-level statement boundaries inside its body (the AST's fresh-line child starts), each piece re-prefixed with the signature header so no piece ever floats @@ -507,6 +507,7 @@ def chunk_file( cls_syms = [s for s in pf.symbols if s.kind in kinds] covered = [False] * (len(src_lines) + 2) + container_source = [False] * len(covered) # function chunks (whole definitions, with the doc comment attached) for fn in sorted(fn_syms, key=lambda s: (s.start_line, -s.end_line)): @@ -608,14 +609,19 @@ def chunk_file( ) for i in range(cls.start_line, cls.end_line + 1): if i < len(covered): - covered[i] = True + if cls.kind in _BASE_CONTAINER_KINDS: + # Skeletons cannot establish physical source coordinates. + # Keep source outside the method chunks for exact reads. + container_source[i] = True + else: + covered[i] = True # file header: leading run of comments/includes/imports/blank lines header_chunk = _file_header_chunk(pf, src_lines, covered, max_chars) if header_chunk is not None: chunks.append(header_chunk) - # leftovers: uncovered top-level runs (blank lines and covered lines end + # leftovers: uncovered source runs (blank lines and covered lines end # a run), optionally merged across blank lines up to merge_target_chars runs: list[tuple[int, int]] = [] run_start = None @@ -630,7 +636,9 @@ def chunk_file( run_start = None for run_start, run_end in _merge_runs(runs, src_lines, covered, merge_target_chars): content = "".join(src_lines[run_start - 1 : run_end]) - if len(content.strip()) < _MIN_CHUNK_CHARS: + if len(content.strip()) < _MIN_CHUNK_CHARS and not any( + container_source[run_start : run_end + 1] + ): continue for piece, s_ln, e_ln in ( [(content, run_start, run_end)] diff --git a/server/code/store/index/search.py b/server/code/store/index/search.py index cf5f70df48..69c387a6b9 100644 --- a/server/code/store/index/search.py +++ b/server/code/store/index/search.py @@ -41,32 +41,39 @@ async def lexical_search_chunks( `scope` adds the pattern / path predicates to the plain WHERE after the identity keys; unset, the statement is unchanged byte for byte. - The gate is `hasAnyTokens(lexemes, tokens)`, the predicate the `text` - index on `lexemes` (tokenizer splitByNonAlpha — the - writer's own token boundaries) answers per granule; - `multiSearchAny(concat(' ', lexemes, ' '), …)` reads every chunk of - the namespace (293 MB and 462 ms per call on the ClickHouse - repository).""" + The String query form of `hasAnyTokens` tokenizes the query with + the text index's splitByNonAlpha tokenizer. Its Array form takes + pre-tokenized terms, so whole underscore identifiers cannot gate + on it. The exact score rejects the coarse gate's partial matches. + Underscore-only lexemes have no text-index token and use only the + exact score predicate within the same scope. + + The text-index gate avoids reading every chunk in the namespace. + """ if not tokens: return [] where_prefix = "AND startsWith(path, {prefix:String})" if path_prefix else "" + gate = ( + "AND hasAnyTokens(lexemes, {gate_query:String})" + if all(tok.strip("_") for tok in tokens) + else "" + ) + score = ( + "arraySum(arrayMap((tok, idf) -> idf * countSubstrings(" + "concat(' ', lexemes, ' '), concat(' ', tok, ' ')), " + "{tokens:Array(String)}, {idfs:Array(Float32)}))" + ) scope_sql, scope_params = _scope_sql(scope) rows = await self._ch.query_all( f""" SELECT chunk_id, path, scope, kind, start_line, end_line, content, symbol_ids, context_description, - arraySum( - arrayMap( - (tok, idf) -> idf * countSubstrings( - concat(' ', lexemes, ' '), concat(' ', tok, ' ') - ), - {{tokens:Array(String)}}, {{idfs:Array(Float32)}} - ) - ) AS lex_score + {score} AS lex_score FROM loom.code_chunks FINAL WHERE org = {{org:String}} AND namespace = {{ns:String}} AND embedding_model = {{model:String}} AND tombstoned_at IS NULL - AND hasAnyTokens(lexemes, {{tokens:Array(String)}}) + {gate} + AND {score} > 0 {where_prefix} {scope_sql} ORDER BY lex_score DESC @@ -79,6 +86,7 @@ async def lexical_search_chunks( "tokens": tokens, "idfs": idfs, "pool": pool, + **({"gate_query": " ".join(tokens)} if gate else {}), **({"prefix": path_prefix} if path_prefix else {}), **scope_params, }, diff --git a/tests/analyzer/cases/code.py b/tests/analyzer/cases/code.py index 83061cfefb..8081b730d1 100644 --- a/tests/analyzer/cases/code.py +++ b/tests/analyzer/cases/code.py @@ -40,12 +40,13 @@ async def _search(ch: RecordingCH, c: dict[str, Any]) -> None: repo = _repo(ch) scope = _scope(c["scope"]) prefix = "src/" if c["path_prefix"] else "" + tokens = ["__"] if c["token_gate"] == "underscores" else ["quota", "key"] await repo.lexical_search_chunks( ORG, NS, MODEL, - ["quota", "key"], - [1.5, 0.7], + tokens, + [1.0] * len(tokens), path_prefix=prefix, scope=scope, ) @@ -421,6 +422,7 @@ def cases() -> list[Case]: { "scope": ("none", "globs", "regex"), "path_prefix": (False, True), + "token_gate": ("words", "underscores"), }, ), Case( diff --git a/tests/analyzer/shapes.txt b/tests/analyzer/shapes.txt index 674d42c7a6..dd8f3abfe7 100644 --- a/tests/analyzer/shapes.txt +++ b/tests/analyzer/shapes.txt @@ -303,10 +303,10 @@ 2430fc6260ce SELECT count() AS n FROM loom.entity_edge_memories WHERE org = {org:String} AND ((namespace, memory_key) IN (SELECT namespace, memory_key FROM loom.memory WHERE org = {org:String} AND user_id = {uid:S 2448faf35e3e SELECT DISTINCT src_path FROM loom.code_test_map FINAL WHERE org = {org:String} AND namespace = {ns:String} AND tombstoned_at IS NULL 244b76b58819 ALTER TABLE loom.memory UPDATE tombstoned_at = {now:DateTime} WHERE (org = {org:String} AND tombstoned_at IS NULL AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_ta +246241942bf7 SELECT chunk_id, path, scope, kind, start_line, end_line, content, symbol_ids, context_description, arraySum(arrayMap((tok, idf) -> idf * countSubstrings(concat(?, lexemes, ?), concat(?, tok, ?)), {to 24c3fd0e6b29 SELECT path, chunk_id FROM loom.code_chunks WHERE org = {org:String} AND namespace = {ns:String} AND embedding_model = {model:String} GROUP BY path, chunk_id HAVING argMax(embedding_dim, version) = ? 24d4d8775291 SELECT start_line, end_line, scope, substring(content, greatest(?, position(content, {lit:String}) - ?), ?) AS excerpt FROM loom.code_chunks FINAL WHERE org = {org:String} AND namespace = {ns:String} 24df34974019 SELECT ts, org, namespace, agent, user_id, session_id, operation, request_id, trace_id, latency_ms, input_tokens, output_tokens, embedding_tokens, upstream_cost_usd, bytes_written, success, error_code -24edbaad24b0 SELECT chunk_id, path, scope, kind, start_line, end_line, content, symbol_ids, context_description, arraySum( arrayMap( (tok, idf) -> idf * countSubstrings( concat(?, lexemes, ?), concat(?, tok, ?) ), 25443b779b21 ALTER TABLE loom.shared DELETE WHERE (org = {org:String} AND namespace IN {ns:Array(String)}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? 25455773978c SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, cosineDistance(e.embedding, {q:Array(Float32)}) AS dist 255a74cf0321 SELECT count() AS n FROM loom.blob_uploads WHERE org = {org:String} AND namespace = {ns:String} AND ? @@ -400,9 +400,9 @@ 2fab97e447f4 SELECT count() AS n FROM loom.entity_vectors FINAL WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND tombstoned_at IS NULL 2fc0cc735773 SELECT toString(evidence_id) AS eid, session_id, role, actor, toString(observed_at) AS observed, content, matched FROM ( SELECT evidence_id, session_id, role, actor, observed_at, content, tombstoned_a 2fd8ca033214 DELETE FROM loom.memory_chunks WHERE org = {org:String} AND namespace = {ns:String} AND agent = {a:String} AND user_id = {u:String} AND parent_memory_key = {k:String} AND if(session_id = {ss:String}, +30364bf4b5e4 SELECT chunk_id, path, scope, kind, start_line, end_line, content, symbol_ids, context_description, arraySum(arrayMap((tok, idf) -> idf * countSubstrings(concat(?, lexemes, ?), concat(?, tok, ?)), {to 303a1827c22d SELECT DISTINCT msg_id FROM loom.conversations WHERE org = {org:String} AND namespace = {ns:String} AND thread_id = {tid:String} AND msg_id IN {mids:Array(String)} AND is_saved AND tombstoned_at IS NU 30406949d371 SELECT memory_id, version, agent, user_id, session_id, memory_key, created_at FROM loom.memory FINAL WHERE org = {org:String} AND namespace = {ns:String} AND memory_key IN {keys:Array(String)} AND tom -30b541a5c832 SELECT chunk_id, path, scope, kind, start_line, end_line, content, symbol_ids, context_description, arraySum( arrayMap( (tok, idf) -> idf * countSubstrings( concat(?, lexemes, ?), concat(?, tok, ?) ), 30e3abcc719b ALTER TABLE loom.query_nodes DELETE WHERE (org = {org:String} AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? 30ef01ea55bc SELECT count() AS n FROM loom.derived_facts WHERE org = {org:String} AND user_id = {uid:String} AND ? 3112e7749090 SELECT agent, user_id, session_id, version, summary_text, cosineDistance(summary_embedding, {q:Array(Float32)}) AS distance, -- NB: alias must NOT be `valid_at` — under the analyzer a -- SELECT alias @@ -455,6 +455,7 @@ 34c0098ab598 SELECT count() AS cnt, sum(numeric_value) AS s, avg(numeric_value) AS a, min(occurred_at) AS first_at, max(occurred_at) AS last_at FROM loom.derived_facts WHERE org = {org:String} AND namespace = {ns: 34e802c88a30 SELECT count() AS n FROM loom.events WHERE org = {org:String} AND namespace = {ns:String} 34eadcb6f3af SELECT count() AS n FROM loom.delete_version_checks WHERE org = {org:String} AND user_id = {uid:String} +34fbfa7dbe58 SELECT chunk_id, path, scope, kind, start_line, end_line, content, symbol_ids, context_description, arraySum(arrayMap((tok, idf) -> idf * countSubstrings(concat(?, lexemes, ?), concat(?, tok, ?)), {to 350d156c1ad6 SELECT concat(?, toString(c.session_id), ?, toString(c.chunk_index)) AS memory_key, ? AS memory_type, substring(c.content, ?, ?) AS content_excerpt, c.content AS content_full, c.agent AS agent, c.user 3511c7e369c0 SELECT concat(?, session_id, ?, toString(chunk_index)) AS memory_key, ? AS memory_type, content AS content_excerpt, toString(created_at) AS created_at, toString(created_at) AS valid_at FROM loom.memor 35454b0b72fe SELECT count() AS n FROM loom.memory FINAL WHERE org = {org:String} AND namespace = {ns:String} AND tombstoned_at IS NULL AND status != ? AND agent = {agent:String} AND user_id = {uid:String} AND sess @@ -501,6 +502,7 @@ 3ac3f004eaa2 ALTER TABLE loom.agent_run_claims DELETE WHERE org = {org:String} AND user_id = {uid:String} 3aee32f6c95a WITH docs AS ( SELECT thread_id, msg_id, any(role) AS msg_role, argMax(content, ts) AS body, max(ts) AS saved_at, arrayFilter(t -> length(t) >= ?, splitByNonAlpha(lowerUTF8(body))) AS words FROM loom. 3b05a28d96f2 SELECT count() AS cnt, sum(numeric_value) AS s, avg(numeric_value) AS a, min(occurred_at) AS first_at, max(occurred_at) AS last_at FROM loom.derived_facts WHERE org = {org:String} AND namespace = {ns: +3b1abae99635 SELECT chunk_id, path, scope, kind, start_line, end_line, content, symbol_ids, context_description, arraySum(arrayMap((tok, idf) -> idf * countSubstrings(concat(?, lexemes, ?), concat(?, tok, ?)), {to 3b1ae278b7f6 SELECT memory_type, toString(status) AS status, source_class, count() AS rows, min(updated_at) AS first_at, max(updated_at) AS last_at FROM loom.memory FINAL WHERE org = {org:String} AND namespace = { 3b4b4580cc9a SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g 3ba235828531 ALTER TABLE loom.documents UPDATE superseded_at = {now:DateTime} WHERE (org = {org:String} AND user_id != {uid:String} AND superseded_at IS NULL AND cityHash64(namespace, source_uri) IN {docs:Array(UI @@ -609,6 +611,7 @@ 46c16fbf6c99 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, cosineDistance(e.embedding, {q:Array(Float32)}) AS dist 46e9280c500c ALTER TABLE loom.conversations UPDATE tombstoned_at = {now:DateTime} WHERE org = {org:String} AND namespace = {ns:String} AND agent = {agent:String} AND thread_id = {tid:String} AND msg_id IN {ids:Arr 46fb9af13bbe SELECT DISTINCT msg_id FROM loom.conversations WHERE org = {org:String} AND namespace = {ns:String} AND thread_id = {tid:String} AND msg_id IN {mids:Array(String)} AND is_saved AND tombstoned_at IS NU +472bce1b04dc SELECT chunk_id, path, scope, kind, start_line, end_line, content, symbol_ids, context_description, arraySum(arrayMap((tok, idf) -> idf * countSubstrings(concat(?, lexemes, ?), concat(?, tok, ?)), {to 472e38ad8c05 SELECT count() AS n FROM loom.entity_triple_memories WHERE org = {org:String} AND namespace IN {ns:Array(String)} 476e1eb1f2e5 ALTER TABLE loom.knowledge_uploads DELETE WHERE (org = {org:String} AND namespace IN {ns:Array(String)}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? 477c2ea34273 SELECT category_id, name, description, substring(summary, ?, ?) AS summary_excerpt, cosineDistance(embedding, {q:Array(Float32)}) AS distance FROM loom.categories FINAL WHERE org = {org:String} AND na @@ -698,6 +701,7 @@ 53d6938a1b94 ALTER TABLE loom.delete_version_checks DELETE WHERE (org = {org:String} AND namespace IN {ns:Array(String)}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync 54082f4a0bda SELECT c.session_id AS session_id, c.chunk_index AS chunk_index, c.parent_memory_key AS parent_memory_key, cosineDistance(c.embedding, {q:Array(Float32)}) AS distance FROM loom.memory_chunks AS c WHER 541a4c06fca2 SELECT agent, user_id, session_id, memory_key, memory_type, content_excerpt, created_at, valid_at, version, status FROM ( SELECT agent, user_id, session_id, memory_key, memory_type, content_excerpt, t +542242ea89a0 SELECT chunk_id, path, scope, kind, start_line, end_line, content, symbol_ids, context_description, arraySum(arrayMap((tok, idf) -> idf * countSubstrings(concat(?, lexemes, ?), concat(?, tok, ?)), {to 54325369241a SELECT org, namespace, agent, user_id, session_id, memory_key, checked, tombstone, done FROM loom.delete_version_checks FINAL WHERE org = {org:String} AND namespace = {ns:String} AND agent = {agent:St 544bda275e23 ALTER TABLE loom.memory_code_link_queue DELETE WHERE (org = {org:String} AND user_id = {uid:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? 548b3de5bbd4 SELECT memory_key, toString(memory_id) AS mid, content_sha256, agent, memory_type, -- a high value is readable by key only if(sensitivity = ?, ?, substring(value, ?, ?)) AS preview, sensitivity, tags, @@ -751,9 +755,11 @@ 5852405dd8a6 SELECT id, org, scope_type, scope_value, requester, actor, urgency, purge, mirror_marked, attempts, steps_completed, requested_at, failure_reason, residue_count, residue, verified_at, census FROM loom 5862d4ef7b73 ALTER TABLE loom.session_brief_revisions DELETE WHERE (org = {org:String} AND user_id = {uid:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? 586a6ce18c21 SELECT concat(?, toString(c.session_id), ?, toString(c.chunk_index)) AS memory_key, ? AS memory_type, substring(c.content, ?, ?) AS content_excerpt, c.content AS content_full, c.agent AS agent, c.user +58d8e597b166 SELECT chunk_id, path, scope, kind, start_line, end_line, content, symbol_ids, context_description, arraySum(arrayMap((tok, idf) -> idf * countSubstrings(concat(?, lexemes, ?), concat(?, tok, ?)), {to 58dd61abeb39 WITH m AS ( SELECT org, namespace, agent, user_id, session_id, memory_key, argMax(status, version) AS ms, argMax(memory_id, version) AS mid FROM loom.memory WHERE org = {org:String} AND namespace = {n 58f4b29e4e3a SELECT org, namespace, policy_id, effect, description, actions, condition, enabled, mode, created_by, created_at, updated_at, tombstoned_at FROM loom.namespace_policies FINAL WHERE org = {org:String} 58f4d68e658f SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, cosineDistance(e.embedding, {q:Array(Float32)}) AS dist +5901e2d006d5 SELECT chunk_id, path, scope, kind, start_line, end_line, content, symbol_ids, context_description, arraySum(arrayMap((tok, idf) -> idf * countSubstrings(concat(?, lexemes, ?), concat(?, tok, ?)), {to 5923bd3d36dc SELECT thread_id, count() AS n, argMax((agent, toString(role), substringUTF8(content, ?, {listed:UInt32})), (ts, msg_id)) AS last_turn FROM loom.conversations WHERE org = {org:String} AND namespace = 593e0e151193 SELECT toString(memory_id) AS memory_id FROM loom.memory FINAL WHERE org = {org:String} AND namespace = {ns:String} AND agent = {agent:String} AND user_id = {uid:String} AND memory_key = {key:String} 596490bd9ba5 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, ? - cosineDistance(e.embedding, {qv:Array(Float32)}) AS similarity, toString(coalesce(e.temporal_ @@ -762,6 +768,7 @@ 5985b4690741 ALTER TABLE loom.memory UPDATE value = ?, value_json = ?, evidence = ?, entities = [], concepts = [], redacted_at = {ts:DateTime}, redaction_reason = {rb:String} WHERE org = {org:String} AND ((namespa 599611be357f SELECT count() AS n FROM loom.memory_chunk_sources FINAL WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND tombstoned_at IS NULL 599725704d75 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g +59b30916ba34 SELECT chunk_id, path, scope, kind, start_line, end_line, content, symbol_ids, context_description, arraySum(arrayMap((tok, idf) -> idf * countSubstrings(concat(?, lexemes, ?), concat(?, tok, ?)), {to 59b3f6e281d9 SELECT count() AS n FROM loom.entity_edge_rows WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND ? 59bc4767d952 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, ? - cosineDistance(e.embedding, {qv:Array(Float32)}) AS similarity, toString(coalesce(e.temporal_ 59cfa551438a ALTER TABLE loom.learnings_supporters DELETE WHERE (org = {org:String} AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? @@ -1295,7 +1302,6 @@ 97f2b91effdf SELECT id, org, scope_type, scope_value, requester, actor, urgency, purge, mirror_marked, attempts, steps_completed, requested_at, failure_reason, residue FROM loom.erasure_requests FINAL WHERE status 982795414e49 SELECT agent, user_id, session_id, memory_key, cosineDistance(embedding, {emb:Array(Float32)}) AS dist FROM loom.memory_embeddings FINAL WHERE org = {org:String} AND namespace = {ns:String} AND memory 9848ee046655 ALTER TABLE loom.memory_belief_sources DELETE WHERE (org = {org:String} AND (namespace, memory_key) IN {keys:Array(Tuple(String, String))}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:S -985f96df6202 SELECT chunk_id, path, scope, kind, start_line, end_line, content, symbol_ids, context_description, arraySum( arrayMap( (tok, idf) -> idf * countSubstrings( concat(?, lexemes, ?), concat(?, tok, ?) ), 98649b515546 SELECT concat(?, toString(c.session_id), ?, toString(c.chunk_index)) AS memory_key, ? AS memory_type, substring(c.content, ?, ?) AS content_excerpt, c.content AS content_full, c.agent AS agent, c.user 9874fd318676 SELECT ? FROM loom.memory WHERE tombstoned_at IS NULL AND expires_at IS NOT NULL AND expires_at <= now() AND (org, namespace) NOT IN (SELECT org, namespace FROM loom.namespaces FINAL WHERE retention_d 98b76c9dedf3 DELETE FROM loom.code_answers WHERE (org = {org:String} AND memory_namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS lightweight_deletes_sync = ? @@ -1353,7 +1359,6 @@ 9e49c20a4fc8 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g 9e7b47b91e9c SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g 9e9adc05a7ef SELECT namespace, kind, owner_email, description, tags, active, conversation_logging_enabled, capture_generation_io, coverage_sweep, consolidation_enabled, daily_cost_cap_usd, daily_user_cost_cap_usd, -9e9e8424f7db SELECT chunk_id, path, scope, kind, start_line, end_line, content, symbol_ids, context_description, arraySum( arrayMap( (tok, idf) -> idf * countSubstrings( concat(?, lexemes, ?), concat(?, tok, ?) ), 9eafee0b23b4 SELECT count() AS n FROM loom.memory_extraction_queue WHERE org = {org:String} AND user_id = {uid:String} AND ? 9ed5e750e8c9 DELETE FROM loom.chunks WHERE org = {org:String} AND namespace = {ns:String} AND (source_uri, doc_version) IN {rows:Array(Tuple(String, UInt64))} SETTINGS lightweight_deletes_sync = ? 9ed6e8cc8634 SELECT count() AS n FROM loom.dataset_rows WHERE org = {org:String} AND namespace = {ns:String} AND ? @@ -1458,6 +1463,7 @@ a86e1a806af2 SELECT DISTINCT src_path FROM loom.code_test_map FINAL WHERE org = a883073e33de SELECT count() AS n, if(n = ?, NULL, max(created_at)) AS latest FROM loom.memory_webhooks FINAL WHERE org = {org:String} AND namespace = {ns:String} AND is_deleted = ? a8b303321d91 SELECT c.session_id AS session_id, c.chunk_index AS chunk_index, c.parent_memory_key AS parent_memory_key, cosineDistance(c.embedding, {q:Array(Float32)}) AS distance FROM loom.memory_chunks AS c WHER a8c6e74f4e8d SELECT event_type, count() AS cnt, min(ts) AS first_ts, max(ts) AS last_ts FROM loom.audit_log WHERE ts >= toDateTime64(now() - INTERVAL {since:UInt32} SECOND, ?, ?) AND NOT match(namespace, ?) AND or +a8e1a646e7fb SELECT chunk_id, path, scope, kind, start_line, end_line, content, symbol_ids, context_description, arraySum(arrayMap((tok, idf) -> idf * countSubstrings(concat(?, lexemes, ?), concat(?, tok, ?)), {to a8ef32fbad8c SELECT content_sha256, count() AS chunk_count, uniqExact(source_uri) AS uri_count, groupArray(?)((source_uri, chunk_idx)) AS samples FROM loom.chunks WHERE org = {org:String} AND namespace = {ns:Strin a8f2636209f6 SELECT ts, event_type, org, namespace, actor, subject, request_id, row_hash FROM loom.audit_log WHERE org = {org:String} AND (NOT startsWith(namespace, {col_prefix:String}) OR namespace IN {col_readab a90054f8b1c7 SELECT DISTINCT b.? AS qid FROM system.mutations ARRAY JOIN {versions:Array(Tuple(String, UInt64))} AS b WHERE database = {db:String} AND table = {t:String} AND is_done = ? AND position(command, {pat: @@ -1523,6 +1529,7 @@ afdcda2bf23a SELECT count() AS n FROM loom.memory_ask_precedents WHERE org = {or aff1e632ca9c SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, e.agent AS agent, e.user_id AS user_id, e.session_id AS b0198f2a1337 SELECT uniqExact(path) AS n FROM loom.code_chunks FINAL WHERE org = {org:String} AND namespace = {ns:String} AND embedding_model = {model:String} AND tombstoned_at IS NULL AND startsWith(path, {prefix b062ee2ddb58 SELECT toString(r.record_id) AS record_id FROM loom.memory_records AS r FINAL WHERE r.org = {org:String} AND r.namespace = {ns:String} AND r.schema_name = {schema:String} AND r.status = ? AND r.tombst +b080660bbca3 SELECT chunk_id, path, scope, kind, start_line, end_line, content, symbol_ids, context_description, arraySum(arrayMap((tok, idf) -> idf * countSubstrings(concat(?, lexemes, ?), concat(?, tok, ?)), {to b0b2f5049b7f SELECT count() AS n FROM loom.memory FINAL WHERE org = {org:String} AND namespace = {ns:String} AND status != ? AND tombstoned_at IS NULL AND agent = {agent:String} AND user_id = {uid:String} AND sess b0eeab7a67c1 SELECT org, namespace, agent, user_id, session_id, memory_key, embedding_model, length(embedding) AS vector_len FROM loom.memory_embeddings FINAL WHERE tombstoned_at IS NULL AND embedding_dim = ? AND b0fbecd9222a SELECT memory_id, org, namespace, agent, user_id, session_id, memory_key, memory_type, if(length(toString(value_json)) > ?, toString(value_json), value) AS value, -- The stored bytes, for version rows @@ -1688,7 +1695,6 @@ c169a496882f SELECT count() AS n FROM loom.llm_response_cache WHERE org = {org:S c1aa2dbb7a43 SELECT count() AS n FROM loom.llm_response_cache WHERE org = {org:String} AND hasAny(window_keys, {keys:Array(String)}) c1ad00d9e230 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g c1dd61e21ceb SELECT window_start, window_end, row_count, toString(rows_digest) AS rows_digest, toString(prev_checkpoint_hash) AS prev_checkpoint_hash, toString(checkpoint_hash) AS checkpoint_hash FROM loom.audit_c -c1e0254cfd85 SELECT chunk_id, path, scope, kind, start_line, end_line, content, symbol_ids, context_description, arraySum( arrayMap( (tok, idf) -> idf * countSubstrings( concat(?, lexemes, ?), concat(?, tok, ?) ), c23f3a6975c8 ALTER TABLE loom.memory_embeddings UPDATE tombstoned_at = {now:DateTime} WHERE (org = {org:String} AND tombstoned_at IS NULL AND scope = ? AND (namespace, memory_key) IN {keys:Array(Tuple(String, Stri c266319cd4ad SELECT count() AS n FROM loom.conversation_threads FINAL WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND tombstoned_at IS NULL c26ddee98bc8 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, ? - cosineDistance(e.embedding, {qv:Array(Float32)}) AS similarity, toString(coalesce(e.temporal_ @@ -1835,7 +1841,6 @@ d04bc22409e3 SELECT uniqExact(msg_id) AS n FROM loom.conversation_embeddings FIN d06419919bf2 SELECT count() AS n FROM loom.memory_chunks FINAL WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND tombstoned_at IS NULL d06c9096df61 SELECT c.session_id AS session_id, c.chunk_index AS chunk_index, c.parent_memory_key AS parent_memory_key, cosineDistance(c.embedding, {q:Array(Float32)}) AS distance FROM loom.memory_chunks AS c WHER d0802f441bc2 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g -d082fe47bb46 SELECT chunk_id, path, scope, kind, start_line, end_line, content, symbol_ids, context_description, arraySum( arrayMap( (tok, idf) -> idf * countSubstrings( concat(?, lexemes, ?), concat(?, tok, ?) ), d089a672528c SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g d0a888305d0b SELECT org, code_namespace, branch_namespace, pr_number, branch, verdict, epistemic, basis, reason, cherry_pick_pr, cherry_pick_sha, backport_pr, label_wanted, wanted_branch, wanted_source, settled, p d0be57c846ee SELECT if(consumer = ?, args[?], consumer) AS consumer_key, count() AS calls, countIf(NOT success) AS errors, countIf(NOT success AND error_code = ?) AS server_errors, round(errors / calls, ?) AS erro @@ -2051,6 +2056,7 @@ e6990fcc8c82 SELECT thread_id, msg_id, role, content_excerpt AS embedded_text, t e6f6f6bc74e2 SELECT count() AS n FROM loom.memory_chunks FINAL WHERE org = {org:String} AND tombstoned_at IS NULL AND namespace IN {ns:Array(String)} e7331ee31d1b SELECT concat(?, toString(c.session_id), ?, toString(c.chunk_index)) AS memory_key, ? AS memory_type, substring(c.content, ?, ?) AS content_excerpt, c.content AS content_full, c.agent AS agent, c.user e747c3134919 ALTER TABLE loom.memory_record_fields UPDATE tombstoned_at = {now:DateTime} WHERE (org = {org:String} AND tombstoned_at IS NULL AND record_id IN {ids:Array(UUID)}) AND ({erasure_mutation_tag:String} = +e74a862439f4 SELECT chunk_id, path, scope, kind, start_line, end_line, content, symbol_ids, context_description, arraySum(arrayMap((tok, idf) -> idf * countSubstrings(concat(?, lexemes, ?), concat(?, tok, ?)), {to e75f4a024984 SELECT count() AS n FROM loom.derived_facts WHERE org = {org:String} AND namespace = {ns:String} AND ? e77afbb3fb28 SELECT count() AS n FROM loom.chunks WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND doc_superseded_at IS NULL AND (org, namespace, embedding_model, source_uri, doc_version) NOT IN (S e77b590b9f54 SELECT org, namespace, name, source_id, provider, generation, fingerprint, provider_id, status, secret_enc, agent, user_id, session_id, actor FROM loom.knowledge_source_push FINAL WHERE org = {org:Str diff --git a/tests/integration/test_code_lexical_token_gate_live_ch.py b/tests/integration/test_code_lexical_token_gate_live_ch.py new file mode 100644 index 0000000000..276c065510 --- /dev/null +++ b/tests/integration/test_code_lexical_token_gate_live_ch.py @@ -0,0 +1,54 @@ +"""Stored underscore identifiers remain visible through the lexical text-index gate.""" + +from __future__ import annotations + +import uuid +from typing import Any + +import pytest + +from server.code.domain.extract.tokenize import code_tokens +from server.code.store.index import CodeRepo +from server.core.ch import get_client, set_query_tag +from tests.integration._resources import unique_org + + +@pytest.mark.asyncio +async def test_lexical_gate_reads_existing_identifier_lexemes(ch_client: Any) -> None: + org, ns = unique_org("lexical-gate"), "code" + ch_client.insert( + "loom.code_chunks", + [ + [org, ns, "m", path, uuid.uuid4(), lexemes, [0.0] * 3072] + for path, lexemes in [ + ("src/constructor.py", "__init__"), + ("src/private.py", "_private"), + ("src/underscores.py", "__"), + ("src/quota.py", "quota_store quota store"), + ("src/partials.py", "init private"), + ] + ], + column_names=[ + "org", + "namespace", + "embedding_model", + "path", + "chunk_id", + "lexemes", + "embedding", + ], + ) + client = get_client() + await client.connect() + set_query_tag("test.code.lexical_token_gate") + repo = CodeRepo(client) + for query, expected in [ + ("__init__", "src/constructor.py"), + ("_private", "src/private.py"), + ("__", "src/underscores.py"), + ("quota_store", "src/quota.py"), + ]: + tokens = code_tokens(query) + rows = await repo.lexical_search_chunks(org, ns, "m", tokens, [1.0] * len(tokens)) + assert [r["path"] for r in rows] == [expected] + assert rows[0]["lex_score"] == len(tokens) diff --git a/tests/unit/test_code_grep_container_source.py b/tests/unit/test_code_grep_container_source.py new file mode 100644 index 0000000000..b3ad9a43c5 --- /dev/null +++ b/tests/unit/test_code_grep_container_source.py @@ -0,0 +1,43 @@ +"""Container documentation and data keep physical source coordinates.""" + +from dataclasses import asdict + +import pytest + +from server.code.api.models_grep import CodeGrepInput +from server.code.app.grep import _line_coverage_fields, _matching_lines +from server.code.domain.extract.chunker import chunk_file +from server.code.domain.extract.lang_python import parse_python + +SOURCE = '''class Handler: + """Keep redirect requests on the declared route.""" + + def run(self): + return route_target() + + policy = "redirect" +''' + + +@pytest.mark.parametrize("pattern", ["redirect requests", 'policy = "redirect"']) +def test_class_source_outside_method_bodies_remains_greppable(pattern: str) -> None: + parsed = parse_python("handler.py", SOURCE.encode()) + rows = [asdict(c) for c in chunk_file(parsed) if pattern in c.content] + matches, capped = _matching_lines(rows, {}, CodeGrepInput(pattern=pattern), None, 20) + expected = [ + (line_no, line.strip()) + for line_no, line in enumerate(SOURCE.splitlines(), 1) + if pattern in line + ] + assert not capped + assert [(hit["line"], hit["text"]) for hit in matches] == expected + # Skeletons remain non-authoritative for source coordinates. + assert _line_coverage_fields(rows, None)["partial"] + + +def test_function_body_keeps_its_existing_exact_source_coordinates() -> None: + parsed = parse_python("handler.py", SOURCE.encode()) + rows = [asdict(c) for c in chunk_file(parsed) if "route_target" in c.content] + matches, capped = _matching_lines(rows, {}, CodeGrepInput(pattern="route_target"), None, 20) + assert not capped + assert [(hit["line"], hit["text"]) for hit in matches] == [(5, "return route_target()")] diff --git a/tests/unit/test_code_lexical_statement.py b/tests/unit/test_code_lexical_statement.py index c366cb774b..cf3b01aaec 100644 --- a/tests/unit/test_code_lexical_statement.py +++ b/tests/unit/test_code_lexical_statement.py @@ -15,18 +15,12 @@ """ SELECT chunk_id, path, scope, kind, start_line, end_line, content, symbol_ids, context_description, - arraySum( - arrayMap( - (tok, idf) -> idf * countSubstrings( - concat(' ', lexemes, ' '), concat(' ', tok, ' ') - ), - {tokens:Array(String)}, {idfs:Array(Float32)} - ) - ) AS lex_score + arraySum(arrayMap((tok, idf) -> idf * countSubstrings(concat(' ', lexemes, ' '), concat(' ', tok, ' ')), {tokens:Array(String)}, {idfs:Array(Float32)})) AS lex_score FROM loom.code_chunks FINAL WHERE org = {org:String} AND namespace = {ns:String} AND embedding_model = {model:String} AND tombstoned_at IS NULL - AND hasAnyTokens(lexemes, {tokens:Array(String)}) + AND hasAnyTokens(lexemes, {gate_query:String}) + AND arraySum(arrayMap((tok, idf) -> idf * countSubstrings(concat(' ', lexemes, ' '), concat(' ', tok, ' ')), {tokens:Array(String)}, {idfs:Array(Float32)})) > 0 """ + " " * 14 + "\n" @@ -69,6 +63,7 @@ def test_lexical_statement_byte_for_byte() -> None: "ns": "n", "model": "m", "tokens": ["merge", "tree"], + "gate_query": "merge tree", "idfs": [2.5, 0.7], "pool": 200, } diff --git a/tests/unit/test_code_lexical_token_gate_sql.py b/tests/unit/test_code_lexical_token_gate_sql.py new file mode 100644 index 0000000000..1975fa8d17 --- /dev/null +++ b/tests/unit/test_code_lexical_token_gate_sql.py @@ -0,0 +1,69 @@ +"""The lexical gate admits every scored identifier and rejects partial tokens.""" + +from __future__ import annotations + +import uuid + +import pytest + +from server.code.domain.extract.tokenize import code_tokens, lexemes_text +from server.code.store.index import CodeRepo +from tests.unit.clickhouse_local import CLICKHOUSE, LocalCH, ddl + +pytestmark = pytest.mark.skipif(CLICKHOUSE is None, reason="clickhouse binary not installed") + + +def _row(path: str, lexemes: str, *, org: str = "o", ns: str = "n", model: str = "m") -> str: + cid = uuid.uuid5(uuid.NAMESPACE_URL, f"{org}/{ns}/{model}/{path}") + return ( + "INSERT INTO loom.code_chunks (org, namespace, embedding_model, path, chunk_id, " + "content, lexemes, embedding) VALUES " + f"('{org}', '{ns}', '{model}', '{path}', '{cid}', '', '{lexemes}', " + "arrayResize([0.1], 3072, 0.1))" + ) + + +def _repo(*rows: str) -> CodeRepo: + return CodeRepo( # type: ignore[arg-type] + LocalCH("SET enable_quantized_codec = 1", ddl("code_chunks"), *rows) + ) + + +@pytest.mark.parametrize( + "identifier", ["__init__", "_private", "_x", "__", "quota_store", "parseQuotaKey"] +) +@pytest.mark.asyncio +async def test_stored_identifier_survives_the_text_index_gate(identifier: str) -> None: + tokens = code_tokens(identifier) + repo = _repo(_row("src/target.py", lexemes_text(identifier))) + rows = await repo.lexical_search_chunks("o", "n", "m", tokens, [1.0] * len(tokens)) + assert [r["path"] for r in rows] == ["src/target.py"] + assert rows[0]["lex_score"] == len(tokens) + + +@pytest.mark.parametrize( + "query,partial", [("__init__", "init"), ("_private", "private"), ("_x", "x")] +) +@pytest.mark.asyncio +async def test_gate_subtokens_without_an_exact_lexeme_do_not_become_hits( + query: str, partial: str +) -> None: + repo = _repo(_row("src/partial.py", partial)) + rows = await repo.lexical_search_chunks("o", "n", "m", code_tokens(query), [1.0]) + assert rows == [] + + +@pytest.mark.asyncio +async def test_identifier_gate_keeps_identity_model_prefix_and_pool() -> None: + repo = _repo( + _row("src/one.py", "__init__"), + _row("src/two.py", "__init__"), + _row("other/hidden.py", "__init__"), + _row("src/org.py", "__init__", org="other"), + _row("src/ns.py", "__init__", ns="other"), + _row("src/model.py", "__init__", model="other"), + ) + rows = await repo.lexical_search_chunks( + "o", "n", "m", ["__init__"], [1.0], path_prefix="src/", pool=1 + ) + assert len(rows) == 1 and rows[0]["path"] in {"src/one.py", "src/two.py"} diff --git a/tests/unit/test_code_symbol_python_forms.py b/tests/unit/test_code_symbol_python_forms.py new file mode 100644 index 0000000000..81de95eeb5 --- /dev/null +++ b/tests/unit/test_code_symbol_python_forms.py @@ -0,0 +1,98 @@ +"""Python import spellings resolve the parser's retained module-qualified names.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from server.code.app.index import symbol_forms as sf +from server.code.domain.extract.lang_python import parse_python + + +def _definitions(path: str, source: bytes) -> list[dict[str, Any]]: + return [ + { + "symbol_id": symbol.id, + "qualified_name": symbol.qualified_name, + "name": symbol.name, + "kind": symbol.kind, + "path": path, + "tier": "treesitter", + } + for symbol in parse_python(path, source).symbols + ] + + +class Repo: + def __init__(self, rows: list[dict[str, Any]]) -> None: + self.rows = rows + self.calls: list[tuple[str, int]] = [] + + async def symbols_by_name(self, org, ns, name, *, limit=50): + assert (org, ns) == ("org", "repo") + self.calls.append((name, limit)) + return [row for row in self.rows if row["name"] == name][:limit] + + async def grep_chunks(self, *args, **kwargs): + return [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("path", "written"), + [ + ("src/pkg/json/__init__.py", "src.pkg.json::dumps"), + ("src/pkg/json/__init__.py", "pkg.json.dumps"), + ("src/pkg/json/__init__.py", "json::dumps"), + ("src/pkg/json/encoder.py", "src.pkg.json.encoder.dumps"), + ("src/pkg/json/encoder.py", "pkg.json.encoder.dumps"), + ], +) +async def test_import_spelling_resolves_existing_function(path: str, written: str) -> None: + definitions = _definitions(path, b"def dumps(value):\n return str(value)\n") + repo = Repo(definitions) + rows, by = await sf.resolve_tolerant(repo, "org", "repo", written) + assert rows == definitions + assert by == "python_module" + assert repo.calls == [("dumps", sf.TAIL_READ_LIMIT)] + + +@pytest.mark.asyncio +async def test_module_spelling_preserves_nested_class_and_constructor() -> None: + definitions = _definitions( + "src/pkg/__init__.py", b"class Client:\n def __init__(self):\n pass\n" + ) + repo = Repo(definitions) + rows, by = await sf.resolve_tolerant(repo, "org", "repo", "pkg.Client.__init__") + assert [row["qualified_name"] for row in rows] == ["src.pkg.__init__::Client::__init__"] + assert by == "python_module" + + +@pytest.mark.asyncio +async def test_duplicate_import_suffixes_remain_ambiguous() -> None: + rows = _definitions("a/pkg/__init__.py", b"def dumps(value):\n return value\n") + rows += _definitions("b/pkg/__init__.py", b"def dumps(value):\n return value\n") + assert await sf.resolve_tolerant(Repo(rows), "org", "repo", "pkg.dumps") == ([], "") + + +@pytest.mark.asyncio +async def test_python_alias_does_not_rewrite_another_language() -> None: + rows = [ + { + "symbol_id": "native", + "qualified_name": "pkg.__init__::dumps", + "name": "dumps", + "kind": "function", + "path": "src/json.cpp", + "tier": "treesitter", + } + ] + assert await sf.resolve_tolerant(Repo(rows), "org", "repo", "pkg.dumps") == ([], "") + + +@pytest.mark.asyncio +async def test_incomplete_candidate_pool_never_claims_a_unique_alias() -> None: + rows = _definitions("pkg/__init__.py", b"def dumps(value):\n return value\n") + rows += [{**rows[0], "qualified_name": f"other{i}::dumps"} for i in range(sf.TAIL_READ_LIMIT)] + assert await sf.resolve_tolerant(Repo(rows), "org", "repo", "pkg.dumps") == ([], "")