diff --git a/Makefile.cbm b/Makefile.cbm index 1b4aeda2..7cb42edd 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -161,10 +161,8 @@ MCP_SRCS = # Graph buffer module is Zig: see CBM_ZIG_SRC. -# Pipeline module: implemented in Zig (src/pipeline/*.zig, part of libcbm_zig). -# lsp_resolve_shim.c compiles one inline function of lsp_resolve.h that -# translate-c cannot hand to the Zig side. -PIPELINE_SRCS = src/pipeline/lsp_resolve_shim.c +# Pipeline module: implemented in Zig (src/pipeline/*.zig, part of libcbm_zig) +PIPELINE_SRCS = # SimHash / MinHash, watcher and git context modules are Zig: see CBM_ZIG_SRC. @@ -457,7 +455,7 @@ CBM_ZIG_SRCS = $(CBM_ZIG_SRC) src/cbm_zig_c.h src/cli_zig/cbm_cli_zig.zig src/cl src/semantic/semantic.zig \ src/semantic/ast_profile.h src/semantic/rotsq.h src/semantic/semantic.h \ vendored/nomic/code_vectors.h vendored/nomic/code_tokens.h \ - src/pipeline/fqn.zig src/pipeline/laravel_routing.zig \ + src/pipeline/fqn.zig src/pipeline/laravel_routing.zig src/pipeline/lsp_resolve.zig \ src/pipeline/pass_compile_commands.zig src/pipeline/pass_complexity.zig \ src/pipeline/pass_configlink.zig src/pipeline/pass_configures.zig \ src/pipeline/pass_enrichment.zig src/pipeline/pass_envscan.zig \ diff --git a/build.zig b/build.zig index 1cf882a1..06ccec85 100644 --- a/build.zig +++ b/build.zig @@ -536,7 +536,6 @@ fn grammarSources(b: *std.Build) []const []const u8 { const native_sources = [_][]const u8{ "src/foundation/hash_table.c", "src/foundation/c_shim.c", - "src/pipeline/lsp_resolve_shim.c", "vendored/yyjson/yyjson.c", "internal/cbm/cbm.c", "internal/cbm/extract_defs.c", diff --git a/src/cbm_zig.zig b/src/cbm_zig.zig index 56ddab4c..c9bddaf7 100644 --- a/src/cbm_zig.zig +++ b/src/cbm_zig.zig @@ -64,6 +64,7 @@ comptime { checkExports(@import("mcp/server.zig")); checkExports(@import("pipeline/fqn.zig")); checkExports(@import("pipeline/laravel_routing.zig")); + checkExports(@import("pipeline/lsp_resolve.zig")); checkExports(@import("pipeline/pass_compile_commands.zig")); checkExports(@import("pipeline/pass_complexity.zig")); checkExports(@import("pipeline/pass_configlink.zig")); diff --git a/src/foundation/vmem.c b/src/foundation/vmem.c deleted file mode 100644 index 86befbe4..00000000 --- a/src/foundation/vmem.c +++ /dev/null @@ -1,196 +0,0 @@ -/* - * vmem.c — Budget-tracked virtual memory allocator. - * - * Allocates via mmap (POSIX) or VirtualAlloc (Windows) to bypass - * ptmalloc2's per-thread arena fragmentation. All allocations are - * page-aligned, zeroed by the OS, and tracked against a configurable - * budget (fraction of physical RAM). - * - * Pressure events are logged with hysteresis to avoid log storms - * near the budget boundary. - */ -#include "vmem.h" -#include "platform.h" -#include "log.h" - -#include -#include /* snprintf */ -#include /* memset */ - -#ifdef _WIN32 -#ifndef WIN32_LEAN_AND_MEAN -#define WIN32_LEAN_AND_MEAN -#endif -#include -#else -#include -#include /* sysconf, _SC_PAGESIZE */ -#endif - -/* ── Static state (initialized once) ──────────────────────────── */ - -static atomic_size_t g_allocated = 0; /* current total allocated */ -static atomic_size_t g_peak = 0; /* high-water mark */ -static size_t g_budget = 0; /* budget in bytes */ -static atomic_int g_was_over = 0; /* hysteresis: was over budget? */ -static atomic_int g_initialized = 0; /* init guard */ - -/* ── Page size ─────────────────────────────────────────────────── */ - -static size_t page_size(void) { -#ifdef _WIN32 - SYSTEM_INFO si; - GetSystemInfo(&si); - return (size_t)si.dwPageSize; -#else - long ps = sysconf(_SC_PAGESIZE); - return ps > 0 ? (size_t)ps : 4096; -#endif -} - -/* Round up to page boundary. */ -static size_t round_to_page(size_t size) { - size_t ps = page_size(); - return (size + ps - SKIP_ONE) & ~(ps - SKIP_ONE); -} - -/* ── Pressure logging ──────────────────────────────────────────── */ - -#define MB_DIVISOR ((size_t)(CBM_SZ_1K * CBM_SZ_1K)) - -static void check_pressure(size_t allocated) { - if (g_budget == 0) { - return; - } - - bool over = allocated > g_budget; - int was = atomic_load(&g_was_over); - - if (over && !was) { - /* Transition: under → over */ - atomic_store(&g_was_over, 1); - char alloc_mb[CBM_SZ_32]; - char budget_mb[CBM_SZ_32]; - char pct_str[CBM_SZ_16]; - snprintf(alloc_mb, sizeof(alloc_mb), "%zu", allocated / MB_DIVISOR); - snprintf(budget_mb, sizeof(budget_mb), "%zu", g_budget / MB_DIVISOR); - snprintf(pct_str, sizeof(pct_str), "%zu", - g_budget > 0 ? (allocated * CBM_PERCENT) / g_budget : 0); - cbm_log_warn("mem.pressure.warn", "allocated_mb", alloc_mb, "budget_mb", budget_mb, "pct", - pct_str); - } else if (!over && was) { - /* Transition: over → under */ - atomic_store(&g_was_over, 0); - char alloc_mb[CBM_SZ_32]; - char budget_mb[CBM_SZ_32]; - char pct_str[CBM_SZ_16]; - snprintf(alloc_mb, sizeof(alloc_mb), "%zu", allocated / MB_DIVISOR); - snprintf(budget_mb, sizeof(budget_mb), "%zu", g_budget / MB_DIVISOR); - snprintf(pct_str, sizeof(pct_str), "%zu", - g_budget > 0 ? (allocated * CBM_PERCENT) / g_budget : 0); - cbm_log_info("mem.pressure.ok", "allocated_mb", alloc_mb, "budget_mb", budget_mb, "pct", - pct_str); - } -} - -/* ── Update peak ───────────────────────────────────────────────── */ - -static void update_peak(size_t allocated) { - size_t old_peak = atomic_load(&g_peak); - while (allocated > old_peak) { - if (atomic_compare_exchange_weak(&g_peak, &old_peak, allocated)) { - break; - } - } -} - -/* ── Public API ────────────────────────────────────────────────── */ - -void cbm_vmem_init(double ram_fraction) { - /* Only first call takes effect */ - int expected = 0; - if (!atomic_compare_exchange_strong(&g_initialized, &expected, 1)) { - return; - } - - if (ram_fraction <= 0.0 || ram_fraction > 1.0) { - ram_fraction = 0.5; - } - - cbm_system_info_t info = cbm_system_info(); - g_budget = (size_t)((double)info.total_ram * ram_fraction); - - char budget_mb[CBM_SZ_32]; - char ram_mb[CBM_SZ_32]; - snprintf(budget_mb, sizeof(budget_mb), "%zu", g_budget / MB_DIVISOR); - snprintf(ram_mb, sizeof(ram_mb), "%zu", info.total_ram / MB_DIVISOR); - cbm_log_info("vmem.init", "budget_mb", budget_mb, "total_ram_mb", ram_mb); -} - -void *cbm_vmem_alloc(size_t size) { - if (size == 0) { - return NULL; - } - - size_t alloc_size = round_to_page(size); - -#ifdef _WIN32 - void *ptr = VirtualAlloc(NULL, alloc_size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); -#else - void *ptr = mmap(NULL, alloc_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (ptr == MAP_FAILED) { - ptr = NULL; - } -#endif - - if (!ptr) { - cbm_log_error("vmem.alloc.fail", "size_mb", size > MB_DIVISOR ? "large" : "small"); - return NULL; - } - - size_t new_total = atomic_fetch_add(&g_allocated, alloc_size) + alloc_size; - update_peak(new_total); - check_pressure(new_total); - - return ptr; -} - -void cbm_vmem_free(void *ptr, size_t size) { - if (!ptr || size == 0) { - return; - } - - size_t free_size = round_to_page(size); - -#ifdef _WIN32 - VirtualFree(ptr, 0, MEM_RELEASE); -#else - munmap(ptr, free_size); -#endif - - size_t new_total = atomic_fetch_sub(&g_allocated, free_size) - free_size; - check_pressure(new_total); -} - -size_t cbm_vmem_allocated(void) { - return atomic_load(&g_allocated); -} - -size_t cbm_vmem_peak(void) { - return atomic_load(&g_peak); -} - -size_t cbm_vmem_budget(void) { - return g_budget; -} - -bool cbm_vmem_over_budget(void) { - return atomic_load(&g_allocated) > g_budget; -} - -size_t cbm_vmem_worker_budget(int num_workers) { - if (num_workers <= 0) { - num_workers = SKIP_ONE; - } - return g_budget / (size_t)num_workers; -} diff --git a/src/foundation/vmem.h b/src/foundation/vmem.h deleted file mode 100644 index ccfc7ca3..00000000 --- a/src/foundation/vmem.h +++ /dev/null @@ -1,50 +0,0 @@ -/* - * vmem.h — Budget-tracked virtual memory allocator. - * - * Allocates via mmap/VirtualAlloc instead of malloc to avoid ptmalloc2's - * per-thread arena fragmentation (the cause of 321GB VSZ on Linux kernel - * indexing with 12 workers). - * - * Budget: configurable fraction of physical RAM (default 50%). When - * allocation exceeds the budget, a pressure event is logged. Workers - * never block — the budget is advisory for monitoring/alerting only. - * - * All allocations are page-aligned and zeroed by the OS. - */ -#ifndef CBM_VMEM_H -#define CBM_VMEM_H - -#include -#include - -/* Initialize vmem with a budget = ram_fraction * total_physical_ram. - * Thread-safe: only the first call takes effect. - * Must be called before any cbm_vmem_alloc() calls. */ -void cbm_vmem_init(double ram_fraction); - -/* Allocate size bytes via mmap/VirtualAlloc. - * Returns page-aligned, zeroed memory. NULL on failure. - * Tracks allocation against the budget and logs pressure events. */ -void *cbm_vmem_alloc(size_t size); - -/* Free memory previously allocated by cbm_vmem_alloc. - * size must match the original allocation size. */ -void cbm_vmem_free(void *ptr, size_t size); - -/* Current total allocated bytes (atomic). */ -size_t cbm_vmem_allocated(void); - -/* Peak allocated bytes seen so far. */ -size_t cbm_vmem_peak(void); - -/* Total budget in bytes. */ -size_t cbm_vmem_budget(void); - -/* Returns true if current allocation exceeds the budget. */ -bool cbm_vmem_over_budget(void); - -/* Per-worker budget hint: budget / num_workers. - * For monitoring/logging only — workers never block. */ -size_t cbm_vmem_worker_budget(int num_workers); - -#endif /* CBM_VMEM_H */ diff --git a/src/pipeline/lsp_resolve.h b/src/pipeline/lsp_resolve.h index 946cf3c2..c2710485 100644 --- a/src/pipeline/lsp_resolve.h +++ b/src/pipeline/lsp_resolve.h @@ -1,26 +1,18 @@ /* * lsp_resolve.h — Shared LSP-override resolver for the call-edge pipeline. * - * The call resolver (pass_parallel.c resolve_file_calls) and the tests - * look up an LSP-resolved call for a given (caller, callee) pair here - * before falling back to the registry's name-based resolver, so every - * caller admits the same set of LSP overrides. Edge emission stays with - * the caller; this header only does the matching. - * - * Inline-only: no .c file needed. + * Implemented in src/pipeline/lsp_resolve.zig. The call resolver + * (pass_parallel.zig) and the tests look up an LSP-resolved call for a given + * (caller, callee) pair here before falling back to the registry's name-based + * resolver, so every caller admits the same set of LSP overrides. */ #ifndef CBM_PIPELINE_LSP_RESOLVE_H #define CBM_PIPELINE_LSP_RESOLVE_H #include "cbm.h" #include "graph_buffer/graph_buffer.h" -#include "foundation/constants.h" -#include -#include -#include -#include -#include +#include /* Confidence floor below which LSP-resolved calls are ignored and the * registry resolver is consulted instead. Locked at 0.6 per the v1 @@ -29,557 +21,20 @@ * (Go, C/C++, Python, PHP). */ #define CBM_LSP_CONFIDENCE_FLOOR 0.6f -/* Bare last segment of a (possibly qualified) name, splitting on the LAST - * member/scope separator. C++ textual callees carry `::` (Class::method, - * Ns::f) and `->` (p->run), while the LSP records dotted internal QNs - * (Class.method). Splitting only on '.' (strrchr) leaves `Math::square` - * and `p->run` intact, so they never match the LSP's `square`/`run` short - * name and the type-aware strategy is silently dropped to the textual - * registry. Treat '.', ':' and '>' as terminal separators so the bare - * method name is recovered on BOTH the QN side (dotted, occasionally `::` - * for template/alias scopes) and the textual side (`.`/`::`/`->`). Other - * languages' callee names contain none of `::`/`->`, so this is a no-op - * for them. */ -static inline const char *cbm_lsp_bare_segment(const char *name) { - if (!name) { - return name; - } - const char *seg = name; - for (const char *p = name; *p; p++) { - /* '.' (dotted QN / Java-style member) and ':' (C++ `::`, last colon - * wins) are member/scope separators. '>' is only a separator when it - * closes the `->` arrow (preceded by '-'); a bare '>' closes a template - * argument list ("identity") and must NOT split, else the segment - * would be the empty string after the trailing '>'. */ - if (*p == '.' || *p == ':' || (*p == '>' && p != name && p[-1] == '-')) { - seg = p + SKIP_ONE; - } - } - return seg; -} - -/* Tail helper: return the start of the final two dot-separated segments - * ("Class.method") or NULL when the QN is too short. */ -static inline const char *cbm_pipeline_qn_class_method_tail(const char *qn) { - if (!qn) { - return NULL; - } - const char *last = strrchr(qn, '.'); - if (!last || last == qn) { - return NULL; - } - const char *second = last; - while (second > qn) { - second--; - if (*second == '.') { - if (second == qn) { - return qn; - } - return second + 1; - } - } - return qn; -} - -static inline const char *cbm_pipeline_call_callee_leaf(const char *callee_name) { - return cbm_lsp_bare_segment(callee_name); -} - -/* Gate for the unique-`Class.method`-tail fallbacks below. Tail-matching by - * leaf is safe where class-per-file package semantics hold — the JVM - * languages (Java/Kotlin): the declared `package` is ground truth, a class - * name is unique within a package, and mixed Gradle/Maven source roots - * (`src/main/java` + `src/main/kotlin`) legitimately produce path-derived - * module QNs that disagree with the package-shaped QNs the LSP emits, so - * the tail is the only reliable join key. In other languages the same-name - * guarantee does not exist (Python/TS re-export shims, Go internal clones, - * C++ template instantiations), and a single wrong-module coincidence - * would fabricate a CALLS edge — so the fallbacks stay off there. */ -static inline bool cbm_pipeline_lsp_allow_tail_match(CBMLanguage lang) { - return lang == CBM_LANG_JAVA || lang == CBM_LANG_KOTLIN; -} - -static inline int cbm_pipeline_qn_class_method_tail_eq(const char *qn, const char *tail) { - const char *qt = cbm_pipeline_qn_class_method_tail(qn); - return qt && tail && strcmp(qt, tail) == 0; -} - -/* An invocation row that carries a site span (end > start) proves ONE call: it - * joins only the call whose node starts at that byte, so `db.deinit()` and - * `st.deinit()` in one function each keep their own target. Rows without a span - * join every same-named call in the caller, as before. */ -static inline bool cbm_pipeline_lsp_site_admits(const CBMResolvedCall *rc, const CBMCall *call) { - return rc->site_end_byte <= rc->site_start_byte || - rc->site_start_byte == call->site_start_byte; -} - -/* Look up the highest-confidence LSP-resolved call entry whose caller QN - * matches the textual call's enclosing function and whose callee QN - * short-name matches the textual callee. Returns a pointer into `arr` - * or NULL if no qualifying entry exists. - * - * Match rule: - * 1. exact caller_qn + callee short-name match wins first; - * 2. if no exact caller match exists AND allow_tail_match is set - * (JVM callers only, see cbm_pipeline_lsp_allow_tail_match), a - * unique Class.method tail match between rc->caller_qn and - * call->enclosing_func_qn may win; - * 3. ambiguous tails return NULL so the registry fallback stays in - * control. - * - * The pointer returned aliases into `arr` and stays valid as long as the - * underlying CBMFileResult is alive. */ -static inline const CBMResolvedCall *cbm_pipeline_find_lsp_resolution( - const CBMResolvedCallArray *arr, const CBMCall *call, bool allow_tail_match) { - if (!arr || arr->count == 0 || !call) { - return NULL; - } - if (!call->enclosing_func_qn || !call->callee_name) { - return NULL; - } - - const CBMResolvedCall *best_exact = NULL; - for (int i = 0; i < arr->count; i++) { - const CBMResolvedCall *rc = &arr->items[i]; - if (!rc->caller_qn || !rc->callee_qn || rc->kind != CBM_RESOLVED_INVOCATION) { - continue; - } - if (rc->confidence < CBM_LSP_CONFIDENCE_FLOOR) { - continue; - } - if (strcmp(rc->caller_qn, call->enclosing_func_qn) != 0) { - continue; - } - if (!cbm_pipeline_lsp_site_admits(rc, call)) { - continue; - } - const char *short_name = cbm_lsp_bare_segment(rc->callee_qn); - /* The call's callee_name is receiver-qualified for method/qualified - * calls ("c.inc", "A.Helper", "Math::square", "p->run"); the LSP - * records the resolved class-qualified callee_qn ("Class.inc"). Compare - * the bare last segment on BOTH sides so method-dispatch resolutions - * join — the LSP already did the receiver->type resolution, and matching - * the full "c.inc" against "inc" would always miss, silently dropping the - * type-aware LSP strategy to the weaker textual registry. Free-function - * calls (bare callee_name) are unaffected. */ - const char *call_short = cbm_lsp_bare_segment(call->callee_name); - if (strcmp(short_name, call_short) != 0) { - /* Indirect/implicit resolution: the textual callee differs from the - * resolved callee_qn's short name. A function-pointer / DLL call's - * callee is the pointer name (`fp`); a C++ destructor's only textual - * anchor is the deleted operand (`p`, vs. the `T.~T` callee QN). In - * both the LSP stashed the original textual name in `reason`. Match - * the call site on that name, gated to those strategies so `reason` - * is never misread as an unresolved-call diagnostic. */ - if (!(rc->reason && rc->strategy && - (strcmp(rc->strategy, "lsp_func_ptr") == 0 || - strcmp(rc->strategy, "lsp_dll_resolve") == 0 || - strcmp(rc->strategy, "lsp_dict_dispatch") == 0 || - strcmp(rc->strategy, "lsp_destructor") == 0 || - strcmp(rc->strategy, "php_method_dynamic") == 0) && - strcmp(cbm_lsp_bare_segment(rc->reason), call_short) == 0)) { - continue; - } - } - if (!best_exact || rc->confidence > best_exact->confidence) { - best_exact = rc; - } - } - if (best_exact) { - return best_exact; - } - if (!allow_tail_match) { - return NULL; - } - - const char *call_tail = cbm_pipeline_qn_class_method_tail(call->enclosing_func_qn); - if (!call_tail) { - return NULL; - } - - const CBMResolvedCall *best_tail = NULL; - for (int i = 0; i < arr->count; i++) { - const CBMResolvedCall *rc = &arr->items[i]; - if (!rc->caller_qn || !rc->callee_qn || rc->kind != CBM_RESOLVED_INVOCATION) { - continue; - } - if (rc->confidence < CBM_LSP_CONFIDENCE_FLOOR) { - continue; - } - const char *short_name = strrchr(rc->callee_qn, '.'); - short_name = short_name ? short_name + SKIP_ONE : rc->callee_qn; - const char *call_leaf = cbm_pipeline_call_callee_leaf(call->callee_name); - if (!call_leaf || strcmp(short_name, call_leaf) != 0) { - continue; - } - if (!cbm_pipeline_qn_class_method_tail_eq(rc->caller_qn, call_tail)) { - continue; - } - if (!cbm_pipeline_lsp_site_admits(rc, call)) { - continue; - } - if (best_tail) { - return NULL; - } - best_tail = rc; - } - return best_tail; -} - -/* Tail-match cost counters (upstream f6e3af43). The tail scan in - * cbm_pipeline_lsp_target_node has no candidate cap and its candidate set - * grows with the corpus, so candidates per lookup is what would turn resolve - * superlinear. Counted always (two relaxed adds), defined in pass_parallel.c - * and logged at the end of parallel resolve as parallel.resolve.scan_cost. */ -extern _Atomic uint64_t g_lsp_tail_lookups; -extern _Atomic uint64_t g_lsp_tail_candidates; - -/* Resolve an LSP-emitted callee_qn to a graph-buffer node. - * - * Per-file LSPs sometimes emit `callee_qn` as the raw package-shaped - * import path the source code uses rather than the project-qualified QN - * the gbuf actually stores. The fallback rule is: - * 1. try the LSP-emitted QN as-is; - * 2. retry with `.` when needed; - * 3. if both fail AND allow_tail_match is set (JVM callers only, see - * cbm_pipeline_lsp_allow_tail_match), use the exact node-name index - * to narrow candidates by short method name and accept exactly one - * Function/Method whose qualified_name has the same Class.method - * tail. - * - * Returns the matching node, or NULL if neither lookup hits. */ -static inline const cbm_gbuf_node_t *cbm_pipeline_lsp_target_node(const cbm_gbuf_t *gbuf, - const char *project_name, - const char *callee_qn, - bool allow_tail_match) { - if (!gbuf || !callee_qn) { - return NULL; - } - const cbm_gbuf_node_t *direct = cbm_gbuf_find_by_qn(gbuf, callee_qn); - if (direct) { - return direct; - } - /* Exact retries, in tie-rule order (CBM_MACRO_QN_SUFFIX): the definition - * first -- the QN as given, then with the project prefix -- and only when - * neither names a node the C-preprocessor macro of that name, in the same - * two spellings. The macro case is a prototype the C resolver knows whose - * only node in the graph is its rename macro (`extern T f(...);` plus - * `#define f f_v2`). One buffer, ".#macro", serves all - * three retries. */ - size_t proj_len = (project_name && project_name[0]) ? strlen(project_name) : 0; - size_t callee_len = strlen(callee_qn); - size_t suffix_len = sizeof(CBM_MACRO_QN_SUFFIX) - 1U; - bool add_prefix = proj_len > 0 && !(strncmp(callee_qn, project_name, proj_len) == 0 && - callee_qn[proj_len] == '.'); - if (proj_len <= SIZE_MAX - callee_len - suffix_len - 2U) { - char *buf = (char *)malloc(proj_len + 1U + callee_len + suffix_len + 1U); - if (buf) { - char *own = buf + proj_len + 1U; /* callee_qn inside the buffer */ - if (proj_len > 0) { - memcpy(buf, project_name, proj_len); - } - buf[proj_len] = '.'; - memcpy(own, callee_qn, callee_len + 1U); - const cbm_gbuf_node_t *hit = add_prefix ? cbm_gbuf_find_by_qn(gbuf, buf) : NULL; - if (!hit) { - memcpy(own + callee_len, CBM_MACRO_QN_SUFFIX, suffix_len + 1U); - hit = cbm_gbuf_find_by_qn(gbuf, own); - if (!hit && add_prefix) { - hit = cbm_gbuf_find_by_qn(gbuf, buf); - } - } - free(buf); - if (hit) { - return hit; - } - } - } - if (!allow_tail_match) { - return NULL; - } - - const char *short_name = strrchr(callee_qn, '.'); - short_name = short_name ? short_name + SKIP_ONE : callee_qn; - const char *callee_tail = cbm_pipeline_qn_class_method_tail(callee_qn); - if (!callee_tail) { - return NULL; - } - const cbm_gbuf_node_t **hits = NULL; - int hit_count = 0; - if (cbm_gbuf_find_by_name(gbuf, short_name, &hits, &hit_count) != 0 || hit_count == 0) { - return NULL; - } - atomic_fetch_add_explicit(&g_lsp_tail_lookups, 1, memory_order_relaxed); - atomic_fetch_add_explicit(&g_lsp_tail_candidates, (uint64_t)hit_count, memory_order_relaxed); - - const cbm_gbuf_node_t *match = NULL; - for (int i = 0; i < hit_count; i++) { - const cbm_gbuf_node_t *cand = hits[i]; - if (!cand || !cand->label || !cand->qualified_name) { - continue; - } - if (strcmp(cand->label, "Function") != 0 && strcmp(cand->label, "Method") != 0) { - continue; - } - if (!cbm_pipeline_qn_class_method_tail_eq(cand->qualified_name, callee_tail)) { - continue; - } - if (match) { - return NULL; - } - match = cand; - } - return match; -} - -/* ── Callable references (CALL_REFERENCE) ───────────────────────────────── - * - * A function or method named as a value is a CALL_REFERENCE only when the - * language LSP proved one exact callable at that source occurrence. The - * semantic join is by (enclosing function, exact byte span of the reference - * token); the textual registry never earns CALL_REFERENCE, it only yields an - * ordinary USAGE. Reference rows never take part in the CALLS join. */ - -/* Extraction marked this usage as a callable-reference candidate: explicit - * reference syntax, or a bare value in a narrow position (direct argument, - * assignment right-hand side) the language LSP knows how to prove. */ -static inline bool cbm_pipeline_usage_is_reference_candidate(const CBMUsage *usage) { - return usage && (usage->kind == CBM_USAGE_CALL_REFERENCE || usage->may_be_call_reference) && - usage->site_end_byte > usage->site_start_byte; -} - -/* Graph labels a CALL_REFERENCE may point at (a Class covers constructor - * references such as Java `Foo::new`). Anything else (a field, a variable, an - * interface) proved by the LSP still lands as USAGE. */ -static inline bool cbm_pipeline_node_is_callable_target(const cbm_gbuf_node_t *node) { - if (!node || !node->label) { - return false; - } - return strcmp(node->label, "Function") == 0 || strcmp(node->label, "Method") == 0 || - strcmp(node->label, "Constructor") == 0 || strcmp(node->label, "Class") == 0; -} - -/* Per-file index of CALL_REFERENCE rows sorted by site, so each candidate - * usage costs one binary search instead of a scan of every resolved row. */ -typedef struct { - const CBMResolvedCall **rows; - int count; -} cbm_pipeline_ref_index_t; - -static inline int cbm_pipeline_ref_row_cmp(const void *a, const void *b) { - const CBMResolvedCall *l = *(const CBMResolvedCall *const *)a; - const CBMResolvedCall *r = *(const CBMResolvedCall *const *)b; - if (l->site_start_byte != r->site_start_byte) { - return l->site_start_byte < r->site_start_byte ? -1 : 1; - } - if (l->site_end_byte != r->site_end_byte) { - return l->site_end_byte < r->site_end_byte ? -1 : 1; - } - /* Stable order within one site: confidence ties keep the first row. */ - return l < r ? -1 : (l > r ? 1 : 0); -} - -/* Returns false only on allocation failure; the caller then finds no semantic - * row and every candidate stays USAGE (fail closed). */ -static inline bool cbm_pipeline_ref_index_build(const CBMResolvedCallArray *arr, - cbm_pipeline_ref_index_t *index) { - index->rows = NULL; - index->count = 0; - if (!arr || arr->count <= 0) { - return true; - } - int n = 0; - for (int i = 0; i < arr->count; i++) { - if (arr->items[i].kind == CBM_RESOLVED_CALL_REFERENCE) { - n++; - } - } - if (n == 0) { - return true; - } - const CBMResolvedCall **rows = (const CBMResolvedCall **)malloc((size_t)n * sizeof(*rows)); - if (!rows) { - return false; - } - int k = 0; - for (int i = 0; i < arr->count; i++) { - if (arr->items[i].kind == CBM_RESOLVED_CALL_REFERENCE) { - rows[k++] = &arr->items[i]; - } - } - qsort(rows, (size_t)n, sizeof(*rows), cbm_pipeline_ref_row_cmp); - index->rows = rows; - index->count = n; - return true; -} - -static inline void cbm_pipeline_ref_index_free(cbm_pipeline_ref_index_t *index) { - free(index->rows); - index->rows = NULL; - index->count = 0; -} - -/* Scan the rows sharing the usage's byte span. `tail` NULL requires the exact - * enclosing function; otherwise the caller's Class.method tail must match. */ -static inline const CBMResolvedCall *cbm_pipeline_ref_scan(const cbm_pipeline_ref_index_t *index, - int first, const CBMUsage *usage, - const char *tail) { - const char *ref_leaf = cbm_lsp_bare_segment(usage->ref_name); - const CBMResolvedCall *best = NULL; - for (int i = first; i < index->count; i++) { - const CBMResolvedCall *r = index->rows[i]; - if (r->site_start_byte != usage->site_start_byte || - r->site_end_byte != usage->site_end_byte) { - break; - } - if (!r->caller_qn || !r->callee_qn || r->confidence < CBM_LSP_CONFIDENCE_FLOOR) { - continue; - } - if (tail ? !cbm_pipeline_qn_class_method_tail_eq(r->caller_qn, tail) - : strcmp(r->caller_qn, usage->enclosing_func_qn) != 0) { - continue; - } - /* The row must name this reference: the callee leaf, or `reason` for an - * alias (`cb` proven to hold `handler`). */ - if (strcmp(cbm_lsp_bare_segment(r->callee_qn), ref_leaf) != 0 && - !(r->reason && strcmp(cbm_lsp_bare_segment(r->reason), ref_leaf) == 0)) { - continue; - } - if (!best) { - best = r; - } else if (strcmp(best->callee_qn, r->callee_qn) != 0) { - return NULL; /* two targets at one occurrence: ambiguous, fail closed */ - } else if (r->confidence > best->confidence) { - best = r; - } - } - return best; -} - -/* The exact semantic row for one candidate usage, or NULL. Rows must share the - * usage's byte span and enclosing function and clear the confidence floor. - * JVM callers (allow_tail_match) may fall back to a Class.method tail match on - * the caller, the same rule the CALLS join uses for package-shaped LSP QNs. */ -static inline const CBMResolvedCall *cbm_pipeline_find_lsp_reference( - const cbm_pipeline_ref_index_t *index, const CBMUsage *usage, bool allow_tail_match) { - if (!index || index->count == 0 || !cbm_pipeline_usage_is_reference_candidate(usage) || - !usage->enclosing_func_qn || !usage->ref_name) { - return NULL; - } - int lo = 0; - int hi = index->count; - while (lo < hi) { - int mid = lo + (hi - lo) / 2; - const CBMResolvedCall *r = index->rows[mid]; - if (r->site_start_byte < usage->site_start_byte || - (r->site_start_byte == usage->site_start_byte && - r->site_end_byte < usage->site_end_byte)) { - lo = mid + 1; - } else { - hi = mid; - } - } - const CBMResolvedCall *exact = cbm_pipeline_ref_scan(index, lo, usage, NULL); - if (exact || !allow_tail_match) { - return exact; - } - const char *tail = cbm_pipeline_qn_class_method_tail(usage->enclosing_func_qn); - return tail ? cbm_pipeline_ref_scan(index, lo, usage, tail) : NULL; -} - -/* Rows that take part in the CALLS join. Count-based heuristics that compare - * LSP output with textual calls must not count reference rows, nor the - * confidence-0 lsp_unresolved diagnostics the join ignores. */ -static inline int cbm_pipeline_invocation_row_count(const CBMResolvedCallArray *arr) { - int n = 0; - for (int i = 0; arr && i < arr->count; i++) { - if (arr->items[i].kind == CBM_RESOLVED_INVOCATION && - arr->items[i].confidence >= CBM_LSP_CONFIDENCE_FLOOR) { - n++; - } - } - return n; -} - -/* Rows that take part in the CALL_REFERENCE join. */ -static inline int cbm_pipeline_reference_row_count(const CBMResolvedCallArray *arr) { - int n = 0; - for (int i = 0; arr && i < arr->count; i++) { - n += arr->items[i].kind == CBM_RESOLVED_CALL_REFERENCE ? 1 : 0; - } - return n; -} - -/* Outcome of the exact semantic join for one usage. */ -typedef enum { - CBM_REF_JOIN_NONE = 0, /* no LSP row at this occurrence: registry fallback decides */ - CBM_REF_JOIN_CALLABLE, /* proven callable target: emit CALL_REFERENCE */ - CBM_REF_JOIN_VALUE, /* proven non-callable target: at most a USAGE */ - CBM_REF_JOIN_SKIP, /* a row owns the occurrence but its target is not in the - * graph: emit nothing rather than guess by name */ -} cbm_ref_join_t; - -static inline cbm_ref_join_t cbm_pipeline_join_lsp_reference( - const cbm_gbuf_t *gbuf, const char *project_name, const cbm_pipeline_ref_index_t *index, - const CBMUsage *usage, bool allow_tail_match, const cbm_gbuf_node_t **out_target, - const CBMResolvedCall **out_row) { - *out_target = NULL; - *out_row = NULL; - const CBMResolvedCall *row = cbm_pipeline_find_lsp_reference(index, usage, allow_tail_match); - if (!row) { - return CBM_REF_JOIN_NONE; - } - *out_row = row; - const cbm_gbuf_node_t *tgt = - cbm_pipeline_lsp_target_node(gbuf, project_name, row->callee_qn, allow_tail_match); - if (!tgt) { - return CBM_REF_JOIN_SKIP; - } - *out_target = tgt; - return cbm_pipeline_node_is_callable_target(tgt) ? CBM_REF_JOIN_CALLABLE : CBM_REF_JOIN_VALUE; -} - -/* Edge properties for a USAGE / CALL_REFERENCE edge. A CALL_REFERENCE always - * comes from an LSP row (row non-NULL), whose strategy and confidence are the - * edge's provenance, in the same keys CALLS edges use; USAGE stays - * {"callee":...}. esc_ref is already JSON-escaped. */ -static inline void cbm_pipeline_reference_props(char *buf, size_t size, const char *esc_ref, - const CBMResolvedCall *row) { - if (row && row->strategy && row->strategy[0]) { - snprintf(buf, size, "{\"callee\":\"%s\",\"confidence\":%.2f,\"strategy\":\"%s\"}", - esc_ref, row->confidence, row->strategy); - } else { - snprintf(buf, size, "{\"callee\":\"%s\"}", esc_ref); - } -} - -/* Registry fallback for a usage the LSP did not prove. A candidate emitted - * only for the join (reference_only) never falls back. Explicit JVM reference - * syntax may not bind by a bare-name guess (unique_name / suffix_match), - * except a Kotlin `::name` whose spelling the file imports. */ -static inline bool cbm_pipeline_reference_fallback_allowed(CBMLanguage lang, const CBMUsage *usage, - const char *strategy, - const char *const *import_names, - int import_count) { - if (usage->reference_only) { - return false; - } - if (usage->kind != CBM_USAGE_CALL_REFERENCE || - (lang != CBM_LANG_JAVA && lang != CBM_LANG_KOTLIN)) { - return true; - } - if (strategy && strategy[0] && strcmp(strategy, "unique_name") != 0 && - strcmp(strategy, "suffix_match") != 0) { - return true; - } - if (lang == CBM_LANG_KOTLIN && usage->ref_name && import_names) { - for (int i = 0; i < import_count; i++) { - if (import_names[i] && strcmp(import_names[i], usage->ref_name) == 0) { - return true; - } - } - } - return false; -} +/* True for the languages whose LSP joins may fall back to a unique + * `Class.method` tail match (Java and Kotlin). */ +bool cbm_pipeline_lsp_allow_tail_match(CBMLanguage lang); + +/* The highest-confidence LSP-resolved call in `arr` for the textual `call`, + * or NULL. The pointer aliases into `arr`. */ +const CBMResolvedCall *cbm_pipeline_find_lsp_resolution(const CBMResolvedCallArray *arr, + const CBMCall *call, + bool allow_tail_match); + +/* The graph-buffer node an LSP-emitted callee_qn names, or NULL. */ +const cbm_gbuf_node_t *cbm_pipeline_lsp_target_node(const cbm_gbuf_t *gbuf, + const char *project_name, + const char *callee_qn, + bool allow_tail_match); #endif /* CBM_PIPELINE_LSP_RESOLVE_H */ diff --git a/src/pipeline/lsp_resolve.zig b/src/pipeline/lsp_resolve.zig new file mode 100644 index 00000000..4e9e4a04 --- /dev/null +++ b/src/pipeline/lsp_resolve.zig @@ -0,0 +1,507 @@ +//! lsp_resolve.zig — Shared LSP-override resolver for the call-edge pipeline. +//! +//! The call resolver (pass_parallel.zig resolveFileCalls) and the tests look +//! up an LSP-resolved call for a given (caller, callee) pair here before +//! falling back to the registry's name-based resolver, so every caller admits +//! the same set of LSP overrides. Edge emission stays with the caller; this +//! module only does the matching. +//! +//! lsp_resolve.h declares the three functions the C tests call and the +//! CBM_LSP_CONFIDENCE_FLOOR constant; everything else is Zig-only. + +const std = @import("std"); +const c = @import("c"); +const interop = @import("../foundation/interop.zig"); +const util = @import("util.zig"); + +const CStr = util.CStr; +const ix = util.ix; +const streq = util.streq; +const Gbuf = c.cbm_gbuf_t; +const Node = c.cbm_gbuf_node_t; +const Row = c.CBMResolvedCall; +const Rows = c.CBMResolvedCallArray; +const Language = c.CBMLanguage; + +/// Confidence floor below which LSP-resolved calls are ignored and the +/// registry resolver is consulted instead (CBM_LSP_CONFIDENCE_FLOOR). +const confidence_floor: f32 = c.CBM_LSP_CONFIDENCE_FLOOR; + +/// Tail-match cost counters (upstream f6e3af43). The tail scan in +/// cbm_pipeline_lsp_target_node has no candidate cap and its candidate set +/// grows with the corpus, so candidates per lookup is what would turn resolve +/// superlinear. Counted always (two relaxed adds), reset and logged by +/// pass_parallel.zig as parallel.resolve.scan_cost. +pub var tail_lookups: std.atomic.Value(u64) = .init(0); +pub var tail_candidates: std.atomic.Value(u64) = .init(0); + +// ── Name helpers ──────────────────────────────────────────────────── + +/// Bare last segment of a (possibly qualified) name, splitting on the LAST +/// member/scope separator. C++ textual callees carry `::` (Class::method, +/// Ns::f) and `->` (p->run), while the LSP records dotted internal QNs +/// (Class.method). Splitting only on '.' (strrchr) leaves `Math::square` +/// and `p->run` intact, so they never match the LSP's `square`/`run` short +/// name and the type-aware strategy is silently dropped to the textual +/// registry. Treat '.', ':' and '>' as terminal separators so the bare +/// method name is recovered on BOTH the QN side (dotted, occasionally `::` +/// for template/alias scopes) and the textual side (`.`/`::`/`->`). Other +/// languages' callee names contain none of `::`/`->`, so this is a no-op +/// for them. +pub fn bareSegment(name: CStr) CStr { + if (name == null) return name; + var seg = name; + var p = name; + while (p[0] != 0) : (p += 1) { + // '.' (dotted QN / Java-style member) and ':' (C++ `::`, last colon + // wins) are member/scope separators. '>' is only a separator when it + // closes the `->` arrow (preceded by '-'); a bare '>' closes a template + // argument list ("identity") and must NOT split, else the segment + // would be the empty string after the trailing '>'. + if (p[0] == '.' or p[0] == ':' or (p[0] == '>' and p != name and (p - 1)[0] == '-')) { + seg = p + 1; + } + } + return seg; +} + +/// The text after the last '.', or the whole name when it has none. +fn lastDotSegment(qn: CStr) CStr { + const dot: CStr = c.strrchr(qn, '.'); + return if (dot != null) dot + 1 else qn; +} + +/// The start of the final two dot-separated segments ("Class.method"), or +/// null when the QN is too short. +fn qnClassMethodTail(qn: CStr) CStr { + if (qn == null) return null; + const last: CStr = c.strrchr(qn, '.'); + if (last == null or last == qn) return null; + var second = last; + while (second != qn) { + second -= 1; + if (second[0] == '.') return if (second == qn) qn else second + 1; + } + return qn; +} + +fn qnClassMethodTailEq(qn: CStr, tail: CStr) bool { + const qt = qnClassMethodTail(qn); + return qt != null and tail != null and streq(qt, tail); +} + +// ── CALLS join ────────────────────────────────────────────────────── + +/// Gate for the unique-`Class.method`-tail fallbacks below. Tail-matching by +/// leaf is safe where class-per-file package semantics hold — the JVM +/// languages (Java/Kotlin): the declared `package` is ground truth, a class +/// name is unique within a package, and mixed Gradle/Maven source roots +/// (`src/main/java` + `src/main/kotlin`) legitimately produce path-derived +/// module QNs that disagree with the package-shaped QNs the LSP emits, so +/// the tail is the only reliable join key. In other languages the same-name +/// guarantee does not exist (Python/TS re-export shims, Go internal clones, +/// C++ template instantiations), and a single wrong-module coincidence +/// would fabricate a CALLS edge — so the fallbacks stay off there. +pub export fn cbm_pipeline_lsp_allow_tail_match(lang: Language) bool { + return lang == c.CBM_LANG_JAVA or lang == c.CBM_LANG_KOTLIN; +} + +/// An invocation row that clears the confidence floor and names both ends. +fn isInvocationRow(rc: *const Row) bool { + return rc.caller_qn != null and rc.callee_qn != null and + rc.kind == c.CBM_RESOLVED_INVOCATION and rc.confidence >= confidence_floor; +} + +/// An invocation row that carries a site span (end > start) proves ONE call: it +/// joins only the call whose node starts at that byte, so `db.deinit()` and +/// `st.deinit()` in one function each keep their own target. Rows without a span +/// join every same-named call in the caller, as before. +fn siteAdmits(rc: *const Row, call: *const c.CBMCall) bool { + return rc.site_end_byte <= rc.site_start_byte or rc.site_start_byte == call.site_start_byte; +} + +/// Indirect/implicit resolution: the textual callee differs from the +/// resolved callee_qn's short name. A function-pointer / DLL call's callee +/// is the pointer name (`fp`); a C++ destructor's only textual anchor is the +/// deleted operand (`p`, vs. the `T.~T` callee QN). In both the LSP stashed +/// the original textual name in `reason`. Match the call site on that name, +/// gated to those strategies so `reason` is never misread as an +/// unresolved-call diagnostic. +fn reasonNamesCall(rc: *const Row, call_short: CStr) bool { + if (rc.reason == null or rc.strategy == null) return false; + const s = rc.strategy; + const indirect = streq(s, "lsp_func_ptr") or streq(s, "lsp_dll_resolve") or + streq(s, "lsp_dict_dispatch") or streq(s, "lsp_destructor") or + streq(s, "php_method_dynamic"); + return indirect and streq(bareSegment(rc.reason), call_short); +} + +/// Look up the highest-confidence LSP-resolved call entry whose caller QN +/// matches the textual call's enclosing function and whose callee QN +/// short-name matches the textual callee. Returns a pointer into `arr` +/// or null if no qualifying entry exists. +/// +/// Match rule: +/// 1. exact caller_qn + callee short-name match wins first; +/// 2. if no exact caller match exists AND allow_tail_match is set +/// (JVM callers only, see cbm_pipeline_lsp_allow_tail_match), a +/// unique Class.method tail match between rc.caller_qn and +/// call.enclosing_func_qn may win; +/// 3. ambiguous tails return null so the registry fallback stays in +/// control. +/// +/// The pointer returned aliases into `arr` and stays valid as long as the +/// underlying CBMFileResult is alive. +pub export fn cbm_pipeline_find_lsp_resolution( + arr_opt: ?*const Rows, + call_opt: ?*const c.CBMCall, + allow_tail_match: bool, +) ?*const Row { + const arr = arr_opt orelse return null; + const call = call_opt orelse return null; + if (arr.count == 0) return null; + if (call.enclosing_func_qn == null or call.callee_name == null) return null; + + // The call's callee_name is receiver-qualified for method/qualified + // calls ("c.inc", "A.Helper", "Math::square", "p->run"); the LSP + // records the resolved class-qualified callee_qn ("Class.inc"). Compare + // the bare last segment on BOTH sides so method-dispatch resolutions + // join — the LSP already did the receiver->type resolution, and matching + // the full "c.inc" against "inc" would always miss, silently dropping the + // type-aware LSP strategy to the weaker textual registry. Free-function + // calls (bare callee_name) are unaffected. + const call_short = bareSegment(call.callee_name); + var best_exact: ?*const Row = null; + for (0..ix(arr.count)) |i| { + const rc = util.at(arr.items, i); + if (!isInvocationRow(rc)) continue; + if (!streq(rc.caller_qn, call.enclosing_func_qn)) continue; + if (!siteAdmits(rc, call)) continue; + if (!streq(bareSegment(rc.callee_qn), call_short) and !reasonNamesCall(rc, call_short)) continue; + if (best_exact == null or rc.confidence > best_exact.?.confidence) best_exact = rc; + } + if (best_exact != null) return best_exact; + if (!allow_tail_match) return null; + + const call_tail = qnClassMethodTail(call.enclosing_func_qn); + if (call_tail == null) return null; + + var best_tail: ?*const Row = null; + for (0..ix(arr.count)) |i| { + const rc = util.at(arr.items, i); + if (!isInvocationRow(rc)) continue; + if (call_short == null or !streq(lastDotSegment(rc.callee_qn), call_short)) continue; + if (!qnClassMethodTailEq(rc.caller_qn, call_tail)) continue; + if (!siteAdmits(rc, call)) continue; + if (best_tail != null) return null; + best_tail = rc; + } + return best_tail; +} + +/// Resolve an LSP-emitted callee_qn to a graph-buffer node. +/// +/// Per-file LSPs sometimes emit `callee_qn` as the raw package-shaped +/// import path the source code uses rather than the project-qualified QN +/// the gbuf actually stores. The fallback rule is: +/// 1. try the LSP-emitted QN as-is; +/// 2. retry with `.` when needed; +/// 3. if both fail AND allow_tail_match is set (JVM callers only, see +/// cbm_pipeline_lsp_allow_tail_match), use the exact node-name index +/// to narrow candidates by short method name and accept exactly one +/// Function/Method whose qualified_name has the same Class.method +/// tail. +/// +/// Returns the matching node, or null if neither lookup hits. +pub export fn cbm_pipeline_lsp_target_node( + gbuf: ?*const Gbuf, + project_name: CStr, + callee_qn: CStr, + allow_tail_match: bool, +) ?*const Node { + if (gbuf == null or callee_qn == null) return null; + const direct: ?*const Node = c.cbm_gbuf_find_by_qn(gbuf, callee_qn); + if (direct != null) return direct; + + // Exact retries, in tie-rule order (CBM_MACRO_QN_SUFFIX): the definition + // first -- the QN as given, then with the project prefix -- and only when + // neither names a node the C-preprocessor macro of that name, in the same + // two spellings. The macro case is a prototype the C resolver knows whose + // only node in the graph is its rename macro (`extern T f(...);` plus + // `#define f f_v2`). One buffer, ".#macro", serves all + // three retries. + const suffix = c.CBM_MACRO_QN_SUFFIX; + const proj_len: usize = if (project_name != null and project_name[0] != 0) c.strlen(project_name) else 0; + const callee_len: usize = c.strlen(callee_qn); + const add_prefix = proj_len > 0 and + !(c.strncmp(callee_qn, project_name, proj_len) == 0 and callee_qn[proj_len] == '.'); + if (proj_len <= std.math.maxInt(usize) -% callee_len -% suffix.len -% 2) { + const buf = interop.mallocN(u8, proj_len + 1 + callee_len + suffix.len + 1); + if (buf != null) { + const own = buf + proj_len + 1; // callee_qn inside the buffer + if (proj_len > 0) @memcpy(buf[0..proj_len], project_name[0..proj_len]); + buf[proj_len] = '.'; + @memcpy(own[0..callee_len], callee_qn[0..callee_len]); + own[callee_len] = 0; + var hit: ?*const Node = if (add_prefix) c.cbm_gbuf_find_by_qn(gbuf, buf) else null; + if (hit == null) { + @memcpy(own[callee_len .. callee_len + suffix.len], suffix); + own[callee_len + suffix.len] = 0; + hit = c.cbm_gbuf_find_by_qn(gbuf, own); + if (hit == null and add_prefix) hit = c.cbm_gbuf_find_by_qn(gbuf, buf); + } + c.free(buf); + if (hit != null) return hit; + } + } + if (!allow_tail_match) return null; + + const short_name = lastDotSegment(callee_qn); + const callee_tail = qnClassMethodTail(callee_qn); + if (callee_tail == null) return null; + var hits: [*c][*c]const Node = null; + var hit_count: c_int = 0; + if (c.cbm_gbuf_find_by_name(gbuf, short_name, &hits, &hit_count) != 0 or hit_count == 0) return null; + _ = tail_lookups.fetchAdd(1, .monotonic); + _ = tail_candidates.fetchAdd(@bitCast(@as(i64, hit_count)), .monotonic); + + var match: ?*const Node = null; + for (0..ix(hit_count)) |i| { + const cand_opt: ?*const Node = hits[i]; + const cand = cand_opt orelse continue; + if (cand.label == null or cand.qualified_name == null) continue; + if (!streq(cand.label, "Function") and !streq(cand.label, "Method")) continue; + if (!qnClassMethodTailEq(cand.qualified_name, callee_tail)) continue; + if (match != null) return null; + match = cand; + } + return match; +} + +/// Rows that take part in the CALLS join. Count-based heuristics that compare +/// LSP output with textual calls must not count reference rows, nor the +/// confidence-0 lsp_unresolved diagnostics the join ignores. +pub fn invocationRowCount(arr: *const Rows) c_int { + var n: c_int = 0; + for (0..ix(arr.count)) |i| { + const rc = util.at(arr.items, i); + n += @intFromBool(rc.kind == c.CBM_RESOLVED_INVOCATION and rc.confidence >= confidence_floor); + } + return n; +} + +// ── Callable references (CALL_REFERENCE) ──────────────────────────── +// +// A function or method named as a value is a CALL_REFERENCE only when the +// language LSP proved one exact callable at that source occurrence. The +// semantic join is by (enclosing function, exact byte span of the reference +// token); the textual registry never earns CALL_REFERENCE, it only yields an +// ordinary USAGE. Reference rows never take part in the CALLS join. + +/// Rows that take part in the CALL_REFERENCE join. +pub fn referenceRowCount(arr: *const Rows) c_int { + var n: c_int = 0; + for (0..ix(arr.count)) |i| { + n += @intFromBool(util.at(arr.items, i).kind == c.CBM_RESOLVED_CALL_REFERENCE); + } + return n; +} + +/// Extraction marked this usage as a callable-reference candidate: explicit +/// reference syntax, or a bare value in a narrow position (direct argument, +/// assignment right-hand side) the language LSP knows how to prove. +fn usageIsReferenceCandidate(usage: *const c.CBMUsage) bool { + return (usage.kind == c.CBM_USAGE_CALL_REFERENCE or usage.may_be_call_reference) and + usage.site_end_byte > usage.site_start_byte; +} + +/// Graph labels a CALL_REFERENCE may point at (a Class covers constructor +/// references such as Java `Foo::new`). Anything else (a field, a variable, an +/// interface) proved by the LSP still lands as USAGE. +pub fn nodeIsCallableTarget(node_opt: ?*const Node) bool { + const node = node_opt orelse return false; + if (node.label == null) return false; + return streq(node.label, "Function") or streq(node.label, "Method") or + streq(node.label, "Constructor") or streq(node.label, "Class"); +} + +/// Per-file index of CALL_REFERENCE rows sorted by site, so each candidate +/// usage costs one binary search instead of a scan of every resolved row. +pub const RefIndex = struct { + rows: []*const Row = &.{}, + + /// On allocation failure the index is empty: the caller then finds no + /// semantic row and every candidate stays USAGE (fail closed). + pub fn build(arr: *const Rows) RefIndex { + const n: usize = @intCast(referenceRowCount(arr)); + if (n == 0) return .{}; + const rows = interop.mallocN(*const Row, n); + if (rows == null) return .{}; + var k: usize = 0; + for (0..ix(arr.count)) |i| { + const rc = util.at(arr.items, i); + if (rc.kind == c.CBM_RESOLVED_CALL_REFERENCE) { + rows[k] = rc; + k += 1; + } + } + c.qsort(@ptrCast(rows), n, @sizeOf(*const Row), rowCompare); + return .{ .rows = rows[0..n] }; + } + + pub fn deinit(self: *RefIndex) void { + if (self.rows.len > 0) c.free(@ptrCast(self.rows.ptr)); + self.* = .{}; + } + + fn rowCompare(a: ?*const anyopaque, b: ?*const anyopaque) callconv(.c) c_int { + const l: *const Row = @as(*const *const Row, @ptrCast(@alignCast(a))).*; + const r: *const Row = @as(*const *const Row, @ptrCast(@alignCast(b))).*; + if (l.site_start_byte != r.site_start_byte) return if (l.site_start_byte < r.site_start_byte) -1 else 1; + if (l.site_end_byte != r.site_end_byte) return if (l.site_end_byte < r.site_end_byte) -1 else 1; + // Stable order within one site: confidence ties keep the first row. + const lp = @intFromPtr(l); + const rp = @intFromPtr(r); + return if (lp < rp) -1 else if (lp > rp) 1 else 0; + } + + /// Scan the rows sharing the usage's byte span. `tail` null requires the + /// exact enclosing function; otherwise the caller's Class.method tail must + /// match. + fn scan(self: RefIndex, first: usize, usage: *const c.CBMUsage, tail: CStr) ?*const Row { + const ref_leaf = bareSegment(usage.ref_name); + var best: ?*const Row = null; + for (self.rows[first..]) |r| { + if (r.site_start_byte != usage.site_start_byte or r.site_end_byte != usage.site_end_byte) break; + if (r.caller_qn == null or r.callee_qn == null or r.confidence < confidence_floor) continue; + const caller_ok = if (tail != null) + qnClassMethodTailEq(r.caller_qn, tail) + else + streq(r.caller_qn, usage.enclosing_func_qn); + if (!caller_ok) continue; + // The row must name this reference: the callee leaf, or `reason` + // for an alias (`cb` proven to hold `handler`). + if (!streq(bareSegment(r.callee_qn), ref_leaf) and + !(r.reason != null and streq(bareSegment(r.reason), ref_leaf))) + { + continue; + } + if (best == null) { + best = r; + } else if (!streq(best.?.callee_qn, r.callee_qn)) { + return null; // two targets at one occurrence: ambiguous, fail closed + } else if (r.confidence > best.?.confidence) { + best = r; + } + } + return best; + } + + /// The exact semantic row for one candidate usage, or null. Rows must + /// share the usage's byte span and enclosing function and clear the + /// confidence floor. JVM callers (allow_tail_match) may fall back to a + /// Class.method tail match on the caller, the same rule the CALLS join + /// uses for package-shaped LSP QNs. + fn find(self: RefIndex, usage: *const c.CBMUsage, allow_tail_match: bool) ?*const Row { + if (self.rows.len == 0 or !usageIsReferenceCandidate(usage) or + usage.enclosing_func_qn == null or usage.ref_name == null) + { + return null; + } + var lo: usize = 0; + var hi: usize = self.rows.len; + while (lo < hi) { + const mid = lo + (hi - lo) / 2; + const r = self.rows[mid]; + if (r.site_start_byte < usage.site_start_byte or + (r.site_start_byte == usage.site_start_byte and r.site_end_byte < usage.site_end_byte)) + { + lo = mid + 1; + } else { + hi = mid; + } + } + const exact = self.scan(lo, usage, null); + if (exact != null or !allow_tail_match) return exact; + const tail = qnClassMethodTail(usage.enclosing_func_qn); + return if (tail != null) self.scan(lo, usage, tail) else null; + } +}; + +/// Outcome of the exact semantic join for one usage. +pub const RefJoinKind = enum { + /// no LSP row at this occurrence: registry fallback decides + none, + /// proven callable target: emit CALL_REFERENCE + callable, + /// proven non-callable target: at most a USAGE + value, + /// a row owns the occurrence but its target is not in the graph: emit + /// nothing rather than guess by name + skip, +}; + +pub const RefJoin = struct { + kind: RefJoinKind = .none, + target: ?*const Node = null, + row: ?*const Row = null, +}; + +pub fn joinLspReference( + gbuf: ?*const Gbuf, + project_name: CStr, + index: RefIndex, + usage: *const c.CBMUsage, + allow_tail_match: bool, +) RefJoin { + const row = index.find(usage, allow_tail_match) orelse return .{}; + const tgt = cbm_pipeline_lsp_target_node(gbuf, project_name, row.callee_qn, allow_tail_match) orelse + return .{ .kind = .skip, .row = row }; + return .{ + .kind = if (nodeIsCallableTarget(tgt)) .callable else .value, + .target = tgt, + .row = row, + }; +} + +/// Edge properties for a USAGE / CALL_REFERENCE edge. A CALL_REFERENCE always +/// comes from an LSP row (row non-null), whose strategy and confidence are the +/// edge's provenance, in the same keys CALLS edges use; USAGE stays +/// {"callee":...}. esc_ref is already JSON-escaped. +pub fn referenceProps(buf: []u8, esc_ref: CStr, row_opt: ?*const Row) void { + if (row_opt) |row| { + if (row.strategy != null and row.strategy[0] != 0) { + _ = c.snprintf(buf.ptr, buf.len, "{\"callee\":\"%s\",\"confidence\":%.2f,\"strategy\":\"%s\"}", esc_ref, @as(f64, row.confidence), row.strategy); + return; + } + } + _ = c.snprintf(buf.ptr, buf.len, "{\"callee\":\"%s\"}", esc_ref); +} + +/// Registry fallback for a usage the LSP did not prove. A candidate emitted +/// only for the join (reference_only) never falls back. Explicit JVM reference +/// syntax may not bind by a bare-name guess (unique_name / suffix_match), +/// except a Kotlin `::name` whose spelling the file imports. +pub fn referenceFallbackAllowed( + lang: Language, + usage: *const c.CBMUsage, + strategy: CStr, + import_names: [*c]const CStr, + import_count: c_int, +) bool { + if (usage.reference_only) return false; + if (usage.kind != c.CBM_USAGE_CALL_REFERENCE or (lang != c.CBM_LANG_JAVA and lang != c.CBM_LANG_KOTLIN)) { + return true; + } + if (strategy != null and strategy[0] != 0 and !streq(strategy, "unique_name") and + !streq(strategy, "suffix_match")) + { + return true; + } + if (lang == c.CBM_LANG_KOTLIN and usage.ref_name != null and import_names != null) { + for (0..ix(import_count)) |i| { + if (import_names[i] != null and streq(import_names[i], usage.ref_name)) return true; + } + } + return false; +} diff --git a/src/pipeline/lsp_resolve_shim.c b/src/pipeline/lsp_resolve_shim.c deleted file mode 100644 index b8538404..00000000 --- a/src/pipeline/lsp_resolve_shim.c +++ /dev/null @@ -1,21 +0,0 @@ -/* - * lsp_resolve_shim.c — cbm_pipeline_lsp_target_node as a linkable function. - * - * lsp_resolve.h defines its helpers as static inline, and the Zig modules - * read them through translate-c. This one counts its tail-match lookups with - * C11 atomics, which translate-c cannot express, so the Zig side sees only its - * declaration. This file gives that declaration a definition by compiling the - * header's own body, so lsp_resolve.h stays the single source of the logic - * for the C code that includes it (internal/cbm/lsp, the tests). - */ -#define cbm_pipeline_lsp_target_node cbm_pipeline_lsp_target_node_inline -#include "pipeline/lsp_resolve.h" -#undef cbm_pipeline_lsp_target_node - -const cbm_gbuf_node_t *cbm_pipeline_lsp_target_node(const cbm_gbuf_t *gbuf, const char *project, - const char *callee_qn, bool allow_tail_match); - -const cbm_gbuf_node_t *cbm_pipeline_lsp_target_node(const cbm_gbuf_t *gbuf, const char *project, - const char *callee_qn, bool allow_tail_match) { - return cbm_pipeline_lsp_target_node_inline(gbuf, project, callee_qn, allow_tail_match); -} diff --git a/src/pipeline/pass_parallel.zig b/src/pipeline/pass_parallel.zig index 87f7e931..02a60e85 100644 --- a/src/pipeline/pass_parallel.zig +++ b/src/pipeline/pass_parallel.zig @@ -13,6 +13,7 @@ const std = @import("std"); const c = @import("c"); const interop = @import("../foundation/interop.zig"); const util = @import("util.zig"); +const lsp_resolve = @import("lsp_resolve.zig"); const CStr = util.CStr; const MutStr = util.MutStr; @@ -97,10 +98,6 @@ const PP_RETAIN_PER_FILE_HARD_MAX_BYTES: usize = 32 * 1024 * 1024; // 32 MiB per /// not re-pay the full nap tax on every file pull when napping cannot reclaim /// memory (the resident floor, not in-flight transients, holds the budget). var g_bp_nap_cycles: std.atomic.Value(c_long) = .init(0); -/// Defined here, declared in lsp_resolve.h: the uncapped tail-match scan's -/// cost, logged at the end of parallel resolve. -pub export var g_lsp_tail_lookups: u64 = 0; -pub export var g_lsp_tail_candidates: u64 = 0; pub export fn cbm_pp_bp_nap_cycles() c_long { return g_bp_nap_cycles.load(.monotonic); @@ -2473,7 +2470,7 @@ fn resolveFileCalls( var key: [1024]u8 = undefined; // A row with a site span proves one call site: key it by that // site so it never answers a same-named call elsewhere in the - // caller (cbm_pipeline_lsp_site_admits). + // caller (lsp_resolve.zig siteAdmits). const kn = if (rc_e.site_end_byte > rc_e.site_start_byte) c.snprintf(&key, key.len, "%s|%s@%u", rc_e.caller_qn, short_name, rc_e.site_start_byte) else @@ -2502,7 +2499,7 @@ fn resolveFileCalls( const source_node = source_node_opt orelse continue; // LSP-resolved calls take precedence over registry textual matching. - // The shared helper and CBM_LSP_CONFIDENCE_FLOOR (lsp_resolve.h) + // The shared helper (lsp_resolve.zig) and CBM_LSP_CONFIDENCE_FLOOR // decide which LSP overrides are admitted. Unique-tail fallbacks // are JVM-only (see cbm_pipeline_lsp_allow_tail_match). const allow_tail = c.cbm_pipeline_lsp_allow_tail_match(lang); @@ -2510,7 +2507,7 @@ fn resolveFileCalls( var lsp: ?*const c.CBMResolvedCall = null; t0 = extractNowNs(); if (lsp_idx != null and call.enclosing_func_qn != null) { - const call_leaf: CStr = c.cbm_pipeline_call_callee_leaf(call.callee_name); + const call_leaf: CStr = lsp_resolve.bareSegment(call.callee_name); var key: [1024]u8 = undefined; var kn: c_int = if (call_leaf != null) c.snprintf(&key, key.len, "%s|%s@%u", call.enclosing_func_qn, call_leaf, call.site_start_byte) @@ -2725,23 +2722,22 @@ fn resolveFileUsages( imp_count: c_int, lang: Language, ) void { - var ref_index: c.cbm_pipeline_ref_index_t = undefined; - _ = c.cbm_pipeline_ref_index_build(&result.resolved_calls, &ref_index); + var ref_index = lsp_resolve.RefIndex.build(&result.resolved_calls); + defer ref_index.deinit(); const allow_tail = c.cbm_pipeline_lsp_allow_tail_match(lang); for (0..ix(result.usages.count)) |u| { const usage = util.at(result.usages.items, u); if (usage.ref_name == null) continue; const src = findSourceNode(rc.main_gbuf, rc.project_name, rel, usage.enclosing_func_qn) orelse continue; - var tgt: ?*const Node = null; - var row: ?*const c.CBMResolvedCall = null; - const join = c.cbm_pipeline_join_lsp_reference(rc.main_gbuf, rc.project_name, &ref_index, usage, allow_tail, @ptrCast(&tgt), @ptrCast(&row)); - if (join == c.CBM_REF_JOIN_SKIP or (join == c.CBM_REF_JOIN_VALUE and usage.reference_only)) { + const join = lsp_resolve.joinLspReference(rc.main_gbuf, rc.project_name, ref_index, usage, allow_tail); + if (join.kind == .skip or (join.kind == .value and usage.reference_only)) { continue; } - if (join == c.CBM_REF_JOIN_NONE) { + var tgt = join.target; + if (join.kind == .none) { const res = c.cbm_registry_resolve(rc.registry, usage.ref_name, module_qn, imp_keys, imp_vals, imp_count); if (res.qualified_name == null or res.qualified_name[0] == 0) continue; - if (!c.cbm_pipeline_reference_fallback_allowed(lang, usage, res.strategy, imp_keys, imp_count)) { + if (!lsp_resolve.referenceFallbackAllowed(lang, usage, res.strategy, imp_keys, imp_count)) { continue; } tgt = c.cbm_gbuf_find_by_qn(rc.main_gbuf, res.qualified_name); @@ -2759,12 +2755,11 @@ fn resolveFileUsages( var uprops: [c.CBM_SZ_512]u8 = undefined; var esc_ref: [c.CBM_SZ_256]u8 = undefined; // sliced source text: escape quotes/newlines _ = c.cbm_json_escape(&esc_ref, esc_ref.len, usage.ref_name); - const callable = join == c.CBM_REF_JOIN_CALLABLE; - _ = c.cbm_pipeline_reference_props(&uprops, uprops.len, &esc_ref, if (callable) row else null); + const callable = join.kind == .callable; + lsp_resolve.referenceProps(&uprops, &esc_ref, if (callable) join.row else null); _ = c.cbm_gbuf_insert_edge(ws.local_edge_buf, src.id, target.id, if (callable) "CALL_REFERENCE" else "USAGE", &uprops); ws.usages_resolved += 1; } - c.cbm_pipeline_ref_index_free(&ref_index); } /// Resolve throws/raises for one file. @@ -3151,7 +3146,7 @@ fn resolveWorker(worker_id: c_int, ctx_ptr: ?*anyopaque) callconv(.c) void { const jvm_cross_lsp = lang == c.CBM_LANG_JAVA or lang == c.CBM_LANG_KOTLIN; const rust_workspace_cross_lsp = lang == c.CBM_LANG_RUST and rc.rust_manifest != null and rc.rust_manifest.?.member_count > 0; - const invocation_rows = c.cbm_pipeline_invocation_row_count(&result.resolved_calls); + const invocation_rows = lsp_resolve.invocationRowCount(&result.resolved_calls); var ref_candidates: c_int = 0; for (0..ix(result.usages.count)) |u| { ref_candidates += @intFromBool(result.usages.items[u].may_be_call_reference); @@ -3160,7 +3155,7 @@ fn resolveWorker(worker_id: c_int, ctx_ptr: ?*anyopaque) callconv(.c) void { rc.all_defs != null and rc.def_count > 0 and c.cbm_pxc_has_cross_lsp(lang) and (result.calls.count > 0 or ref_candidates > 0) and (jvm_cross_lsp or rust_workspace_cross_lsp or invocation_rows < result.calls.count or - c.cbm_pipeline_reference_row_count(&result.resolved_calls) < ref_candidates or + lsp_resolve.referenceRowCount(&result.resolved_calls) < ref_candidates or (lang == c.CBM_LANG_ZIG and rc.def_modules != null and c.cbm_pxc_zig_args_in_module(rc.cross_registries, rc.def_modules[fidx]))) and !is_generated; @@ -3422,8 +3417,8 @@ pub export fn cbm_parallel_resolve( .rust_manifest = rust_manifest_ptr, .lsp_failures = lsp_failures, }; - @atomicStore(u64, &g_lsp_tail_lookups, 0, .monotonic); - @atomicStore(u64, &g_lsp_tail_candidates, 0, .monotonic); + lsp_resolve.tail_lookups.store(0, .monotonic); + lsp_resolve.tail_candidates.store(0, .monotonic); c.cbm_pxc_filter_stats_reset(); // F4 lazy shared Rust registry: mutex up before workers spawn. c.cbm_mutex_init(&rc.rust_shared_mu); @@ -3589,8 +3584,8 @@ pub export fn cbm_parallel_resolve( // Candidate-scan cost (upstream f6e3af43). per_lookup is a property of // the corpus, not the machine; if it grows with repo size the tail-match // scan is turning resolve superlinear. - const tail_lookups = @atomicLoad(u64, &g_lsp_tail_lookups, .monotonic); - const tail_cands = @atomicLoad(u64, &g_lsp_tail_candidates, .monotonic); + const tail_lookups = lsp_resolve.tail_lookups.load(.monotonic); + const tail_cands = lsp_resolve.tail_candidates.load(.monotonic); var tl_buf: [32]u8 = undefined; var tc_buf: [32]u8 = undefined; var tp_buf: [32]u8 = undefined; diff --git a/src/pipeline/pipeline_incremental.zig b/src/pipeline/pipeline_incremental.zig index 2448a027..898b3b75 100644 --- a/src/pipeline/pipeline_incremental.zig +++ b/src/pipeline/pipeline_incremental.zig @@ -12,6 +12,7 @@ const std = @import("std"); const c = @import("c"); const interop = @import("../foundation/interop.zig"); const util = @import("util.zig"); +const lsp_resolve = @import("lsp_resolve.zig"); const CStr = util.CStr; const MutStr = util.MutStr; @@ -820,7 +821,7 @@ fn incrRestoreInboundEdges(gbuf: ?*Gbuf, cap: *EdgeCapture) c_int { // re-prove its CALL_REFERENCE. Keep the snapshot edge only while the // same QN is still a callable; never guess a rename. if (s.type != null and streq(s.type, "CALL_REFERENCE") and - !c.cbm_pipeline_node_is_callable_target(tgt)) + !lsp_resolve.nodeIsCallableTarget(tgt)) { continue; } diff --git a/tests/test_vmem.c b/tests/test_vmem.c deleted file mode 100644 index b3e3dc9d..00000000 --- a/tests/test_vmem.c +++ /dev/null @@ -1,810 +0,0 @@ -/* - * test_vmem.c — Tests for vmem budget-tracked virtual memory allocator, - * arena-vmem integration, and slab+vmem parallel extraction. - */ -#include "test_framework.h" -#include "test_helpers.h" -#include "../src/foundation/vmem.h" -#include "../src/foundation/arena.h" -#include "../src/foundation/slab_alloc.h" -#include "pipeline/pipeline.h" -#include "pipeline/pipeline_internal.h" -#include "graph_buffer/graph_buffer.h" -#include "discover/discover.h" -#include "cbm.h" - -#include -#include - -/* ── vmem basic tests ─────────────────────────────────────────── */ - -TEST(vmem_budget_zero_before_init) { - /* Before init, budget should be 0 — arenas/read_file fall back to malloc */ - /* NOTE: vmem_init uses atomic CAS, only first call takes effect. - * These tests verify the pre-init state if vmem was never initialized, - * or the post-init state if it was. We test what we can. */ - size_t budget = cbm_vmem_budget(); - /* Budget is either 0 (never inited) or >0 (inited by earlier test/main) */ - (void)budget; - PASS(); -} - -TEST(vmem_alloc_and_free) { - /* Allocate 1MB, write to it, verify, free */ - size_t sz = 1024 * 1024; - void *p = cbm_vmem_alloc(sz); - if (!p) { - /* vmem may not be initialized — skip gracefully */ - PASS(); - } - /* vmem guarantees zeroed memory */ - unsigned char *bytes = (unsigned char *)p; - for (size_t i = 0; i < sz; i += 4096) { - ASSERT_EQ(bytes[i], 0); - } - /* Write pattern */ - memset(p, 0xAB, sz); - ASSERT_EQ(bytes[0], 0xAB); - ASSERT_EQ(bytes[sz - 1], 0xAB); - - cbm_vmem_free(p, sz); - PASS(); -} - -TEST(vmem_alloc_zero_returns_zeroed) { - size_t sz = 64 * 1024; - void *p = cbm_vmem_alloc(sz); - if (!p) { - PASS(); - } - unsigned char *bytes = (unsigned char *)p; - int nonzero = 0; - for (size_t i = 0; i < sz; i++) { - if (bytes[i] != 0) { - nonzero++; - } - } - ASSERT_EQ(nonzero, 0); - cbm_vmem_free(p, sz); - PASS(); -} - -TEST(vmem_budget_tracking) { - size_t before = cbm_vmem_allocated(); - size_t sz = 256 * 1024; - void *p = cbm_vmem_alloc(sz); - if (!p) { - PASS(); - } - size_t after = cbm_vmem_allocated(); - /* allocated should have increased (may be rounded to page) */ - ASSERT_GT(after, before); - - cbm_vmem_free(p, sz); - size_t freed = cbm_vmem_allocated(); - /* Should be back to (approximately) before */ - ASSERT_LTE(freed, before + 4096); /* within one page */ - PASS(); -} - -TEST(vmem_peak_tracks) { - size_t sz = 512 * 1024; - void *p1 = cbm_vmem_alloc(sz); - if (!p1) { - PASS(); - } - size_t peak1 = cbm_vmem_peak(); - ASSERT_GT(peak1, 0); - - void *p2 = cbm_vmem_alloc(sz); - if (!p2) { - cbm_vmem_free(p1, sz); - PASS(); - } - size_t peak2 = cbm_vmem_peak(); - ASSERT_GTE(peak2, peak1); - - cbm_vmem_free(p2, sz); - cbm_vmem_free(p1, sz); - - /* Peak should not decrease after free */ - size_t peak3 = cbm_vmem_peak(); - ASSERT_GTE(peak3, peak2); - PASS(); -} - -TEST(vmem_worker_budget) { - size_t budget = cbm_vmem_budget(); - if (budget == 0) { - /* Not initialized — worker budget should be 0 */ - ASSERT_EQ(cbm_vmem_worker_budget(4), 0); - PASS(); - } - /* Budget divides correctly */ - size_t wb4 = cbm_vmem_worker_budget(4); - size_t wb8 = cbm_vmem_worker_budget(8); - ASSERT_EQ(wb4, budget / 4); - ASSERT_EQ(wb8, budget / 8); - /* Edge case: 0 workers */ - ASSERT_EQ(cbm_vmem_worker_budget(0), 0); - PASS(); -} - -/* ── vmem edge-case and resource management tests ────────────── */ - -TEST(vmem_alloc_zero_returns_null) { - cbm_vmem_init(0.5); - /* alloc(0) must return NULL per the API contract */ - void *p = cbm_vmem_alloc(0); - ASSERT_NULL(p); - PASS(); -} - -TEST(vmem_free_null_zero_no_crash) { - cbm_vmem_init(0.5); - /* free(NULL, 0) must be a no-op */ - cbm_vmem_free(NULL, 0); - PASS(); -} - -TEST(vmem_free_null_nonzero_no_crash) { - cbm_vmem_init(0.5); - /* free(NULL, 100) must be a no-op — ptr==NULL short-circuits */ - cbm_vmem_free(NULL, 100); - PASS(); -} - -TEST(vmem_alloc_very_large) { - cbm_vmem_init(0.5); - /* Attempt 1 GB allocation — may succeed or fail depending on system. - * Either way it must not crash. */ - size_t sz = (size_t)1024 * 1024 * 1024; - void *p = cbm_vmem_alloc(sz); - if (p) { - /* If it succeeded, verify we can touch the first and last page */ - ((unsigned char *)p)[0] = 0xAA; - ((unsigned char *)p)[sz - 1] = 0xBB; - cbm_vmem_free(p, sz); - } - /* Success or graceful NULL — either is fine */ - PASS(); -} - -TEST(vmem_sequential_alloc_free_no_leak) { - cbm_vmem_init(0.5); - size_t before = cbm_vmem_allocated(); - - /* 20 alloc/free cycles — allocated must return to baseline */ - size_t sz = 64 * 1024; /* 64 KB */ - for (int i = 0; i < 20; i++) { - void *p = cbm_vmem_alloc(sz); - ASSERT_NOT_NULL(p); - memset(p, (unsigned char)(i & 0xFF), sz); - cbm_vmem_free(p, sz); - } - - size_t after = cbm_vmem_allocated(); - ASSERT_EQ(after, before); - PASS(); -} - -TEST(vmem_worker_budget_negative_workers) { - cbm_vmem_init(0.5); - size_t budget = cbm_vmem_budget(); - if (budget == 0) { - /* Not initialized in this process — skip */ - PASS(); - } - /* Negative workers clamps to 1 → worker_budget == full budget */ - size_t wb = cbm_vmem_worker_budget(-3); - ASSERT_EQ(wb, budget); - PASS(); -} - -TEST(vmem_over_budget_when_nothing_allocated) { - cbm_vmem_init(0.5); - /* With nothing (or near-nothing) allocated, should not be over budget */ - bool over = cbm_vmem_over_budget(); - ASSERT_FALSE(over); - PASS(); -} - -TEST(vmem_allocated_tracks_alloc_free_cycle) { - cbm_vmem_init(0.5); - size_t base = cbm_vmem_allocated(); - - size_t sz = 128 * 1024; /* 128 KB */ - void *p = cbm_vmem_alloc(sz); - ASSERT_NOT_NULL(p); - - size_t after_alloc = cbm_vmem_allocated(); - ASSERT_GT(after_alloc, base); - - cbm_vmem_free(p, sz); - - size_t after_free = cbm_vmem_allocated(); - /* Must be back at or near baseline */ - ASSERT_LTE(after_free, base + 4096); - PASS(); -} - -TEST(vmem_multiple_alloc_tracks_cumulative) { - cbm_vmem_init(0.5); - size_t base = cbm_vmem_allocated(); - - size_t sz = 64 * 1024; /* 64 KB */ - void *p1 = cbm_vmem_alloc(sz); - ASSERT_NOT_NULL(p1); - size_t after_one = cbm_vmem_allocated(); - - void *p2 = cbm_vmem_alloc(sz); - ASSERT_NOT_NULL(p2); - size_t after_two = cbm_vmem_allocated(); - - /* Two allocs should be roughly double one alloc above base */ - ASSERT_GT(after_two, after_one); - ASSERT_GT(after_one, base); - - cbm_vmem_free(p2, sz); - cbm_vmem_free(p1, sz); - - size_t after_free = cbm_vmem_allocated(); - ASSERT_LTE(after_free, base + 4096); - PASS(); -} - -TEST(vmem_peak_never_decreases) { - cbm_vmem_init(0.5); - - size_t sz = 256 * 1024; - void *p = cbm_vmem_alloc(sz); - ASSERT_NOT_NULL(p); - size_t peak_with_alloc = cbm_vmem_peak(); - - cbm_vmem_free(p, sz); - size_t peak_after_free = cbm_vmem_peak(); - - /* Peak must never decrease */ - ASSERT_GTE(peak_after_free, peak_with_alloc); - PASS(); -} - -TEST(vmem_worker_budget_one_worker) { - cbm_vmem_init(0.5); - size_t budget = cbm_vmem_budget(); - if (budget == 0) { - PASS(); - } - /* 1 worker → equals full budget */ - size_t wb = cbm_vmem_worker_budget(1); - ASSERT_EQ(wb, budget); - PASS(); -} - -TEST(vmem_worker_budget_many_workers) { - cbm_vmem_init(0.5); - size_t budget = cbm_vmem_budget(); - if (budget == 0) { - PASS(); - } - /* 1000 workers → must still be non-zero (budget is huge) */ - size_t wb = cbm_vmem_worker_budget(1000); - ASSERT_GT(wb, 0); - ASSERT_EQ(wb, budget / 1000); - PASS(); -} - -/* ── Arena-vmem integration tests ─────────────────────────────── */ - -TEST(arena_vmem_alloc_and_destroy) { - /* When vmem is initialized, arena should use vmem for blocks. - * When not initialized, falls back to malloc. Either way, this must work. */ - CBMArena a; - cbm_arena_init(&a); - ASSERT_EQ(a.nblocks, 1); - /* block_sizes[0] should track the initial block size */ - ASSERT_EQ(a.block_sizes[0], CBM_ARENA_DEFAULT_BLOCK_SIZE); - - /* Allocate some data */ - char *s = cbm_arena_strdup(&a, "hello vmem integration"); - ASSERT_NOT_NULL(s); - ASSERT_STR_EQ(s, "hello vmem integration"); - - cbm_arena_destroy(&a); - ASSERT_EQ(a.nblocks, 0); - PASS(); -} - -TEST(arena_vmem_grow_tracks_sizes) { - CBMArena a; - cbm_arena_init_sized(&a, 64); - ASSERT_EQ(a.block_sizes[0], 64); - - /* Force growth */ - cbm_arena_alloc(&a, 48); - cbm_arena_alloc(&a, 48); /* triggers grow */ - ASSERT_GTE(a.nblocks, 2); - /* Second block should be larger */ - ASSERT_GT(a.block_sizes[1], 0); - ASSERT_GTE(a.block_sizes[1], 96); /* at least min_size */ - - cbm_arena_destroy(&a); - PASS(); -} - -TEST(arena_vmem_large_alloc) { - /* Allocate > 64KB to test vmem for larger arena blocks */ - CBMArena a; - cbm_arena_init(&a); - - size_t big = 128 * 1024; - void *p = cbm_arena_alloc(&a, big); - ASSERT_NOT_NULL(p); - - /* Write pattern to verify memory is writable */ - memset(p, 0xCD, big); - unsigned char *bytes = (unsigned char *)p; - ASSERT_EQ(bytes[0], 0xCD); - ASSERT_EQ(bytes[big - 1], 0xCD); - - cbm_arena_destroy(&a); - PASS(); -} - -TEST(arena_vmem_reset_frees_blocks) { - CBMArena a; - cbm_arena_init_sized(&a, 128); - - /* Create multiple blocks */ - cbm_arena_alloc(&a, 100); - cbm_arena_alloc(&a, 100); - ASSERT_GTE(a.nblocks, 2); - - /* Reset should free extra blocks */ - cbm_arena_reset(&a); - ASSERT_EQ(a.nblocks, 1); - ASSERT_EQ(a.block_sizes[1], 0); /* freed block's size cleared */ - - /* Should still be usable */ - void *p = cbm_arena_alloc(&a, 16); - ASSERT_NOT_NULL(p); - - cbm_arena_destroy(&a); - PASS(); -} - -/* ── Tier 2 slab allocator tests ──────────────────────────── */ - -TEST(tier2_alloc_and_free_128) { - /* Allocate 100 bytes — rounds up to 128-byte class (with 16-byte header) */ - cbm_vmem_init(0.5); - cbm_slab_install(); - - void *p = cbm_slab_test_malloc(100); - ASSERT_NOT_NULL(p); - - /* Verify memory is writable */ - memset(p, 0xAA, 100); - ASSERT_EQ(((unsigned char *)p)[0], 0xAA); - ASSERT_EQ(((unsigned char *)p)[99], 0xAA); - - cbm_slab_test_free(p); - - /* Allocate again — should reuse from free list (same size class) */ - void *p2 = cbm_slab_test_malloc(100); - ASSERT_NOT_NULL(p2); - /* May or may not be same address, but must be valid */ - memset(p2, 0xBB, 100); - cbm_slab_test_free(p2); - - cbm_slab_destroy_thread(); - PASS(); -} - -TEST(tier2_alloc_all_classes) { - /* Test all 6 size classes: 128, 256, 512, 1024, 2048, 4096 */ - cbm_vmem_init(0.5); - cbm_slab_install(); - - size_t test_sizes[] = {65, 200, 300, 800, 1500, 3000}; - void *ptrs[6]; - - for (int i = 0; i < 6; i++) { - ptrs[i] = cbm_slab_test_malloc(test_sizes[i]); - ASSERT_NOT_NULL(ptrs[i]); - /* Write pattern to verify each allocation is independent */ - memset(ptrs[i], (unsigned char)(0x10 + i), test_sizes[i]); - } - - /* Verify patterns are intact (no overlap) */ - for (int i = 0; i < 6; i++) { - unsigned char *bytes = (unsigned char *)ptrs[i]; - ASSERT_EQ(bytes[0], (unsigned char)(0x10 + i)); - ASSERT_EQ(bytes[test_sizes[i] - 1], (unsigned char)(0x10 + i)); - } - - /* Free all */ - for (int i = 0; i < 6; i++) { - cbm_slab_test_free(ptrs[i]); - } - - cbm_slab_destroy_thread(); - PASS(); -} - -TEST(tier2_free_list_reuse) { - /* Verify free list provides O(1) reuse within same size class */ - cbm_vmem_init(0.5); - cbm_slab_install(); - - /* Allocate and free 10 blocks of class 256 */ - void *addrs[10]; - for (int i = 0; i < 10; i++) { - addrs[i] = cbm_slab_test_malloc(200); - ASSERT_NOT_NULL(addrs[i]); - } - for (int i = 0; i < 10; i++) { - cbm_slab_test_free(addrs[i]); - } - - /* Re-allocate 10 blocks — all should come from free list - * (LIFO order means addrs come back in reverse) */ - for (int i = 0; i < 10; i++) { - void *p = cbm_slab_test_malloc(200); - ASSERT_NOT_NULL(p); - memset(p, 0xCC, 200); - cbm_slab_test_free(p); - } - - cbm_slab_destroy_thread(); - PASS(); -} - -TEST(tier2_oversized_dedicated) { - /* Allocate >4096 bytes — gets dedicated page, freed immediately on free() */ - cbm_vmem_init(0.5); - cbm_slab_install(); - - size_t before_alloc = cbm_vmem_allocated(); - - void *big = cbm_slab_test_malloc(8192); - ASSERT_NOT_NULL(big); - memset(big, 0xDD, 8192); - - size_t after_alloc = cbm_vmem_allocated(); - /* vmem allocated should have grown for the dedicated page */ - ASSERT_GT(after_alloc, before_alloc); - - /* Free — dedicated page should be vmem_free'd immediately */ - cbm_slab_test_free(big); - - size_t after_free = cbm_vmem_allocated(); - /* vmem allocated should have decreased (page was freed) */ - ASSERT_LTE(after_free, after_alloc); - - cbm_slab_destroy_thread(); - PASS(); -} - -TEST(tier2_realloc_same_class) { - /* realloc within same size class should return same pointer */ - cbm_vmem_init(0.5); - cbm_slab_install(); - - void *p = cbm_slab_test_malloc(100); - ASSERT_NOT_NULL(p); - memset(p, 0xEE, 100); - - /* Grow to 110 — still fits in 128-byte class */ - void *p2 = cbm_slab_test_realloc(p, 110); - ASSERT_NOT_NULL(p2); - ASSERT_EQ(p, p2); /* same pointer, same class */ - /* Original data should be preserved */ - ASSERT_EQ(((unsigned char *)p2)[0], 0xEE); - ASSERT_EQ(((unsigned char *)p2)[99], 0xEE); - - /* Shrink to 70 — still fits in 128-byte class */ - void *p3 = cbm_slab_test_realloc(p2, 70); - ASSERT_NOT_NULL(p3); - ASSERT_EQ(p2, p3); /* same pointer */ - - cbm_slab_test_free(p3); - cbm_slab_destroy_thread(); - PASS(); -} - -TEST(tier2_realloc_grows_class) { - /* realloc to larger class should copy data correctly */ - cbm_vmem_init(0.5); - cbm_slab_install(); - - void *p = cbm_slab_test_malloc(100); - ASSERT_NOT_NULL(p); - /* Write known pattern */ - for (int i = 0; i < 100; i++) { - ((unsigned char *)p)[i] = (unsigned char)(i & 0xFF); - } - - /* Grow to 300 — moves from class 128 to class 512 */ - void *p2 = cbm_slab_test_realloc(p, 300); - ASSERT_NOT_NULL(p2); - - /* Verify data was copied */ - unsigned char *bytes = (unsigned char *)p2; - for (int i = 0; i < 100; i++) { - ASSERT_EQ(bytes[i], (unsigned char)(i & 0xFF)); - } - - /* Write to the extended area */ - memset(bytes + 100, 0xFF, 200); - ASSERT_EQ(bytes[299], 0xFF); - - cbm_slab_test_free(p2); - cbm_slab_destroy_thread(); - PASS(); -} - -TEST(tier2_realloc_slab_to_tier2) { - /* realloc from Tier 1 (≤64B) to Tier 2 (>64B) */ - cbm_vmem_init(0.5); - cbm_slab_install(); - - void *p = cbm_slab_test_malloc(32); /* Tier 1 slab */ - ASSERT_NOT_NULL(p); - memset(p, 0x42, 32); - - /* Promote to Tier 2 */ - void *p2 = cbm_slab_test_realloc(p, 200); - ASSERT_NOT_NULL(p2); - /* First 32 bytes should be preserved */ - ASSERT_EQ(((unsigned char *)p2)[0], 0x42); - ASSERT_EQ(((unsigned char *)p2)[31], 0x42); - - cbm_slab_test_free(p2); - cbm_slab_destroy_thread(); - PASS(); -} - -TEST(tier2_calloc_zeroed) { - /* calloc via tier2 must return zeroed memory */ - cbm_vmem_init(0.5); - cbm_slab_install(); - - void *p = cbm_slab_test_calloc(1, 200); - ASSERT_NOT_NULL(p); - - /* Verify all bytes are zero */ - unsigned char *bytes = (unsigned char *)p; - int nonzero = 0; - for (int i = 0; i < 200; i++) { - if (bytes[i] != 0) { - nonzero++; - } - } - ASSERT_EQ(nonzero, 0); - - /* Free and re-calloc — recycled memory must still be zeroed */ - cbm_slab_test_free(p); - void *p2 = cbm_slab_test_calloc(1, 200); - ASSERT_NOT_NULL(p2); - bytes = (unsigned char *)p2; - nonzero = 0; - for (int i = 0; i < 200; i++) { - if (bytes[i] != 0) { - nonzero++; - } - } - ASSERT_EQ(nonzero, 0); - - cbm_slab_test_free(p2); - cbm_slab_destroy_thread(); - PASS(); -} - -TEST(tier2_mixed_alloc_free_stress) { - /* Stress test: interleaved allocs and frees across Tier 1 and Tier 2 */ - cbm_vmem_init(0.5); - cbm_slab_install(); - - void *ptrs[100]; - size_t sizes[100]; - - /* Allocate 100 blocks of varying sizes */ - for (int i = 0; i < 100; i++) { - sizes[i] = (size_t)(16 + (i * 47) % 4000); /* 16..4000 */ - ptrs[i] = cbm_slab_test_malloc(sizes[i]); - ASSERT_NOT_NULL(ptrs[i]); - memset(ptrs[i], (unsigned char)(i & 0xFF), sizes[i]); - } - - /* Free odd-indexed blocks */ - for (int i = 1; i < 100; i += 2) { - cbm_slab_test_free(ptrs[i]); - ptrs[i] = NULL; - } - - /* Re-allocate freed slots with different sizes */ - for (int i = 1; i < 100; i += 2) { - sizes[i] = (size_t)(32 + (i * 31) % 2000); - ptrs[i] = cbm_slab_test_malloc(sizes[i]); - ASSERT_NOT_NULL(ptrs[i]); - memset(ptrs[i], (unsigned char)((i + 1) & 0xFF), sizes[i]); - } - - /* Verify even-indexed blocks still have original data */ - for (int i = 0; i < 100; i += 2) { - ASSERT_EQ(((unsigned char *)ptrs[i])[0], (unsigned char)(i & 0xFF)); - } - - /* Free all */ - for (int i = 0; i < 100; i++) { - cbm_slab_test_free(ptrs[i]); - } - - cbm_slab_destroy_thread(); - PASS(); -} - -/* ── Slab + vmem parallel extraction test ──────────────────── */ - -static char g_vmem_tmpdir[256]; - -static int setup_vmem_test_repo(void) { - snprintf(g_vmem_tmpdir, sizeof(g_vmem_tmpdir), "/tmp/cbm_vmem_XXXXXX"); - if (!mkdtemp(g_vmem_tmpdir)) { - return -1; - } - - char path[512]; - - /* Create multiple Go files to force multi-file parallel extraction. - * We need enough files to exercise slab reset between files on a worker. */ - for (int i = 0; i < 6; i++) { - snprintf(path, sizeof(path), "%s/file%d.go", g_vmem_tmpdir, i); - FILE *f = fopen(path, "w"); - if (!f) { - return -1; - } - fprintf(f, - "package main\n\nfunc F%d() {\n\tprintln(\"hello\")\n}\n\n" - "func G%d() int {\n\treturn F%d() + %d\n}\n", - i, i, i, i); - fclose(f); - } - - /* Add a C file to exercise the preprocessor second-pass path */ - snprintf(path, sizeof(path), "%s/util.c", g_vmem_tmpdir); - FILE *f = fopen(path, "w"); - if (!f) { - return -1; - } - fprintf(f, "#include \nvoid util_func(void) { printf(\"hi\"); }\n" - "int util_add(int a, int b) { return a + b; }\n"); - fclose(f); - - return 0; -} - -static void teardown_vmem_test_repo(void) { - if (g_vmem_tmpdir[0]) { - th_rmtree(g_vmem_tmpdir); - g_vmem_tmpdir[0] = '\0'; - } -} - -TEST(vmem_parallel_extract_with_slab) { - /* This test reproduces a SIGSEGV that occurred when: - * 1. vmem is active (arena + source via mmap) - * 2. slab allocator is installed (tree-sitter uses slab) - * 3. slab_reset_thread() was called between files, corrupting - * the parser's live slab-allocated internal state (subtree pool, - * stack entries, cached tokens). - * - * The fix: don't call slab_reset_thread() between files. Normal - * slab_free() from ts_tree_delete() returns chunks for reuse. - * slab_destroy_thread() reclaims everything on worker exit. */ - cbm_vmem_init(0.5); - - if (setup_vmem_test_repo() != 0) { - FAIL("tmpdir setup failed"); - } - - cbm_discover_opts_t opts = {.mode = CBM_MODE_FULL}; - cbm_file_info_t *files = NULL; - int file_count = 0; - if (cbm_discover(g_vmem_tmpdir, &opts, &files, &file_count) != 0) { - teardown_vmem_test_repo(); - FAIL("discover failed"); - } - - ASSERT_GTE(file_count, 5); - - cbm_gbuf_t *gbuf = cbm_gbuf_new("vmem-test", g_vmem_tmpdir); - cbm_registry_t *reg = cbm_registry_new(); - atomic_int cancelled; - atomic_init(&cancelled, 0); - - cbm_pipeline_ctx_t ctx = { - .project_name = "vmem-test", - .repo_path = g_vmem_tmpdir, - .gbuf = gbuf, - .registry = reg, - .cancelled = &cancelled, - }; - - _Atomic int64_t shared_ids; - int64_t gbuf_next = cbm_gbuf_next_id(gbuf); - atomic_init(&shared_ids, gbuf_next); - - CBMFileResult **result_cache = calloc(file_count, sizeof(CBMFileResult *)); - ASSERT_NOT_NULL(result_cache); - - /* Run parallel extraction with 2 workers — enough to trigger - * multi-file slab reuse on at least one worker. */ - int rc = cbm_parallel_extract(&ctx, files, file_count, result_cache, &shared_ids, 2); - ASSERT_EQ(rc, 0); - - /* Verify extraction produced results */ - int cached_count = 0; - for (int i = 0; i < file_count; i++) { - if (result_cache[i]) { - cached_count++; - } - } - ASSERT_GTE(cached_count, 5); - - /* Verify nodes were created */ - ASSERT_GT(cbm_gbuf_node_count(gbuf), 0); - - /* Clean up */ - for (int i = 0; i < file_count; i++) { - if (result_cache[i]) { - cbm_free_result(result_cache[i]); - } - } - free(result_cache); - cbm_registry_free(reg); - cbm_gbuf_free(gbuf); - cbm_discover_free(files, file_count); - teardown_vmem_test_repo(); - PASS(); -} - -SUITE(vmem) { - RUN_TEST(vmem_budget_zero_before_init); - RUN_TEST(vmem_alloc_and_free); - RUN_TEST(vmem_alloc_zero_returns_zeroed); - RUN_TEST(vmem_budget_tracking); - RUN_TEST(vmem_peak_tracks); - RUN_TEST(vmem_worker_budget); - /* Edge cases and resource management */ - RUN_TEST(vmem_alloc_zero_returns_null); - RUN_TEST(vmem_free_null_zero_no_crash); - RUN_TEST(vmem_free_null_nonzero_no_crash); - RUN_TEST(vmem_alloc_very_large); - RUN_TEST(vmem_sequential_alloc_free_no_leak); - RUN_TEST(vmem_worker_budget_negative_workers); - RUN_TEST(vmem_over_budget_when_nothing_allocated); - RUN_TEST(vmem_allocated_tracks_alloc_free_cycle); - RUN_TEST(vmem_multiple_alloc_tracks_cumulative); - RUN_TEST(vmem_peak_never_decreases); - RUN_TEST(vmem_worker_budget_one_worker); - RUN_TEST(vmem_worker_budget_many_workers); - /* Arena-vmem integration */ - RUN_TEST(arena_vmem_alloc_and_destroy); - RUN_TEST(arena_vmem_grow_tracks_sizes); - RUN_TEST(arena_vmem_large_alloc); - RUN_TEST(arena_vmem_reset_frees_blocks); - /* Tier 2 slab allocator tests */ - RUN_TEST(tier2_alloc_and_free_128); - RUN_TEST(tier2_alloc_all_classes); - RUN_TEST(tier2_free_list_reuse); - RUN_TEST(tier2_oversized_dedicated); - RUN_TEST(tier2_realloc_same_class); - RUN_TEST(tier2_realloc_grows_class); - RUN_TEST(tier2_realloc_slab_to_tier2); - RUN_TEST(tier2_calloc_zeroed); - RUN_TEST(tier2_mixed_alloc_free_stress); - /* Integration */ - RUN_TEST(vmem_parallel_extract_with_slab); -}