diff --git a/Makefile.cbm b/Makefile.cbm index a58c32d9..ef268f89 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -161,41 +161,15 @@ MCP_SRCS = # Graph buffer module is Zig: see CBM_ZIG_SRC. -# Pipeline module (new) -PIPELINE_SRCS = \ - src/pipeline/fqn.c \ - src/pipeline/path_alias.c \ - src/pipeline/laravel_routing.c \ - src/pipeline/registry.c \ - src/pipeline/pipeline.c \ - src/pipeline/pipeline_incremental.c \ - src/pipeline/worker_pool.c \ - src/pipeline/pass_parallel.c \ - src/pipeline/result_spill.c \ - src/pipeline/service_bindings.c \ - src/pipeline/pass_lsp_cross.c \ - src/pipeline/pass_semantic.c \ - src/pipeline/pass_tests.c \ - src/pipeline/pass_githistory.c \ - src/pipeline/pass_gitdiff.c \ - src/pipeline/pass_configures.c \ - src/pipeline/pass_configlink.c \ - src/pipeline/pass_route_nodes.c \ - src/pipeline/pass_enrichment.c \ - src/pipeline/pass_envscan.c \ - src/pipeline/pass_compile_commands.c \ - src/pipeline/pass_infrascan.c \ - src/pipeline/pass_k8s.c \ - src/pipeline/pass_similarity.c \ - src/pipeline/pass_semantic_edges.c \ - src/pipeline/pass_complexity.c \ - src/pipeline/pass_importance.c \ - src/pipeline/pass_pkgmap.c +# Pipeline module: implemented in Zig (src/pipeline/*.zig, part of libcbm_zig). +# lsp_resolve_shim.c compiles one inline function of lsp_resolve.h that +# translate-c cannot hand to the Zig side. +PIPELINE_SRCS = src/pipeline/lsp_resolve_shim.c # SimHash / MinHash, watcher and git context modules are Zig: see CBM_ZIG_SRC. -# Semantic embedding module -SEMANTIC_SRCS = src/semantic/semantic.c src/semantic/ast_profile.c src/semantic/rotsq.c +# Semantic embedding module: implemented in Zig (src/semantic/*.zig, part of libcbm_zig) +SEMANTIC_SRCS = # nomic-embed-code pretrained vectors (assembler blob) UNIXCODER_BLOB_SRC = vendored/nomic/code_vectors_blob.S @@ -479,6 +453,28 @@ CBM_ZIG_SRCS = $(CBM_ZIG_SRC) src/cbm_zig_c.h src/cli_zig/cbm_cli_zig.zig src/cl src/discover/discover.h src/discover/userconfig.h \ src/git/git_context.zig src/git/git_context.h \ src/graph_buffer/graph_buffer.zig src/graph_buffer/graph_buffer.h \ + src/semantic/ast_profile.zig src/semantic/fp.zig src/semantic/rotsq.zig \ + src/semantic/semantic.zig \ + src/semantic/ast_profile.h src/semantic/rotsq.h src/semantic/semantic.h \ + vendored/nomic/code_vectors.h vendored/nomic/code_tokens.h \ + src/pipeline/fqn.zig src/pipeline/laravel_routing.zig \ + src/pipeline/pass_compile_commands.zig src/pipeline/pass_complexity.zig \ + src/pipeline/pass_configlink.zig src/pipeline/pass_configures.zig \ + src/pipeline/pass_enrichment.zig src/pipeline/pass_envscan.zig \ + src/pipeline/pass_gitdiff.zig src/pipeline/pass_githistory.zig \ + src/pipeline/pass_importance.zig src/pipeline/pass_infrascan.zig \ + src/pipeline/pass_k8s.zig src/pipeline/pass_lsp_cross.zig \ + src/pipeline/pass_parallel.zig src/pipeline/pass_pkgmap.zig \ + src/pipeline/pass_route_nodes.zig src/pipeline/pass_semantic.zig \ + src/pipeline/pass_semantic_edges.zig src/pipeline/pass_similarity.zig \ + src/pipeline/pass_tests.zig src/pipeline/path_alias.zig src/pipeline/pipeline.zig \ + src/pipeline/pipeline_incremental.zig src/pipeline/registry.zig \ + src/pipeline/result_spill.zig src/pipeline/service_bindings.zig \ + src/pipeline/util.zig src/pipeline/worker_pool.zig src/pipeline/laravel_routing.h \ + src/pipeline/lsp_resolve.h src/pipeline/pass_lsp_cross.h \ + src/pipeline/path_alias.h src/pipeline/pipeline_internal.h \ + src/pipeline/result_spill.h src/pipeline/worker_pool.h \ + src/graph_buffer/graph_buffer.h \ src/simhash/minhash.zig src/simhash/minhash.h \ src/store/store.zig src/store/store.h \ src/watcher/watcher.zig src/watcher/watcher.h \ diff --git a/build.zig b/build.zig index c1023226..1cf882a1 100644 --- a/build.zig +++ b/build.zig @@ -536,37 +536,7 @@ fn grammarSources(b: *std.Build) []const []const u8 { const native_sources = [_][]const u8{ "src/foundation/hash_table.c", "src/foundation/c_shim.c", - "src/pipeline/fqn.c", - "src/pipeline/path_alias.c", - "src/pipeline/laravel_routing.c", - "src/pipeline/registry.c", - "src/pipeline/pipeline.c", - "src/pipeline/pipeline_incremental.c", - "src/pipeline/worker_pool.c", - "src/pipeline/pass_parallel.c", - "src/pipeline/result_spill.c", - "src/pipeline/service_bindings.c", - "src/pipeline/pass_lsp_cross.c", - "src/pipeline/pass_semantic.c", - "src/pipeline/pass_tests.c", - "src/pipeline/pass_githistory.c", - "src/pipeline/pass_gitdiff.c", - "src/pipeline/pass_configures.c", - "src/pipeline/pass_configlink.c", - "src/pipeline/pass_route_nodes.c", - "src/pipeline/pass_enrichment.c", - "src/pipeline/pass_envscan.c", - "src/pipeline/pass_compile_commands.c", - "src/pipeline/pass_infrascan.c", - "src/pipeline/pass_k8s.c", - "src/pipeline/pass_similarity.c", - "src/pipeline/pass_semantic_edges.c", - "src/pipeline/pass_complexity.c", - "src/pipeline/pass_importance.c", - "src/pipeline/pass_pkgmap.c", - "src/semantic/semantic.c", - "src/semantic/ast_profile.c", - "src/semantic/rotsq.c", + "src/pipeline/lsp_resolve_shim.c", "vendored/yyjson/yyjson.c", "internal/cbm/cbm.c", "internal/cbm/extract_defs.c", diff --git a/scripts/extract_nomic_vectors.py b/scripts/extract_nomic_vectors.py index ef38c2ca..60c5bd5e 100755 --- a/scripts/extract_nomic_vectors.py +++ b/scripts/extract_nomic_vectors.py @@ -520,7 +520,7 @@ def main(): print() print("next steps:") print(f" 1. update Makefile.cbm: change UNIXCODER_BLOB_SRC path to vendored/nomic/") - print(f" 2. update #include in semantic.c: \"vendored/nomic/code_vectors.h\"") + print(f" 2. update #include in src/cbm_zig_c.h: \"nomic/code_vectors.h\"") print(f" 3. arch -arm64 make -j12 -f Makefile.cbm clean-c && arch -arm64 make -j12 -f Makefile.cbm") diff --git a/scripts/security-allowlist.txt b/scripts/security-allowlist.txt index 1b65db06..cfb66e0d 100644 --- a/scripts/security-allowlist.txt +++ b/scripts/security-allowlist.txt @@ -30,7 +30,7 @@ src/mcp/server.zig:cbm_popen:update check to api.github.com (hardcoded URL) src/mcp/server.zig:popen:via cbm_popen wrapper calls # ── Pipeline: git history parsing (git log) ──────────────────────────────── -src/pipeline/pass_githistory.c:cbm_popen_git:git log for file history (path validated) +src/pipeline/pass_githistory.zig:cbm_popen_git:git log for file history (path validated) # ── Allowed URLs ─────────────────────────────────────────────────────────── # Format: URL:justification diff --git a/scripts/security-audit.sh b/scripts/security-audit.sh index 4814664b..de8605d5 100755 --- a/scripts/security-audit.sh +++ b/scripts/security-audit.sh @@ -171,7 +171,7 @@ while IFS= read -r match; do file=$(echo "$match" | cut -d: -f1) relfile="${file#"$ROOT/"}" case "$relfile" in - src/cli/cli.zig|src/pipeline/*.c|src/foundation/diagnostics.zig|src/mcp/search_tools.zig|src/mcp/index_tools.zig) + src/cli/cli.zig|src/pipeline/*.zig|src/foundation/diagnostics.zig|src/mcp/search_tools.zig|src/mcp/index_tools.zig) ;; # Known safe (diagnostics.zig: atomic .tmp+rename metrics dump to configured path; # search_tools.zig: search pattern/file-list temp files; index_tools.zig: # index supervisor response/quarantine files) diff --git a/src/cbm_zig.zig b/src/cbm_zig.zig index 36eeb008..e62f9a26 100644 --- a/src/cbm_zig.zig +++ b/src/cbm_zig.zig @@ -61,6 +61,37 @@ comptime { checkExports(@import("mcp/project_tools.zig")); checkExports(@import("mcp/search_tools.zig")); checkExports(@import("mcp/server.zig")); + checkExports(@import("pipeline/fqn.zig")); + checkExports(@import("pipeline/laravel_routing.zig")); + checkExports(@import("pipeline/pass_compile_commands.zig")); + checkExports(@import("pipeline/pass_complexity.zig")); + checkExports(@import("pipeline/pass_configlink.zig")); + checkExports(@import("pipeline/pass_configures.zig")); + checkExports(@import("pipeline/pass_enrichment.zig")); + checkExports(@import("pipeline/pass_envscan.zig")); + checkExports(@import("pipeline/pass_gitdiff.zig")); + checkExports(@import("pipeline/pass_githistory.zig")); + checkExports(@import("pipeline/pass_importance.zig")); + checkExports(@import("pipeline/pass_infrascan.zig")); + checkExports(@import("pipeline/pass_k8s.zig")); + checkExports(@import("pipeline/pass_lsp_cross.zig")); + checkExports(@import("pipeline/pass_parallel.zig")); + checkExports(@import("pipeline/pass_pkgmap.zig")); + checkExports(@import("pipeline/pass_route_nodes.zig")); + checkExports(@import("pipeline/pass_semantic.zig")); + checkExports(@import("pipeline/pass_semantic_edges.zig")); + checkExports(@import("pipeline/pass_similarity.zig")); + checkExports(@import("pipeline/pass_tests.zig")); + checkExports(@import("pipeline/path_alias.zig")); + checkExports(@import("pipeline/pipeline.zig")); + checkExports(@import("pipeline/pipeline_incremental.zig")); + checkExports(@import("pipeline/registry.zig")); + checkExports(@import("pipeline/result_spill.zig")); + checkExports(@import("pipeline/service_bindings.zig")); + checkExports(@import("pipeline/worker_pool.zig")); + checkExports(@import("semantic/ast_profile.zig")); + checkExports(@import("semantic/rotsq.zig")); + checkExports(@import("semantic/semantic.zig")); checkExports(@import("simhash/minhash.zig")); checkExports(@import("store/store.zig")); checkExports(@import("watcher/watcher.zig")); diff --git a/src/cbm_zig_c.h b/src/cbm_zig_c.h index 473c2025..43b50c20 100644 --- a/src/cbm_zig_c.h +++ b/src/cbm_zig_c.h @@ -63,6 +63,35 @@ #ifdef __linux__ #include #endif + +/* C11 atomics, as translate-c can read them. translate-c has no atomic + * types, so a declaration that names one (the pipeline context's cancel flag, + * the shared id counter) would be dropped together with everything that uses + * it. Here an atomic type is its plain type and the atomic operations are the + * compiler builtins, which keeps those declarations and their layout. The Zig + * side reads and writes such fields with @atomicLoad / @atomicStore / + * @atomicRmw. Only this translation unit sees the view: the C sources keep + * the real . */ +#define __CLANG_STDATOMIC_H /* keeps clang's out */ +#define _Atomic +typedef _Bool atomic_bool; +typedef int atomic_int; +typedef unsigned int atomic_uint; +typedef long atomic_long; +typedef unsigned long atomic_ulong; +typedef long long atomic_llong; +typedef unsigned long long atomic_ullong; +#define memory_order_relaxed __ATOMIC_RELAXED +#define memory_order_consume __ATOMIC_CONSUME +#define memory_order_acquire __ATOMIC_ACQUIRE +#define memory_order_release __ATOMIC_RELEASE +#define memory_order_acq_rel __ATOMIC_ACQ_REL +#define memory_order_seq_cst __ATOMIC_SEQ_CST +#define atomic_load(p) __atomic_load_n((p), __ATOMIC_SEQ_CST) +#define atomic_load_explicit(p, o) __atomic_load_n((p), (o)) +#define atomic_fetch_add(p, v) __atomic_fetch_add((p), (v), __ATOMIC_SEQ_CST) +#define atomic_fetch_add_explicit(p, v, o) __atomic_fetch_add((p), (v), (o)) + #include #include "foundation/arena.h" @@ -91,12 +120,26 @@ #include "foundation/system_info_internal.h" #include "foundation/compat_fs_internal.h" #include "foundation/yaml.h" +#include "arena.h" +#include "cbm.h" +#include "helpers.h" +#include "lang_specs.h" +#include "lsp/c_lsp.h" +#include "lsp/go_lsp.h" +#include "lsp/java_lsp.h" +#include "lsp/kotlin_lsp.h" +#include "lsp/php_lsp.h" +#include "lsp/py_lsp.h" +#include "lsp/rust_cargo.h" +#include "lsp/rust_lsp.h" +#include "lsp/ts_lsp.h" +#include "lsp/zig_lsp.h" #include "mcp/mcp.h" -#include "pipeline/pipeline.h" -#include "pipeline/worker_pool.h" -#include "semantic/semantic.h" +#include "nomic/code_vectors.h" +#include "service_patterns.h" #include "store/store.h" #include "tree_sitter/api.h" +#include "xxhash/xxhash.h" #include "zova/cbm_zova.h" #include "zova/cbm_zova_bridge.h" #include "zova/cbm_zova_coordination.h" @@ -139,6 +182,16 @@ static inline FILE *cbm_zig_stderr(void) { #include "git/git_context.h" #include "graph_buffer/graph_buffer.h" #include "mcp/index_supervisor.h" +#include "pipeline/laravel_routing.h" +#include "pipeline/lsp_resolve.h" +#include "pipeline/pass_lsp_cross.h" +#include "pipeline/path_alias.h" +#include "pipeline/pipeline.h" +#include "pipeline/pipeline_internal.h" +#include "pipeline/result_spill.h" +#include "pipeline/worker_pool.h" +#include "semantic/ast_profile.h" +#include "semantic/semantic.h" #include "simhash/minhash.h" #include "watcher/watcher.h" diff --git a/src/pipeline/fqn.c b/src/pipeline/fqn.c deleted file mode 100644 index d55153f8..00000000 --- a/src/pipeline/fqn.c +++ /dev/null @@ -1,648 +0,0 @@ -/* - * fqn.c — Fully Qualified Name computation for graph nodes. - * - * Implements the FQN scheme: project.dir.parts.name - * Handles Python __init__.py, JS/TS index.{js,ts}, path separators. - */ -#include "pipeline/pipeline.h" -#include "foundation/constants.h" -#include "foundation/platform.h" - -#include -#include // NULL -#include // uint32_t -#include -#include -#include // strdup -#ifdef _WIN32 -#include -#include -#endif - -/* Maximum path segments in a FQN (CBM_SZ_256 slots total, -2 for project + name) */ -#define FQN_MAX_PATH_SEGS 254 -#define FQN_MAX_DIR_SEGS 255 - -/* Max bytes for a derived project name. Names were once filename components, - * so they stay under the 255-byte component limit. #571 hex-encodes each - * non-ASCII byte to 2 chars, so a deep CJK path can triple past 255 (#624). */ -#define FQN_MAX_NAME_LEN 200 - -/* ── Internal helpers ─────────────────────────────────────────────── */ - -/* Build a dot-joined string from segments. Returns heap-allocated string. */ -static char *join_segments(const char **segments, int count) { - if (count == 0) { - return strdup(""); - } - size_t total = 0; - for (int i = 0; i < count; i++) { - total += strlen(segments[i]); - if (i > 0) { - total++; /* dot separator */ - } - } - char *result = malloc(total + SKIP_ONE); - if (!result) { - return NULL; - } - char *p = result; - for (int i = 0; i < count; i++) { - if (i > 0) { - *p++ = '.'; - } - size_t len = strlen(segments[i]); - memcpy(p, segments[i], len); - p += len; - } - *p = '\0'; - return result; -} - -/* Strip file extension from the last path component. */ -static void strip_file_extension(char *path) { - char *last_slash = strrchr(path, '/'); - char *start = last_slash ? last_slash + SKIP_ONE : path; - char *ext = strrchr(start, '.'); - if (ext) { - *ext = '\0'; - } -} - -/* Tokenize path by '/' into segments array. Returns number of segments added. */ -static int tokenize_path(char *path, const char **segments, int max_segs) { - int count = 0; - if (path[0] == '\0') { - return 0; - } - char *tok = path; - while (tok && *tok && count < max_segs) { - char *slash = strchr(tok, '/'); - if (slash) { - *slash = '\0'; - } - if (tok[0] != '\0') { - segments[count++] = tok; - } - tok = slash ? slash + SKIP_ONE : NULL; - } - return count; -} - -/* Strip __init__ (Python) / index (JS/TS) from the last segment when a - * symbol name is provided. Keeps it when no name is given to avoid QN - * collision with Folder nodes for the same directory. */ -static void strip_init_or_index(const char **segments, int *seg_count, const char *name) { - if (*seg_count <= SKIP_ONE) { - return; - } - const char *last = segments[*seg_count - SKIP_ONE]; - if (strcmp(last, "__init__") != 0 && strcmp(last, "index") != 0) { - return; - } - if (name && name[0] != '\0') { - (*seg_count)--; - } -} - -/* ── Public API ──────────────────────────────────────────────────── */ - -char *cbm_pipeline_fqn_compute(const char *project, const char *rel_path, const char *name) { - if (!project) { - return strdup(""); - } - - char *path = strdup(rel_path ? rel_path : ""); - cbm_normalize_path_sep(path); - /* File-node QNs (name=="__file__") keep the full filename so files that - * share a stem get distinct nodes: auth.py/auth.ts, node.h/node.c, - * .env/.env.local (upstream #769/#1077/#964). Module and symbol QNs keep - * stripping; C/C++ declaration<->definition resolution relies on that. - * Changing this scheme requires bumping CBM_INDEX_FORMAT_VERSION_STR. */ - if (!name || strcmp(name, "__file__") != 0) { - strip_file_extension(path); - } - - const char *segments[CBM_SZ_256]; - int seg_count = 0; - segments[seg_count++] = project; - seg_count += tokenize_path(path, segments + seg_count, FQN_MAX_PATH_SEGS); - - strip_init_or_index(segments, &seg_count, name); - - if (name && name[0] != '\0') { - segments[seg_count++] = name; - } - - char *result = join_segments(segments, seg_count); - free(path); - return result; -} - -char *cbm_pipeline_fqn_module(const char *project, const char *rel_path) { - return cbm_pipeline_fqn_compute(project, rel_path, NULL); -} - -char *cbm_pipeline_fqn_module_dir(const char *project, const char *rel_path, bool module_is_dir) { - if (!module_is_dir) { - /* Filename-stem module (default for all but Java/Go). */ - return cbm_pipeline_fqn_module(project, rel_path); - } - /* Directory-module languages (Java package, Go package): the module is the - * CONTAINING DIRECTORY — strip the basename so a sibling file in the same - * dir shares the module QN. This MUST agree with the extraction-side - * cbm_fqn_module_source_lang() (internal/cbm/helpers.c) so the cross-file - * LSP caller_qn matches the def-node QN. */ - const char *src = rel_path ? rel_path : ""; - /* Strip the last path segment using either separator (the extraction side - * normalizes too); look for the rightmost '/' or '\\'. */ - const char *last_fwd = strrchr(src, '/'); - const char *last_bwd = strrchr(src, '\\'); - const char *last_sep = last_fwd > last_bwd ? last_fwd : last_bwd; - if (!last_sep) { - /* Root file: empty directory → module is just the project. */ - return cbm_pipeline_fqn_folder(project, ""); - } - size_t dir_len = (size_t)(last_sep - src); - char *dir = (char *)malloc(dir_len + 1); /* +1 for NUL */ - if (!dir) { - return NULL; - } - memcpy(dir, src, dir_len); - dir[dir_len] = '\0'; - char *res = cbm_pipeline_fqn_folder(project, dir); - free(dir); - return res; -} - -enum { - FQN_PATH_BUF = 1024, - FQN_SEP_LEN = 1, /* one byte for the '/' separator */ - FQN_NUL_LEN = 1, /* one byte for the terminating NUL */ - FQN_DOTDOT_LEN = 2, - FQN_MIN_PY_DOTS = 1, /* first leading dot is "current package", not a pop */ - FQN_REL_KIND_NONE = 0, - FQN_REL_KIND_PYTHON = 1, - FQN_REL_KIND_JS = 2, -}; - -/* Append a single path segment to a mutable buffer that already holds a - * normalized slash-separated path. Adds a '/' separator when needed, - * returns false if the buffer would overflow. */ -static bool path_append_segment(char *buf, size_t buf_size, const char *seg, size_t seg_len) { - size_t cur = strlen(buf); - size_t need = cur + (cur > 0 ? FQN_SEP_LEN : 0) + seg_len + FQN_NUL_LEN; - if (need > buf_size) { - return false; - } - if (cur > 0) { - buf[cur++] = '/'; - } - memcpy(buf + cur, seg, seg_len); - buf[cur + seg_len] = '\0'; - return true; -} - -/* Pop the last segment from a mutable slash-separated path. */ -static void path_pop_segment(char *buf) { - char *last = strrchr(buf, '/'); - if (last) { - *last = '\0'; - } else { - buf[0] = '\0'; - } -} - -/* Seed `buf` with the source file's directory (strip the basename) and - * normalize backslashes. */ -static void seed_source_dir(char *buf, size_t buf_size, const char *source_rel) { - snprintf(buf, buf_size, "%s", source_rel ? source_rel : ""); - for (char *p = buf; *p; p++) { - if (*p == '\\') { - *p = '/'; - } - } - char *last = strrchr(buf, '/'); - if (last) { - *last = '\0'; - } else { - buf[0] = '\0'; - } -} - -/* Detect the flavor of relative import based on the leading characters. - * Returns 1 for Python dotted form (e.g. ".foo" or "..bar.baz"), - * 2 for JS/TS slash form (e.g. "./foo" or "../bar/baz") and for a Zig - * file import ("db.zig", "sub/x.zig"), which Zig resolves against - * the importing file's directory with or without a leading "./", - * 0 for anything not relative (caller should skip). */ -static int classify_relative_import(const char *module_path) { - size_t len = module_path ? strlen(module_path) : 0; - if (len > 4 && module_path[0] != '/' && strcmp(module_path + len - 4, ".zig") == 0) { - return FQN_REL_KIND_JS; - } - if (!module_path || module_path[0] != '.') { - return FQN_REL_KIND_NONE; - } - bool has_slash = strchr(module_path, '/') != NULL; - bool js_like = module_path[FQN_SEP_LEN] == '/' || - (module_path[FQN_SEP_LEN] == '.' && module_path[FQN_DOTDOT_LEN] == '/'); - if (has_slash || js_like) { - return FQN_REL_KIND_JS; - } - return FQN_REL_KIND_PYTHON; -} - -/* Python relative import: ".foo", "..bar.baz" → resolve against source dir. */ -static char *resolve_python_relative(char *buf, size_t buf_size, const char *module_path) { - const char *p = module_path; - int dot_count = 0; - while (*p == '.') { - dot_count++; - p++; - } - for (int i = FQN_MIN_PY_DOTS; i < dot_count; i++) { - path_pop_segment(buf); - } - while (*p) { - const char *seg_start = p; - while (*p && *p != '.') { - p++; - } - size_t seg_len = (size_t)(p - seg_start); - if (seg_len > 0 && !path_append_segment(buf, buf_size, seg_start, seg_len)) { - return NULL; - } - if (*p == '.') { - p++; - } - } - return strdup(buf); -} - -/* Strip an explicit JS/TS (or Zig) module file extension while preserving dots - * that are part of an extensionless basename (e.g. "featureX.engine"). */ -static size_t strip_js_module_ext(const char *seg_start, size_t seg_len) { - static const char *const extensions[] = { - ".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", ".mts", ".cts", ".json", ".zig", - }; - for (size_t i = 0; i < sizeof(extensions) / sizeof(extensions[0]); i++) { - size_t ext_len = strlen(extensions[i]); - if (seg_len > ext_len && - memcmp(seg_start + seg_len - ext_len, extensions[i], ext_len) == 0) { - return seg_len - ext_len; - } - } - return seg_len; -} - -/* JS/TS relative import: "./foo", "../bar/baz" → resolve against source dir. */ -static char *resolve_js_relative(char *buf, size_t buf_size, const char *module_path) { - const char *p = module_path; - while (*p) { - while (*p == '/') { - p++; - } - if (!*p) { - break; - } - const char *seg_start = p; - while (*p && *p != '/') { - p++; - } - size_t seg_len = (size_t)(p - seg_start); - if (seg_len == FQN_SEP_LEN && seg_start[0] == '.') { - continue; - } - if (seg_len == FQN_DOTDOT_LEN && seg_start[0] == '.' && seg_start[FQN_SEP_LEN] == '.') { - path_pop_segment(buf); - continue; - } - if (*p == '\0') { - seg_len = strip_js_module_ext(seg_start, seg_len); - } - if (seg_len > 0 && !path_append_segment(buf, buf_size, seg_start, seg_len)) { - return NULL; - } - } - return strdup(buf); -} - -char *cbm_pipeline_resolve_relative_import(const char *source_rel, const char *module_path) { - int kind = classify_relative_import(module_path); - if (kind == FQN_REL_KIND_NONE) { - return NULL; - } - char buf[FQN_PATH_BUF]; - seed_source_dir(buf, sizeof(buf), source_rel); - if (kind == FQN_REL_KIND_PYTHON) { - return resolve_python_relative(buf, sizeof(buf), module_path); - } - return resolve_js_relative(buf, sizeof(buf), module_path); -} - -char *cbm_pipeline_fqn_folder(const char *project, const char *rel_dir) { - if (!project) { - return strdup(""); - } - - /* Work on mutable copy */ - char *dir = strdup(rel_dir ? rel_dir : ""); - cbm_normalize_path_sep(dir); - - const char *segments[CBM_SZ_256]; - int seg_count = 0; - segments[seg_count++] = project; - - if (dir[0] != '\0') { - char *tok = dir; - while (tok && *tok && seg_count < FQN_MAX_DIR_SEGS) { - char *slash = strchr(tok, '/'); - if (slash) { - *slash = '\0'; - } - if (tok[0] != '\0') { - segments[seg_count++] = tok; - } - tok = slash ? slash + SKIP_ONE : NULL; - } - } - - char *result = join_segments(segments, seg_count); - free(dir); - return result; -} - -/* Bound a derived project name to FQN_MAX_NAME_LEN bytes so "/.db" - * stays within the filesystem's 255-byte filename-component limit (#624). Names - * within the cap are returned UNCHANGED (no drift). Longer names keep their first - * (CAP-9) bytes and get a "-XXXXXXXX" FNV-1a hash of the FULL name appended, so - * two long paths that share a prefix but differ later still map to distinct - * names. The suffix ends in a hex digit, so the result stays validator-safe. */ -static char *fqn_bound_name_len(char *name) { - if (!name) { - return name; - } - size_t n = strlen(name); - if (n <= FQN_MAX_NAME_LEN) { - return name; /* within cap → unchanged, no drift */ - } - uint32_t h = 2166136261u; /* FNV-1a offset basis over the FULL name */ - for (size_t i = 0; i < n; i++) { - h ^= (unsigned char)name[i]; - h *= 16777619u; - } - /* Keep first (CAP-9) bytes + "-" + 8 hex = CAP total. The buffer holds n+1 - * bytes and n > CAP, so writing 9 chars + NUL at offset (CAP-9) fits. */ - snprintf(name + (FQN_MAX_NAME_LEN - 9), 10, "-%08x", h); - return name; -} - -static bool path_is_root_syntax(const char *path) { - if (!path || !path[0]) { - return false; - } - for (const char *p = path; *p; p++) { - if (*p != '/' && *p != '\\' && *p != ':') { - return false; - } - } - return true; -} - -char *cbm_project_name_from_path(const char *abs_path) { - if (!abs_path || !abs_path[0]) { - return strdup("root"); - } - if (path_is_root_syntax(abs_path)) { - return strdup("root"); - } - - char real[CBM_SZ_4K]; - const char *name_path = abs_path; -#ifdef _WIN32 - if (_access(abs_path, 0) == 0 && _fullpath(real, abs_path, sizeof(real))) { - cbm_normalize_path_sep(real); - name_path = real; - } -#else - if (realpath(abs_path, real)) { - cbm_normalize_path_sep(real); - name_path = real; - } -#endif - - /* Work on mutable copy */ - char *path = strdup(name_path); - if (!path) { - return NULL; - } - size_t len = strlen(path); - - /* Normalize path separators */ - cbm_normalize_path_sep(path); - - /* Map every character that is unsafe for portable project DB names. We - * keep derived names in [A-Za-z0-9._-], so anything else — path - * separators, ':', spaces, '@', '+', … — must be normalized here. - * Otherwise a repo like - * "/home/u/my project" yields the name "home-u-my project": indexing - * creates the DB and it shows in list_projects, but resolve_store rejects - * the space and reports project-not-found (#349). - * - * Non-ASCII bytes (UTF-8 of CJK and other scripts, all >= 0x80) are NOT - * dropped to '-' — that silently erased whole path segments and produced - * unrecognizable / colliding names (#571). Instead each non-ASCII byte is - * transliterated to its two lowercase hex digits, which use only [0-9a-f] - * and therefore stay validator-safe while preserving the segment. */ - static const char hex_digits[] = "0123456789abcdef"; - char *mapped = malloc(len * 2 + 1); /* worst case: every byte → 2 hex chars */ - if (!mapped) { - free(path); - return strdup("root"); - } - size_t mlen = 0; - for (size_t i = 0; i < len; i++) { - unsigned char c = (unsigned char)path[i]; - bool safe = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || - c == '.' || c == '_' || c == '-'; - if (safe) { - mapped[mlen++] = (char)c; - } else if (c >= 0x80) { - mapped[mlen++] = hex_digits[(c >> 4) & 0xF]; - mapped[mlen++] = hex_digits[c & 0xF]; - } else { - mapped[mlen++] = '-'; - } - } - mapped[mlen] = '\0'; - free(path); - path = mapped; - len = mlen; - - /* Collapse consecutive dashes, and consecutive dots (the validator also - * rejects any ".." sequence). */ - char *dst = path; - char prev = 0; - for (size_t i = 0; i < len; i++) { - if ((path[i] == '-' && prev == '-') || (path[i] == '.' && prev == '.')) { - continue; - } - *dst++ = path[i]; - prev = path[i]; - } - *dst = '\0'; - - /* Trim leading dashes and dots (the validator rejects a leading dot). */ - char *start = path; - while (*start == '-' || *start == '.') { - start++; - } - - /* Trim trailing dashes */ - size_t slen = strlen(start); - while (slen > 0 && start[slen - SKIP_ONE] == '-') { - start[--slen] = '\0'; - } - - if (*start == '\0') { - free(path); - return strdup("root"); - } - - char *result = strdup(start); - free(path); - if (result) { - result = fqn_bound_name_len(result); /* #624: cap filename-component length */ - } - return result; -} - -static char *project_selector_component(const char *component) { - if (!component || !component[0]) { - return strdup("root"); - } - - size_t len = strlen(component); - char *mapped = malloc(len * 2 + 1); - if (!mapped) { - return NULL; - } - static const char hex_digits[] = "0123456789abcdef"; - size_t used = 0; - char previous = 0; - for (size_t i = 0; i < len; i++) { - unsigned char c = (unsigned char)component[i]; - bool safe = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || - (c >= '0' && c <= '9') || c == '.' || c == '_' || c == '-'; - if (safe) { - if ((c == '-' && previous == '-') || (c == '.' && previous == '.')) { - continue; - } - mapped[used++] = (char)c; - previous = (char)c; - } else if (c >= 0x80) { - mapped[used++] = hex_digits[(c >> 4) & 0xf]; - mapped[used++] = hex_digits[c & 0xf]; - previous = mapped[used - 1]; - } else if (previous != '-') { - mapped[used++] = '-'; - previous = '-'; - } - } - mapped[used] = '\0'; - - char *start = mapped; - while (*start == '-' || *start == '.') { - start++; - } - size_t clean_len = strlen(start); - while (clean_len > 0 && start[clean_len - 1] == '-') { - start[--clean_len] = '\0'; - } - char *result = strdup(start[0] ? start : "root"); - free(mapped); - return result ? fqn_bound_name_len(result) : NULL; -} - -char *cbm_project_selector_from_path(const char *abs_path, size_t parent_levels) { - if (!abs_path || !abs_path[0] || path_is_root_syntax(abs_path)) { - return strdup("root"); - } - - char *path = strdup(abs_path); - if (!path) { - return NULL; - } - cbm_normalize_path_sep(path); - size_t len = strlen(path); - while (len > 0 && path[len - 1] == '/') { - path[--len] = '\0'; - } - if (!len) { - free(path); - return strdup("root"); - } - - size_t component_capacity = len + 1; - char **components = malloc(component_capacity * sizeof(*components)); - if (!components) { - free(path); - return NULL; - } - size_t component_count = 0; - char *cursor = path; - while (*cursor) { - while (*cursor == '/') cursor++; - if (!*cursor) break; - components[component_count++] = cursor; - while (*cursor && *cursor != '/') cursor++; - if (*cursor) *cursor++ = '\0'; - } - if (!component_count) { - free(components); - free(path); - return strdup("root"); - } - - size_t wanted = parent_levels == SIZE_MAX ? component_count : parent_levels + 1; - if (wanted > component_count) wanted = component_count; - size_t first = component_count - wanted; - char **clean = calloc(wanted, sizeof(*clean)); - size_t output_len = wanted > 0 ? wanted - 1 : 0; - if (!clean) { - free(components); - free(path); - return NULL; - } - bool ok = true; - for (size_t i = 0; i < wanted; i++) { - clean[i] = project_selector_component(components[first + i]); - if (!clean[i]) { - ok = false; - break; - } - if (SIZE_MAX - output_len < strlen(clean[i])) { - ok = false; - break; - } - output_len += strlen(clean[i]); - } - - char *result = ok ? malloc(output_len + 1) : NULL; - if (result) { - char *out = result; - for (size_t i = 0; i < wanted; i++) { - if (i) *out++ = '/'; - size_t part_len = strlen(clean[i]); - memcpy(out, clean[i], part_len); - out += part_len; - } - *out = '\0'; - } - for (size_t i = 0; i < wanted; i++) free(clean[i]); - free(clean); - free(components); - free(path); - return result; -} diff --git a/src/pipeline/fqn.zig b/src/pipeline/fqn.zig new file mode 100644 index 00000000..11b89c8a --- /dev/null +++ b/src/pipeline/fqn.zig @@ -0,0 +1,582 @@ +//! fqn.zig — Fully Qualified Name computation for graph nodes. +//! +//! Implements the FQN functions of src/pipeline/pipeline.h. The FQN scheme is +//! project.dir.parts.name. Handles Python __init__.py, JS/TS index.{js,ts}, +//! path separators. + +const std = @import("std"); +const builtin = @import("builtin"); +const c = @import("c"); + +const CStr = [*c]const u8; +const MutStr = [*c]u8; + +/// Maximum path segments in a FQN (CBM_SZ_256 slots total, -2 for project + name) +const FQN_MAX_PATH_SEGS = 254; +const FQN_MAX_DIR_SEGS = 255; + +/// Max bytes for a derived project name. Names were once filename components, +/// so they stay under the 255-byte component limit. #571 hex-encodes each +/// non-ASCII byte to 2 chars, so a deep CJK path can triple past 255 (#624). +const FQN_MAX_NAME_LEN = 200; + +// ── Internal helpers ──────────────────────────────────────────────── + +/// Build a dot-joined string from segments. Returns heap-allocated string. +fn joinSegments(segments: []const CStr) MutStr { + if (segments.len == 0) return c.strdup(""); + var total: usize = 0; + for (segments, 0..) |seg, i| { + total += c.strlen(seg); + if (i > 0) total += 1; // dot separator + } + const result: MutStr = @ptrCast(c.malloc(total + 1)); + if (result == null) return null; + var p: [*]u8 = result; + for (segments, 0..) |seg, i| { + if (i > 0) { + p[0] = '.'; + p += 1; + } + const len = c.strlen(seg); + @memcpy(p[0..len], seg[0..len]); + p += len; + } + p[0] = 0; + return result; +} + +/// Strip file extension from the last path component. +fn stripFileExtension(path: MutStr) void { + const last_slash: MutStr = c.strrchr(path, '/'); + const start: MutStr = if (last_slash != null) last_slash + 1 else path; + const ext: MutStr = c.strrchr(start, '.'); + if (ext != null) ext[0] = 0; +} + +/// Tokenize path by '/' into segments array. Returns number of segments added. +fn tokenizePath(path: MutStr, segments: []CStr) usize { + var count: usize = 0; + if (path[0] == 0) return 0; + var tok: MutStr = path; + while (tok != null and tok[0] != 0 and count < segments.len) { + const slash: MutStr = c.strchr(tok, '/'); + if (slash != null) slash[0] = 0; + if (tok[0] != 0) { + segments[count] = tok; + count += 1; + } + tok = if (slash != null) slash + 1 else null; + } + return count; +} + +/// Strip __init__ (Python) / index (JS/TS) from the last segment when a +/// symbol name is provided. Keeps it when no name is given to avoid QN +/// collision with Folder nodes for the same directory. +fn stripInitOrIndex(segments: []const CStr, seg_count: *usize, name: CStr) void { + if (seg_count.* <= 1) return; + const last = segments[seg_count.* - 1]; + if (c.strcmp(last, "__init__") != 0 and c.strcmp(last, "index") != 0) return; + if (name != null and name[0] != 0) seg_count.* -= 1; +} + +// ── Public API ────────────────────────────────────────────────────── + +pub export fn cbm_pipeline_fqn_compute(project: CStr, rel_path: CStr, name: CStr) MutStr { + if (project == null) return c.strdup(""); + + const path: MutStr = c.strdup(if (rel_path != null) rel_path else ""); + _ = c.cbm_normalize_path_sep(path); + // File-node QNs (name=="__file__") keep the full filename so files that + // share a stem get distinct nodes: auth.py/auth.ts, node.h/node.c, + // .env/.env.local (upstream #769/#1077/#964). Module and symbol QNs keep + // stripping; C/C++ declaration<->definition resolution relies on that. + // Changing this scheme requires bumping CBM_INDEX_FORMAT_VERSION_STR. + if (name == null or c.strcmp(name, "__file__") != 0) { + stripFileExtension(path); + } + + var segments: [c.CBM_SZ_256]CStr = undefined; + var seg_count: usize = 0; + segments[seg_count] = project; + seg_count += 1; + seg_count += tokenizePath(path, segments[seg_count..][0..FQN_MAX_PATH_SEGS]); + + stripInitOrIndex(&segments, &seg_count, name); + + if (name != null and name[0] != 0) { + segments[seg_count] = name; + seg_count += 1; + } + + const result = joinSegments(segments[0..seg_count]); + c.free(path); + return result; +} + +pub export fn cbm_pipeline_fqn_module(project: CStr, rel_path: CStr) MutStr { + return cbm_pipeline_fqn_compute(project, rel_path, null); +} + +pub export fn cbm_pipeline_fqn_module_dir(project: CStr, rel_path: CStr, module_is_dir: bool) MutStr { + if (!module_is_dir) { + // Filename-stem module (default for all but Java/Go). + return cbm_pipeline_fqn_module(project, rel_path); + } + // Directory-module languages (Java package, Go package): the module is the + // CONTAINING DIRECTORY — strip the basename so a sibling file in the same + // dir shares the module QN. This MUST agree with the extraction-side + // cbm_fqn_module_source_lang() (internal/cbm/helpers.c) so the cross-file + // LSP caller_qn matches the def-node QN. + const src: CStr = if (rel_path != null) rel_path else ""; + // Strip the last path segment using either separator (the extraction side + // normalizes too); look for the rightmost '/' or '\\'. + const last_fwd: CStr = c.strrchr(src, '/'); + const last_bwd: CStr = c.strrchr(src, '\\'); + const last_sep: CStr = if (@intFromPtr(last_fwd) > @intFromPtr(last_bwd)) last_fwd else last_bwd; + if (last_sep == null) { + // Root file: empty directory → module is just the project. + return cbm_pipeline_fqn_folder(project, ""); + } + const dir_len = @intFromPtr(last_sep) - @intFromPtr(src); + const dir: MutStr = @ptrCast(c.malloc(dir_len + 1)); // +1 for NUL + if (dir == null) return null; + @memcpy(dir[0..dir_len], src[0..dir_len]); + dir[dir_len] = 0; + const res = cbm_pipeline_fqn_folder(project, dir); + c.free(dir); + return res; +} + +const FQN_PATH_BUF = 1024; +const FQN_SEP_LEN = 1; // one byte for the '/' separator +const FQN_NUL_LEN = 1; // one byte for the terminating NUL +const FQN_DOTDOT_LEN = 2; +const FQN_MIN_PY_DOTS = 1; // first leading dot is "current package", not a pop + +const RelKind = enum { none, python, js }; + +/// Append a single path segment to a mutable buffer that already holds a +/// normalized slash-separated path. Adds a '/' separator when needed, +/// returns false if the buffer would overflow. +fn pathAppendSegment(buf: []u8, seg: []const u8) bool { + var cur = c.strlen(buf.ptr); + const need = cur + @as(usize, if (cur > 0) FQN_SEP_LEN else 0) + seg.len + FQN_NUL_LEN; + if (need > buf.len) return false; + if (cur > 0) { + buf[cur] = '/'; + cur += 1; + } + @memcpy(buf[cur..][0..seg.len], seg); + buf[cur + seg.len] = 0; + return true; +} + +/// Pop the last segment from a mutable slash-separated path. +fn pathPopSegment(buf: [*]u8) void { + const last: MutStr = c.strrchr(buf, '/'); + if (last != null) { + last[0] = 0; + } else { + buf[0] = 0; + } +} + +/// Seed `buf` with the source file's directory (strip the basename) and +/// normalize backslashes. +fn seedSourceDir(buf: []u8, source_rel: CStr) void { + _ = c.snprintf(buf.ptr, buf.len, "%s", @as(CStr, if (source_rel != null) source_rel else "")); + var i: usize = 0; + while (buf[i] != 0) : (i += 1) { + if (buf[i] == '\\') buf[i] = '/'; + } + pathPopSegment(buf.ptr); +} + +/// Detect the flavor of relative import based on the leading characters. +/// Returns .python for the Python dotted form (e.g. ".foo" or "..bar.baz"), +/// .js for the JS/TS slash form (e.g. "./foo" or "../bar/baz") and for +/// a Zig file import ("db.zig", "sub/x.zig"), which Zig resolves +/// against the importing file's directory with or without a leading +/// "./", +/// .none for anything not relative (caller should skip). +fn classifyRelativeImport(module_path: CStr) RelKind { + const len: usize = if (module_path != null) c.strlen(module_path) else 0; + if (len > 4 and module_path[0] != '/' and c.strcmp(module_path + len - 4, ".zig") == 0) { + return .js; + } + if (module_path == null or module_path[0] != '.') return .none; + const has_slash = c.strchr(module_path, '/') != null; + const js_like = module_path[FQN_SEP_LEN] == '/' or + (module_path[FQN_SEP_LEN] == '.' and module_path[FQN_DOTDOT_LEN] == '/'); + if (has_slash or js_like) return .js; + return .python; +} + +/// Python relative import: ".foo", "..bar.baz" → resolve against source dir. +fn resolvePythonRelative(buf: []u8, module_path: [*:0]const u8) MutStr { + var p = module_path; + var dot_count: c_int = 0; + while (p[0] == '.') { + dot_count += 1; + p += 1; + } + var i: c_int = FQN_MIN_PY_DOTS; + while (i < dot_count) : (i += 1) { + pathPopSegment(buf.ptr); + } + while (p[0] != 0) { + const seg_start = p; + while (p[0] != 0 and p[0] != '.') p += 1; + const seg_len = @intFromPtr(p) - @intFromPtr(seg_start); + if (seg_len > 0 and !pathAppendSegment(buf, seg_start[0..seg_len])) return null; + if (p[0] == '.') p += 1; + } + return c.strdup(buf.ptr); +} + +/// Strip an explicit JS/TS (or Zig) module file extension while preserving dots +/// that are part of an extensionless basename (e.g. "featureX.engine"). +fn stripJsModuleExt(seg: []const u8) usize { + const extensions = [_][]const u8{ + ".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", ".mts", ".cts", ".json", ".zig", + }; + for (extensions) |ext| { + if (seg.len > ext.len and std.mem.eql(u8, seg[seg.len - ext.len ..], ext)) { + return seg.len - ext.len; + } + } + return seg.len; +} + +/// JS/TS relative import: "./foo", "../bar/baz" → resolve against source dir. +fn resolveJsRelative(buf: []u8, module_path: [*:0]const u8) MutStr { + var p = module_path; + while (p[0] != 0) { + while (p[0] == '/') p += 1; + if (p[0] == 0) break; + const seg_start = p; + while (p[0] != 0 and p[0] != '/') p += 1; + var seg_len = @intFromPtr(p) - @intFromPtr(seg_start); + if (seg_len == FQN_SEP_LEN and seg_start[0] == '.') continue; + if (seg_len == FQN_DOTDOT_LEN and seg_start[0] == '.' and seg_start[FQN_SEP_LEN] == '.') { + pathPopSegment(buf.ptr); + continue; + } + if (p[0] == 0) { + seg_len = stripJsModuleExt(seg_start[0..seg_len]); + } + if (seg_len > 0 and !pathAppendSegment(buf, seg_start[0..seg_len])) return null; + } + return c.strdup(buf.ptr); +} + +pub export fn cbm_pipeline_resolve_relative_import(source_rel: CStr, module_path: CStr) MutStr { + const kind = classifyRelativeImport(module_path); + if (kind == .none) return null; + var buf: [FQN_PATH_BUF]u8 = undefined; + seedSourceDir(&buf, source_rel); + if (kind == .python) { + return resolvePythonRelative(&buf, module_path); + } + return resolveJsRelative(&buf, module_path); +} + +pub export fn cbm_pipeline_fqn_folder(project: CStr, rel_dir: CStr) MutStr { + if (project == null) return c.strdup(""); + + // Work on mutable copy + const dir: MutStr = c.strdup(if (rel_dir != null) rel_dir else ""); + _ = c.cbm_normalize_path_sep(dir); + + var segments: [c.CBM_SZ_256]CStr = undefined; + var seg_count: usize = 0; + segments[seg_count] = project; + seg_count += 1; + + if (dir[0] != 0) { + var tok: MutStr = dir; + while (tok != null and tok[0] != 0 and seg_count < FQN_MAX_DIR_SEGS) { + const slash: MutStr = c.strchr(tok, '/'); + if (slash != null) slash[0] = 0; + if (tok[0] != 0) { + segments[seg_count] = tok; + seg_count += 1; + } + tok = if (slash != null) slash + 1 else null; + } + } + + const result = joinSegments(segments[0..seg_count]); + c.free(dir); + return result; +} + +/// Bound a derived project name to FQN_MAX_NAME_LEN bytes so "/.db" +/// stays within the filesystem's 255-byte filename-component limit (#624). Names +/// within the cap are returned UNCHANGED (no drift). Longer names keep their first +/// (CAP-9) bytes and get a "-XXXXXXXX" FNV-1a hash of the FULL name appended, so +/// two long paths that share a prefix but differ later still map to distinct +/// names. The suffix ends in a hex digit, so the result stays validator-safe. +fn fqnBoundNameLen(name: MutStr) MutStr { + if (name == null) return name; + const n = c.strlen(name); + if (n <= FQN_MAX_NAME_LEN) return name; // within cap → unchanged, no drift + var h: u32 = 2166136261; // FNV-1a offset basis over the FULL name + for (name[0..n]) |ch| { + h ^= ch; + h *%= 16777619; + } + // Keep first (CAP-9) bytes + "-" + 8 hex = CAP total. The buffer holds n+1 + // bytes and n > CAP, so writing 9 chars + NUL at offset (CAP-9) fits. + _ = c.snprintf(name + (FQN_MAX_NAME_LEN - 9), 10, "-%08x", h); + return name; +} + +fn pathIsRootSyntax(path: CStr) bool { + if (path == null or path[0] == 0) return false; + var i: usize = 0; + while (path[i] != 0) : (i += 1) { + if (path[i] != '/' and path[i] != '\\' and path[i] != ':') return false; + } + return true; +} + +fn isNameSafe(ch: u8) bool { + return (ch >= 'a' and ch <= 'z') or (ch >= 'A' and ch <= 'Z') or (ch >= '0' and ch <= '9') or + ch == '.' or ch == '_' or ch == '-'; +} + +extern fn _fullpath(abs: [*]u8, rel: [*:0]const u8, max: usize) ?[*:0]u8; +extern fn _access(path: [*:0]const u8, mode: c_int) c_int; + +/// realpath(3), or _access + _fullpath on Windows. True when out holds the +/// resolved path. +fn resolveRealPath(path: [*:0]const u8, out: *[c.CBM_SZ_4K]u8) bool { + if (builtin.os.tag == .windows) { + return _access(path, 0) == 0 and _fullpath(out, path, out.len) != null; + } + return c.realpath(path, out) != null; +} + +const hex_digits = "0123456789abcdef"; + +pub export fn cbm_project_name_from_path(abs_path: CStr) MutStr { + if (abs_path == null or abs_path[0] == 0) return c.strdup("root"); + if (pathIsRootSyntax(abs_path)) return c.strdup("root"); + + var real: [c.CBM_SZ_4K]u8 = undefined; + var name_path: CStr = abs_path; + if (resolveRealPath(abs_path, &real)) { + _ = c.cbm_normalize_path_sep(&real); + name_path = ℜ + } + + // Work on mutable copy + var path: MutStr = c.strdup(name_path); + if (path == null) return null; + var len = c.strlen(path); + + // Normalize path separators + _ = c.cbm_normalize_path_sep(path); + + // Map every character that is unsafe for portable project DB names. We + // keep derived names in [A-Za-z0-9._-], so anything else — path + // separators, ':', spaces, '@', '+', … — must be normalized here. + // Otherwise a repo like + // "/home/u/my project" yields the name "home-u-my project": indexing + // creates the DB and it shows in list_projects, but resolve_store rejects + // the space and reports project-not-found (#349). + // + // Non-ASCII bytes (UTF-8 of CJK and other scripts, all >= 0x80) are NOT + // dropped to '-' — that silently erased whole path segments and produced + // unrecognizable / colliding names (#571). Instead each non-ASCII byte is + // transliterated to its two lowercase hex digits, which use only [0-9a-f] + // and therefore stay validator-safe while preserving the segment. + const mapped: MutStr = @ptrCast(c.malloc(len * 2 + 1)); // worst case: every byte → 2 hex chars + if (mapped == null) { + c.free(path); + return c.strdup("root"); + } + var mlen: usize = 0; + for (path[0..len]) |ch| { + if (isNameSafe(ch)) { + mapped[mlen] = ch; + mlen += 1; + } else if (ch >= 0x80) { + mapped[mlen] = hex_digits[(ch >> 4) & 0xF]; + mapped[mlen + 1] = hex_digits[ch & 0xF]; + mlen += 2; + } else { + mapped[mlen] = '-'; + mlen += 1; + } + } + mapped[mlen] = 0; + c.free(path); + path = mapped; + len = mlen; + + // Collapse consecutive dashes, and consecutive dots (the validator also + // rejects any ".." sequence). + var dst: usize = 0; + var prev: u8 = 0; + for (0..len) |i| { + if ((path[i] == '-' and prev == '-') or (path[i] == '.' and prev == '.')) continue; + path[dst] = path[i]; + dst += 1; + prev = path[i]; + } + path[dst] = 0; + + // Trim leading dashes and dots (the validator rejects a leading dot). + var start: MutStr = path; + while (start[0] == '-' or start[0] == '.') start += 1; + + // Trim trailing dashes + var slen = c.strlen(start); + while (slen > 0 and start[slen - 1] == '-') { + slen -= 1; + start[slen] = 0; + } + + if (start[0] == 0) { + c.free(path); + return c.strdup("root"); + } + + var result: MutStr = c.strdup(start); + c.free(path); + if (result != null) { + result = fqnBoundNameLen(result); // #624: cap filename-component length + } + return result; +} + +fn projectSelectorComponent(component: CStr) MutStr { + if (component == null or component[0] == 0) return c.strdup("root"); + + const len = c.strlen(component); + const mapped: MutStr = @ptrCast(c.malloc(len * 2 + 1)); + if (mapped == null) return null; + var used: usize = 0; + var previous: u8 = 0; + for (component[0..len]) |ch| { + if (isNameSafe(ch)) { + if ((ch == '-' and previous == '-') or (ch == '.' and previous == '.')) continue; + mapped[used] = ch; + used += 1; + previous = ch; + } else if (ch >= 0x80) { + mapped[used] = hex_digits[(ch >> 4) & 0xf]; + mapped[used + 1] = hex_digits[ch & 0xf]; + used += 2; + previous = mapped[used - 1]; + } else if (previous != '-') { + mapped[used] = '-'; + used += 1; + previous = '-'; + } + } + mapped[used] = 0; + + var start: MutStr = mapped; + while (start[0] == '-' or start[0] == '.') start += 1; + var clean_len = c.strlen(start); + while (clean_len > 0 and start[clean_len - 1] == '-') { + clean_len -= 1; + start[clean_len] = 0; + } + const result: MutStr = c.strdup(if (start[0] != 0) start else "root"); + c.free(mapped); + return if (result != null) fqnBoundNameLen(result) else null; +} + +pub export fn cbm_project_selector_from_path(abs_path: CStr, parent_levels: usize) MutStr { + if (abs_path == null or abs_path[0] == 0 or pathIsRootSyntax(abs_path)) { + return c.strdup("root"); + } + + const path: MutStr = c.strdup(abs_path); + if (path == null) return null; + _ = c.cbm_normalize_path_sep(path); + var len = c.strlen(path); + while (len > 0 and path[len - 1] == '/') { + len -= 1; + path[len] = 0; + } + if (len == 0) { + c.free(path); + return c.strdup("root"); + } + + const component_capacity = len + 1; + const components: [*c]MutStr = @ptrCast(@alignCast(c.malloc(component_capacity * @sizeOf(MutStr)))); + if (components == null) { + c.free(path); + return null; + } + var component_count: usize = 0; + var cursor: [*]u8 = path; + while (cursor[0] != 0) { + while (cursor[0] == '/') cursor += 1; + if (cursor[0] == 0) break; + components[component_count] = cursor; + component_count += 1; + while (cursor[0] != 0 and cursor[0] != '/') cursor += 1; + if (cursor[0] != 0) { + cursor[0] = 0; + cursor += 1; + } + } + if (component_count == 0) { + c.free(@ptrCast(components)); + c.free(path); + return c.strdup("root"); + } + + var wanted = if (parent_levels == std.math.maxInt(usize)) component_count else parent_levels + 1; + if (wanted > component_count) wanted = component_count; + const first = component_count - wanted; + const clean: [*c]MutStr = @ptrCast(@alignCast(c.calloc(wanted, @sizeOf(MutStr)))); + var output_len: usize = if (wanted > 0) wanted - 1 else 0; + if (clean == null) { + c.free(@ptrCast(components)); + c.free(path); + return null; + } + var ok = true; + for (0..wanted) |i| { + clean[i] = projectSelectorComponent(components[first + i]); + if (clean[i] == null) { + ok = false; + break; + } + if (std.math.maxInt(usize) - output_len < c.strlen(clean[i])) { + ok = false; + break; + } + output_len += c.strlen(clean[i]); + } + + const result: MutStr = if (ok) @ptrCast(c.malloc(output_len + 1)) else null; + if (result != null) { + var out: [*]u8 = result; + for (0..wanted) |i| { + if (i != 0) { + out[0] = '/'; + out += 1; + } + const part_len = c.strlen(clean[i]); + @memcpy(out[0..part_len], clean[i][0..part_len]); + out += part_len; + } + out[0] = 0; + } + for (0..wanted) |i| c.free(clean[i]); + c.free(@ptrCast(clean)); + c.free(@ptrCast(components)); + c.free(path); + return result; +} diff --git a/src/pipeline/laravel_routing.c b/src/pipeline/laravel_routing.c deleted file mode 100644 index 12ebe253..00000000 --- a/src/pipeline/laravel_routing.c +++ /dev/null @@ -1,402 +0,0 @@ -/* - * laravel_routing.c — Laravel 11+ framework route mounts (#1146). - * - * See laravel_routing.h for the contract. bootstrap/app.php is parsed with - * the same PHP grammar the extractor uses; only named `withRouting(...)` - * arguments whose values are plain literals (or `__DIR__`/`dirname(__DIR__)` - * concatenations and `base_path()` calls over literals) are trusted. - */ -#include "pipeline/laravel_routing.h" - -#include "cbm.h" -#include "lang_specs.h" -#include "service_patterns.h" -#include "tree_sitter/api.h" -#include "foundation/compat_fs.h" -#include "foundation/constants.h" -#include -#include -#include -#include -#include - -enum { - LR_PATH_MAX = CBM_SZ_512, /* repo-relative path buffer */ - LR_KEY_MAX = CBM_SZ_32, /* named-argument key */ - LR_PREFIX_MAX = CBM_SZ_128, /* apiPrefix literal */ - LR_SEG_MAX = CBM_SZ_64, /* path segments tracked while normalising */ - LR_SOURCE_MAX = CBM_SZ_256 * CBM_SZ_1K, /* bootstrap/app.php read cap */ -}; - -static const char LR_BOOTSTRAP_REL[] = "bootstrap/app.php"; - -/* Copy a node's source text into buf. False when it does not fit. */ -static bool lr_node_text(const char *src, TSNode node, char *buf, size_t buf_sz) { - if (ts_node_is_null(node)) { - return false; - } - uint32_t start = ts_node_start_byte(node); - uint32_t end = ts_node_end_byte(node); - size_t len = end > start ? (size_t)(end - start) : 0; - if (len >= buf_sz) { - return false; - } - memcpy(buf, src + start, len); - buf[len] = '\0'; - return true; -} - -static bool lr_node_is(TSNode node, const char *type) { - return !ts_node_is_null(node) && strcmp(ts_node_type(node), type) == 0; -} - -static bool lr_node_text_is(const char *src, TSNode node, const char *text) { - char buf[LR_KEY_MAX]; - return lr_node_text(src, node, buf, sizeof(buf)) && strcmp(buf, text) == 0; -} - -/* A plain string literal: '...' or "..." whose only children are - * string_content (no interpolation, no escapes). Writes the unquoted value. */ -static bool lr_string_literal(const char *src, TSNode node, char *buf, size_t buf_sz) { - if (!lr_node_is(node, "string") && !lr_node_is(node, "encapsed_string")) { - return false; - } - size_t pos = 0; - buf[0] = '\0'; - uint32_t nc = ts_node_named_child_count(node); - for (uint32_t i = 0; i < nc; i++) { - TSNode part = ts_node_named_child(node, i); - if (!lr_node_is(part, "string_content") || - !lr_node_text(src, part, buf + pos, buf_sz - pos)) { - return false; - } - pos += strlen(buf + pos); - } - return true; -} - -/* The value expression of a call argument: its first named child that is - * neither the `name:` label nor a comment. */ -static TSNode lr_arg_value(TSNode arg, TSNode label) { - uint32_t nc = ts_node_named_child_count(arg); - for (uint32_t i = 0; i < nc; i++) { - TSNode child = ts_node_named_child(arg, i); - if ((ts_node_is_null(label) || !ts_node_eq(child, label)) && - !lr_node_is(child, "comment")) { - return child; - } - } - TSNode none = {0}; - return none; -} - -/* The single literal argument of `fn('...')` when the callee is `fn`. */ -static bool lr_call_literal_arg(const char *src, TSNode call, const char *fn, char *buf, - size_t buf_sz) { - if (!lr_node_is(call, "function_call_expression") || - !lr_node_text_is(src, ts_node_child_by_field_name(call, TS_FIELD("function")), fn)) { - return false; - } - TSNode args = ts_node_child_by_field_name(call, TS_FIELD("arguments")); - if (ts_node_is_null(args) || ts_node_named_child_count(args) != SKIP_ONE) { - return false; - } - TSNode arg = ts_node_named_child(args, 0); - TSNode val = lr_arg_value(arg, ts_node_child_by_field_name(arg, TS_FIELD("name"))); - return lr_string_literal(src, val, buf, buf_sz); -} - -/* `dirname(__DIR__)` — the application base directory. */ -static bool lr_is_dirname_dir(const char *src, TSNode node) { - if (!lr_node_is(node, "function_call_expression") || - !lr_node_text_is(src, ts_node_child_by_field_name(node, TS_FIELD("function")), "dirname")) { - return false; - } - TSNode args = ts_node_child_by_field_name(node, TS_FIELD("arguments")); - if (ts_node_is_null(args) || ts_node_named_child_count(args) != SKIP_ONE) { - return false; - } - TSNode arg = ts_node_named_child(args, 0); - TSNode val = lr_arg_value(arg, ts_node_child_by_field_name(arg, TS_FIELD("name"))); - return lr_node_is(val, "name") && lr_node_text_is(src, val, "__DIR__"); -} - -/* Resolve "." and ".." segments of a '/'-joined path in place. False when the - * path climbs above the repository root. */ -static bool lr_normalize(char *path) { - size_t seg_start[LR_SEG_MAX]; - int depth = 0; - size_t out = 0; - const char *p = path; - while (*p) { - const char *slash = strchr(p, '/'); - size_t len = slash ? (size_t)(slash - p) : strlen(p); - if (len == PAIR_LEN && p[0] == '.' && p[SKIP_ONE] == '.') { - if (depth == 0) { - return false; - } - out = seg_start[--depth]; - } else if (len > 0 && !(len == SKIP_ONE && p[0] == '.')) { - if (depth == LR_SEG_MAX) { - return false; - } - size_t at = out > 0 ? out + SKIP_ONE : 0; - seg_start[depth++] = out; - if (out > 0) { - path[out] = '/'; - } - memmove(path + at, p, len); - out = at + len; - } - p += len + (slash ? SKIP_ONE : 0); - } - path[out] = '\0'; - return true; -} - -/* Resolve one `api:` path expression of the app rooted at app_dir to a - * normalised repo-relative path: - * __DIR__ . '/../routes/api.php' (relative to bootstrap/) - * dirname(__DIR__) . '/routes/api.php' (relative to the app base) - * base_path('routes/api.php') (relative to the app base) */ -static bool lr_route_file_path(const char *src, TSNode expr, const char *app_dir, char *out, - size_t out_sz) { - char lit[LR_PATH_MAX]; - const char *base_suffix = NULL; - if (lr_node_is(expr, "binary_expression")) { - TSNode op = ts_node_child_by_field_name(expr, TS_FIELD("operator")); - TSNode left = ts_node_child_by_field_name(expr, TS_FIELD("left")); - if (!lr_node_text_is(src, op, ".") || - !lr_string_literal(src, ts_node_child_by_field_name(expr, TS_FIELD("right")), lit, - sizeof(lit))) { - return false; - } - if (lr_node_is(left, "name") && lr_node_text_is(src, left, "__DIR__")) { - base_suffix = "bootstrap"; - } else if (lr_is_dirname_dir(src, left)) { - base_suffix = ""; - } else { - return false; - } - } else if (lr_call_literal_arg(src, expr, "base_path", lit, sizeof(lit))) { - base_suffix = ""; - } else { - return false; - } - int n = snprintf(out, out_sz, "%s/%s/%s", app_dir, base_suffix, lit); - return n > 0 && (size_t)n < out_sz && lr_normalize(out); -} - -/* Does the `api:` value (one path or an array of paths) name rel_path? */ -static bool lr_api_names_file(const char *src, TSNode value, const char *app_dir, - const char *rel_path) { - char resolved[LR_PATH_MAX]; - if (!lr_node_is(value, "array_creation_expression")) { - return lr_route_file_path(src, value, app_dir, resolved, sizeof(resolved)) && - strcmp(resolved, rel_path) == 0; - } - uint32_t nc = ts_node_named_child_count(value); - for (uint32_t i = 0; i < nc; i++) { - TSNode elem = ts_node_named_child(value, i); - uint32_t ec = ts_node_named_child_count(elem); - if (!lr_node_is(elem, "array_element_initializer") || ec == 0) { - continue; - } - TSNode path_expr = ts_node_named_child(elem, ec - SKIP_ONE); /* value of `k => v` too */ - if (lr_route_file_path(src, path_expr, app_dir, resolved, sizeof(resolved)) && - strcmp(resolved, rel_path) == 0) { - return true; - } - } - return false; -} - -/* Evaluate the named arguments of a withRouting(...) call. */ -static bool lr_eval_with_routing(const char *src, TSNode call, const char *app_dir, - const char *rel_path, char *out, size_t out_sz) { - TSNode args = ts_node_child_by_field_name(call, TS_FIELD("arguments")); - char prefix[LR_PREFIX_MAX] = "api"; /* ApplicationBuilder::withRouting default */ - bool prefix_known = true; - bool mounted = false; - uint32_t nc = ts_node_is_null(args) ? 0 : ts_node_named_child_count(args); - for (uint32_t i = 0; i < nc; i++) { - TSNode arg = ts_node_named_child(args, i); - if (lr_node_is(arg, "comment")) { - continue; - } - TSNode label = ts_node_child_by_field_name(arg, TS_FIELD("name")); - char key[LR_KEY_MAX]; - /* Positional arguments would need the parameter order modelled, and a - * `using:` callback replaces the web/api registration entirely. */ - if (!lr_node_is(arg, "argument") || ts_node_is_null(label) || - !lr_node_text(src, label, key, sizeof(key)) || strcmp(key, "using") == 0) { - return false; - } - TSNode value = lr_arg_value(arg, label); - if (strcmp(key, "api") == 0) { - mounted = lr_api_names_file(src, value, app_dir, rel_path); - } else if (strcmp(key, "apiPrefix") == 0) { - prefix_known = lr_string_literal(src, value, prefix, sizeof(prefix)); - } - } - if (!mounted || !prefix_known) { - return false; - } - /* Laravel trims the group prefix of '/' on both ends. */ - const char *start = prefix; - size_t len = strlen(prefix); - while (*start == '/') { - start++; - len--; - } - while (len > 0 && start[len - SKIP_ONE] == '/') { - len--; - } - int n = snprintf(out, out_sz, "/%.*s", (int)len, start); - return len > 0 && n > 0 && (size_t)n < out_sz; -} - -/* First `->withRouting(...)` call in the tree (iterative pre-order walk). */ -static TSNode lr_find_with_routing(const char *src, TSNode root) { - TSNode found = {0}; - TSTreeCursor cur = ts_tree_cursor_new(root); - bool more = true; - while (more) { - TSNode node = ts_tree_cursor_current_node(&cur); - if ((lr_node_is(node, "member_call_expression") || - lr_node_is(node, "nullsafe_member_call_expression")) && - lr_node_text_is(src, ts_node_child_by_field_name(node, TS_FIELD("name")), - "withRouting")) { - found = node; - break; - } - if (ts_tree_cursor_goto_first_child(&cur)) { - continue; - } - while (!ts_tree_cursor_goto_next_sibling(&cur)) { - if (!ts_tree_cursor_goto_parent(&cur)) { - more = false; - break; - } - } - } - ts_tree_cursor_delete(&cur); - return found; -} - -static bool lr_mount_from_bootstrap(const char *src, uint32_t len, const char *app_dir, - const char *rel_path, char *out, size_t out_sz) { - const TSLanguage *php = cbm_ts_language(CBM_LANG_PHP); - TSParser *parser = php ? ts_parser_new() : NULL; - if (!parser) { - return false; - } - bool mounted = false; - TSTree *tree = - ts_parser_set_language(parser, php) ? ts_parser_parse_string(parser, NULL, src, len) : NULL; - if (tree) { - TSNode call = lr_find_with_routing(src, ts_tree_root_node(tree)); - mounted = !ts_node_is_null(call) && - lr_eval_with_routing(src, call, app_dir, rel_path, out, out_sz); - ts_tree_delete(tree); - } - ts_parser_delete(parser); - return mounted; -} - -/* Read //bootstrap/app.php. NULL when absent or unreadable. */ -static char *lr_read_bootstrap(const char *repo_path, const char *app_dir, uint32_t *out_len) { - char path[CBM_SZ_4K]; - int n = snprintf(path, sizeof(path), "%s/%s%s%s", repo_path, app_dir, app_dir[0] ? "/" : "", - LR_BOOTSTRAP_REL); - if (n <= 0 || (size_t)n >= sizeof(path)) { - return NULL; - } - FILE *f = cbm_fopen(path, "rb"); - if (!f) { - return NULL; - } - (void)fseek(f, 0, SEEK_END); - long size = ftell(f); - (void)fseek(f, 0, SEEK_SET); - char *buf = (size > 0 && size <= (long)LR_SOURCE_MAX) - ? malloc((size_t)size + SKIP_ONE) - : NULL; - if (buf) { - size_t nread = fread(buf, SKIP_ONE, (size_t)size, f); - buf[nread] = '\0'; - *out_len = (uint32_t)nread; - } - (void)fclose(f); - return buf; -} - -bool cbm_laravel_api_mount(const char *repo_path, const char *rel_path, char *out, size_t out_sz) { - if (!out || out_sz == 0) { - return false; - } - out[0] = '\0'; - size_t rel_len = rel_path ? strlen(rel_path) : 0; - if (!repo_path || rel_len < sizeof(".php") || rel_len >= LR_PATH_MAX || - strcmp(rel_path + rel_len - (sizeof(".php") - SKIP_ONE), ".php") != 0) { - return false; - } - /* Walk the file's ancestor directories nearest-first; the nearest one - * holding bootstrap/app.php is the Laravel app that owns the file. */ - char app_dir[LR_PATH_MAX]; - memcpy(app_dir, rel_path, rel_len + SKIP_ONE); - char *slash = strrchr(app_dir, '/'); - for (;;) { - if (slash) { - *slash = '\0'; - } else { - app_dir[0] = '\0'; - } - uint32_t len = 0; - char *src = lr_read_bootstrap(repo_path, app_dir, &len); - if (src) { - bool mounted = lr_mount_from_bootstrap(src, len, app_dir, rel_path, out, out_sz); - free(src); - if (!mounted) { - out[0] = '\0'; - } - return mounted; - } - if (!app_dir[0]) { - return false; - } - slash = strrchr(app_dir, '/'); - } -} - -void cbm_laravel_file_route_mount(const char *repo_path, const char *rel_path, CBMLanguage lang, - const CBMFileResult *result, char *out, size_t out_sz) { - if (!out || out_sz == 0) { - return; - } - out[0] = '\0'; - if (lang != CBM_LANG_PHP || !result) { - return; - } - for (int i = 0; i < result->calls.count; i++) { - const CBMCall *call = &result->calls.items[i]; - if (call->callee_name && call->first_string_arg && call->first_string_arg[0] == '/' && - cbm_service_pattern_route_method(call->callee_name) != NULL) { - (void)cbm_laravel_api_mount(repo_path, rel_path, out, out_sz); - return; - } - } -} - -const char *cbm_laravel_mount_route(const char *mount, const char *path, char *buf, size_t buf_sz) { - if (!mount || !mount[0] || !path || !buf || buf_sz == 0) { - return path; - } - const char *rel = path; - while (*rel == '/') { - rel++; - } - int n = - rel[0] ? snprintf(buf, buf_sz, "%s/%s", mount, rel) : snprintf(buf, buf_sz, "%s", mount); - return (n > 0 && (size_t)n < buf_sz) ? buf : path; -} diff --git a/src/pipeline/laravel_routing.zig b/src/pipeline/laravel_routing.zig new file mode 100644 index 00000000..2c8fc9f1 --- /dev/null +++ b/src/pipeline/laravel_routing.zig @@ -0,0 +1,379 @@ +//! laravel_routing.zig — Laravel 11+ framework route mounts (#1146). +//! +//! See laravel_routing.h for the contract. bootstrap/app.php is parsed with +//! the same PHP grammar the extractor uses; only named `withRouting(...)` +//! arguments whose values are plain literals (or `__DIR__`/`dirname(__DIR__)` +//! concatenations and `base_path()` calls over literals) are trusted. + +const std = @import("std"); +const c = @import("c"); +const util = @import("util.zig"); + +const CStr = [*c]const u8; +const MutStr = [*c]u8; +const TSNode = c.TSNode; + +const LR_PATH_MAX = c.CBM_SZ_512; // repo-relative path buffer +const LR_KEY_MAX = c.CBM_SZ_32; // named-argument key +const LR_PREFIX_MAX = c.CBM_SZ_128; // apiPrefix literal +const LR_SEG_MAX = c.CBM_SZ_64; // path segments tracked while normalising +const LR_SOURCE_MAX = c.CBM_SZ_256 * c.CBM_SZ_1K; // bootstrap/app.php read cap + +const LR_BOOTSTRAP_REL = "bootstrap/app.php"; + +/// ts_node_child_by_field_name with a literal field name (TS_FIELD in C). +fn field(node: TSNode, comptime name: [:0]const u8) TSNode { + return c.ts_node_child_by_field_name(node, name, name.len); +} + +/// Copy a node's source text into buf. False when it does not fit. +fn lrNodeText(src: [*]const u8, node: TSNode, buf: []u8) bool { + if (c.ts_node_is_null(node)) return false; + const start = c.ts_node_start_byte(node); + const end = c.ts_node_end_byte(node); + const len: usize = if (end > start) end - start else 0; + if (len >= buf.len) return false; + @memcpy(buf[0..len], src[start..][0..len]); + buf[len] = 0; + return true; +} + +fn lrNodeIs(node: TSNode, node_type: [*:0]const u8) bool { + return !c.ts_node_is_null(node) and c.strcmp(c.ts_node_type(node), node_type) == 0; +} + +fn lrNodeTextIs(src: [*]const u8, node: TSNode, text: [*:0]const u8) bool { + var buf: [LR_KEY_MAX]u8 = undefined; + return lrNodeText(src, node, &buf) and c.strcmp(&buf, text) == 0; +} + +/// A plain string literal: '...' or "..." whose only children are +/// string_content (no interpolation, no escapes). Writes the unquoted value. +fn lrStringLiteral(src: [*]const u8, node: TSNode, buf: []u8) bool { + if (!lrNodeIs(node, "string") and !lrNodeIs(node, "encapsed_string")) return false; + var pos: usize = 0; + buf[0] = 0; + const nc = c.ts_node_named_child_count(node); + var i: u32 = 0; + while (i < nc) : (i += 1) { + const part = c.ts_node_named_child(node, i); + if (!lrNodeIs(part, "string_content") or !lrNodeText(src, part, buf[pos..])) { + return false; + } + pos += c.strlen(buf[pos..].ptr); + } + return true; +} + +/// The value expression of a call argument: its first named child that is +/// neither the `name:` label nor a comment. +fn lrArgValue(arg: TSNode, label: TSNode) TSNode { + const nc = c.ts_node_named_child_count(arg); + var i: u32 = 0; + while (i < nc) : (i += 1) { + const child = c.ts_node_named_child(arg, i); + if ((c.ts_node_is_null(label) or !c.ts_node_eq(child, label)) and + !lrNodeIs(child, "comment")) + { + return child; + } + } + return std.mem.zeroes(TSNode); +} + +/// The single literal argument of `fn('...')` when the callee is `fn`. +fn lrCallLiteralArg(src: [*]const u8, call: TSNode, func: [*:0]const u8, buf: []u8) bool { + if (!lrNodeIs(call, "function_call_expression") or + !lrNodeTextIs(src, field(call, "function"), func)) + { + return false; + } + const args = field(call, "arguments"); + if (c.ts_node_is_null(args) or c.ts_node_named_child_count(args) != 1) return false; + const arg = c.ts_node_named_child(args, 0); + const val = lrArgValue(arg, field(arg, "name")); + return lrStringLiteral(src, val, buf); +} + +/// `dirname(__DIR__)` — the application base directory. +fn lrIsDirnameDir(src: [*]const u8, node: TSNode) bool { + if (!lrNodeIs(node, "function_call_expression") or + !lrNodeTextIs(src, field(node, "function"), "dirname")) + { + return false; + } + const args = field(node, "arguments"); + if (c.ts_node_is_null(args) or c.ts_node_named_child_count(args) != 1) return false; + const arg = c.ts_node_named_child(args, 0); + const val = lrArgValue(arg, field(arg, "name")); + return lrNodeIs(val, "name") and lrNodeTextIs(src, val, "__DIR__"); +} + +/// Resolve "." and ".." segments of a '/'-joined path in place. False when the +/// path climbs above the repository root. +fn lrNormalize(path: [*:0]u8) bool { + var seg_start: [LR_SEG_MAX]usize = undefined; + var depth: usize = 0; + var out: usize = 0; + var p: [*:0]const u8 = path; + while (p[0] != 0) { + const slash: CStr = c.strchr(p, '/'); + const len: usize = if (slash != null) @intFromPtr(slash) - @intFromPtr(p) else c.strlen(p); + if (len == 2 and p[0] == '.' and p[1] == '.') { + if (depth == 0) return false; + depth -= 1; + out = seg_start[depth]; + } else if (len > 0 and !(len == 1 and p[0] == '.')) { + if (depth == LR_SEG_MAX) return false; + const at: usize = if (out > 0) out + 1 else 0; + seg_start[depth] = out; + depth += 1; + if (out > 0) path[out] = '/'; + _ = c.memmove(path + at, p, len); + out = at + len; + } + p += len + @as(usize, if (slash != null) 1 else 0); + } + path[out] = 0; + return true; +} + +/// Resolve one `api:` path expression of the app rooted at app_dir to a +/// normalised repo-relative path: +/// __DIR__ . '/../routes/api.php' (relative to bootstrap/) +/// dirname(__DIR__) . '/routes/api.php' (relative to the app base) +/// base_path('routes/api.php') (relative to the app base) +fn lrRouteFilePath(src: [*]const u8, expr: TSNode, app_dir: CStr, out: []u8) bool { + var lit: [LR_PATH_MAX]u8 = undefined; + var base_suffix: CStr = null; + if (lrNodeIs(expr, "binary_expression")) { + const op = field(expr, "operator"); + const left = field(expr, "left"); + if (!lrNodeTextIs(src, op, ".") or !lrStringLiteral(src, field(expr, "right"), &lit)) { + return false; + } + if (lrNodeIs(left, "name") and lrNodeTextIs(src, left, "__DIR__")) { + base_suffix = "bootstrap"; + } else if (lrIsDirnameDir(src, left)) { + base_suffix = ""; + } else { + return false; + } + } else if (lrCallLiteralArg(src, expr, "base_path", &lit)) { + base_suffix = ""; + } else { + return false; + } + const n = c.snprintf(out.ptr, out.len, "%s/%s/%s", app_dir, base_suffix, @as(CStr, &lit)); + return n > 0 and @as(usize, @intCast(n)) < out.len and lrNormalize(@ptrCast(out.ptr)); +} + +/// Does the `api:` value (one path or an array of paths) name rel_path? +fn lrApiNamesFile(src: [*]const u8, value: TSNode, app_dir: CStr, rel_path: CStr) bool { + var resolved: [LR_PATH_MAX]u8 = undefined; + if (!lrNodeIs(value, "array_creation_expression")) { + return lrRouteFilePath(src, value, app_dir, &resolved) and + c.strcmp(&resolved, rel_path) == 0; + } + const nc = c.ts_node_named_child_count(value); + var i: u32 = 0; + while (i < nc) : (i += 1) { + const elem = c.ts_node_named_child(value, i); + const ec = c.ts_node_named_child_count(elem); + if (!lrNodeIs(elem, "array_element_initializer") or ec == 0) continue; + const path_expr = c.ts_node_named_child(elem, ec - 1); // value of `k => v` too + if (lrRouteFilePath(src, path_expr, app_dir, &resolved) and + c.strcmp(&resolved, rel_path) == 0) + { + return true; + } + } + return false; +} + +/// Evaluate the named arguments of a withRouting(...) call. +fn lrEvalWithRouting(src: [*]const u8, call: TSNode, app_dir: CStr, rel_path: CStr, out: []u8) bool { + const args = field(call, "arguments"); + // ApplicationBuilder::withRouting default + var prefix: [LR_PREFIX_MAX]u8 = @splat(0); + prefix[0..3].* = "api".*; + var prefix_known = true; + var mounted = false; + const nc: u32 = if (c.ts_node_is_null(args)) 0 else c.ts_node_named_child_count(args); + var i: u32 = 0; + while (i < nc) : (i += 1) { + const arg = c.ts_node_named_child(args, i); + if (lrNodeIs(arg, "comment")) continue; + const label = field(arg, "name"); + var key: [LR_KEY_MAX]u8 = undefined; + // Positional arguments would need the parameter order modelled, and a + // `using:` callback replaces the web/api registration entirely. + if (!lrNodeIs(arg, "argument") or c.ts_node_is_null(label) or + !lrNodeText(src, label, &key) or c.strcmp(&key, "using") == 0) + { + return false; + } + const value = lrArgValue(arg, label); + if (c.strcmp(&key, "api") == 0) { + mounted = lrApiNamesFile(src, value, app_dir, rel_path); + } else if (c.strcmp(&key, "apiPrefix") == 0) { + prefix_known = lrStringLiteral(src, value, &prefix); + } + } + if (!mounted or !prefix_known) return false; + // Laravel trims the group prefix of '/' on both ends. + var start: [*]const u8 = &prefix; + var len = c.strlen(&prefix); + while (start[0] == '/') { + start += 1; + len -= 1; + } + while (len > 0 and start[len - 1] == '/') len -= 1; + const n = c.snprintf(out.ptr, out.len, "/%.*s", @as(c_int, @intCast(len)), start); + return len > 0 and n > 0 and @as(usize, @intCast(n)) < out.len; +} + +/// First `->withRouting(...)` call in the tree (iterative pre-order walk). +fn lrFindWithRouting(src: [*]const u8, root: TSNode) TSNode { + var found = std.mem.zeroes(TSNode); + var cur = c.ts_tree_cursor_new(root); + var more = true; + while (more) { + const node = c.ts_tree_cursor_current_node(&cur); + if ((lrNodeIs(node, "member_call_expression") or + lrNodeIs(node, "nullsafe_member_call_expression")) and + lrNodeTextIs(src, field(node, "name"), "withRouting")) + { + found = node; + break; + } + if (c.ts_tree_cursor_goto_first_child(&cur)) continue; + while (!c.ts_tree_cursor_goto_next_sibling(&cur)) { + if (!c.ts_tree_cursor_goto_parent(&cur)) { + more = false; + break; + } + } + } + c.ts_tree_cursor_delete(&cur); + return found; +} + +fn lrMountFromBootstrap(src: [*]const u8, len: u32, app_dir: CStr, rel_path: CStr, out: []u8) bool { + const php = c.cbm_ts_language(c.CBM_LANG_PHP); + const parser = if (php != null) c.ts_parser_new() else null; + if (parser == null) return false; + var mounted = false; + const tree = if (c.ts_parser_set_language(parser, php)) + c.ts_parser_parse_string(parser, null, src, len) + else + null; + if (tree != null) { + const call = lrFindWithRouting(src, c.ts_tree_root_node(tree)); + mounted = !c.ts_node_is_null(call) and + lrEvalWithRouting(src, call, app_dir, rel_path, out); + c.ts_tree_delete(tree); + } + c.ts_parser_delete(parser); + return mounted; +} + +/// Read //bootstrap/app.php. NULL when absent or unreadable. +fn lrReadBootstrap(repo_path: CStr, app_dir: [*:0]const u8, out_len: *u32) MutStr { + var path: [c.CBM_SZ_4K]u8 = undefined; + const n = c.snprintf( + &path, + path.len, + "%s/%s%s%s", + repo_path, + app_dir, + @as(CStr, if (app_dir[0] != 0) "/" else ""), + @as(CStr, LR_BOOTSTRAP_REL), + ); + if (n <= 0 or @as(usize, @intCast(n)) >= path.len) return null; + const f = c.cbm_fopen(&path, "rb") orelse return null; + _ = c.fseek(f, 0, c.SEEK_END); + const size = c.ftell(f); + _ = c.fseek(f, 0, c.SEEK_SET); + const buf: MutStr = if (size > 0 and size <= LR_SOURCE_MAX) + @ptrCast(c.malloc(@as(usize, @intCast(size)) + 1)) + else + null; + if (buf != null) { + const nread = c.fread(buf, 1, @intCast(size), f); + buf[nread] = 0; + out_len.* = @intCast(nread); + } + _ = c.fclose(f); + return buf; +} + +pub export fn cbm_laravel_api_mount(repo_path: CStr, rel_path: CStr, out_c: MutStr, out_sz: usize) bool { + if (out_c == null or out_sz == 0) return false; + const out = out_c[0..out_sz]; + out[0] = 0; + const rel_len: usize = if (rel_path != null) c.strlen(rel_path) else 0; + if (repo_path == null or rel_len < ".php".len + 1 or rel_len >= LR_PATH_MAX or + c.strcmp(rel_path + rel_len - ".php".len, ".php") != 0) + { + return false; + } + // Walk the file's ancestor directories nearest-first; the nearest one + // holding bootstrap/app.php is the Laravel app that owns the file. + var app_dir: [LR_PATH_MAX:0]u8 = undefined; + @memcpy(app_dir[0 .. rel_len + 1], rel_path[0 .. rel_len + 1]); + var slash: MutStr = c.strrchr(&app_dir, '/'); + while (true) { + if (slash != null) { + slash[0] = 0; + } else { + app_dir[0] = 0; + } + var len: u32 = 0; + const src = lrReadBootstrap(repo_path, &app_dir, &len); + if (src != null) { + const mounted = lrMountFromBootstrap(src, len, &app_dir, rel_path, out); + c.free(src); + if (!mounted) out[0] = 0; + return mounted; + } + if (app_dir[0] == 0) return false; + slash = c.strrchr(&app_dir, '/'); + } +} + +pub export fn cbm_laravel_file_route_mount( + repo_path: CStr, + rel_path: CStr, + lang: c.CBMLanguage, + result_c: ?*const c.CBMFileResult, + out: MutStr, + out_sz: usize, +) void { + if (out == null or out_sz == 0) return; + out[0] = 0; + const result = result_c orelse return; + if (lang != c.CBM_LANG_PHP) return; + var i: usize = 0; + while (i < result.calls.count) : (i += 1) { + const call = util.at(result.calls.items, i); + if (call.callee_name != null and call.first_string_arg != null and + call.first_string_arg[0] == '/' and + c.cbm_service_pattern_route_method(call.callee_name) != null) + { + _ = cbm_laravel_api_mount(repo_path, rel_path, out, out_sz); + return; + } + } +} + +pub export fn cbm_laravel_mount_route(mount: CStr, path: CStr, buf: MutStr, buf_sz: usize) CStr { + if (mount == null or mount[0] == 0 or path == null or buf == null or buf_sz == 0) return path; + var rel: [*]const u8 = path; + while (rel[0] == '/') rel += 1; + const n = if (rel[0] != 0) + c.snprintf(buf, buf_sz, "%s/%s", mount, rel) + else + c.snprintf(buf, buf_sz, "%s", mount); + return if (n > 0 and @as(usize, @intCast(n)) < buf_sz) buf else path; +} diff --git a/src/pipeline/lsp_resolve_shim.c b/src/pipeline/lsp_resolve_shim.c new file mode 100644 index 00000000..b8538404 --- /dev/null +++ b/src/pipeline/lsp_resolve_shim.c @@ -0,0 +1,21 @@ +/* + * lsp_resolve_shim.c — cbm_pipeline_lsp_target_node as a linkable function. + * + * lsp_resolve.h defines its helpers as static inline, and the Zig modules + * read them through translate-c. This one counts its tail-match lookups with + * C11 atomics, which translate-c cannot express, so the Zig side sees only its + * declaration. This file gives that declaration a definition by compiling the + * header's own body, so lsp_resolve.h stays the single source of the logic + * for the C code that includes it (internal/cbm/lsp, the tests). + */ +#define cbm_pipeline_lsp_target_node cbm_pipeline_lsp_target_node_inline +#include "pipeline/lsp_resolve.h" +#undef cbm_pipeline_lsp_target_node + +const cbm_gbuf_node_t *cbm_pipeline_lsp_target_node(const cbm_gbuf_t *gbuf, const char *project, + const char *callee_qn, bool allow_tail_match); + +const cbm_gbuf_node_t *cbm_pipeline_lsp_target_node(const cbm_gbuf_t *gbuf, const char *project, + const char *callee_qn, bool allow_tail_match) { + return cbm_pipeline_lsp_target_node_inline(gbuf, project, callee_qn, allow_tail_match); +} diff --git a/src/pipeline/pass_compile_commands.c b/src/pipeline/pass_compile_commands.c deleted file mode 100644 index 17498397..00000000 --- a/src/pipeline/pass_compile_commands.c +++ /dev/null @@ -1,285 +0,0 @@ -/* - * pass_compile_commands.c — compile_commands.json parsing helpers. - * - * Parses compile_commands.json to extract per-file include paths, defines, - * and C/C++ standard flags. - */ -#include "foundation/constants.h" - -enum { CC_FLAG_IDX = 1, CC_FLAG_SKIP = 2 }; - -#define SLEN(s) (sizeof(s) - 1) -#include "pipeline/pipeline.h" -#include "pipeline/pipeline_internal.h" - -#include -#include -#include -#include "yyjson/yyjson.h" - -/* Emit the current token if non-empty. Returns updated count. */ -static int emit_token(char *current, int *clen, char **out, int count, int max_out) { - if (*clen > 0 && count < max_out) { - current[*clen] = '\0'; - out[count++] = strdup(current); - *clen = 0; - } - return count; -} - -int cbm_split_command(const char *cmd, char **out, int max_out) { - if (!cmd || !out || max_out <= 0) { - return 0; - } - - int count = 0; - char current[CBM_SZ_4K]; - int clen = 0; - char in_quote = 0; - - for (int i = 0; cmd[i]; i++) { - char c = cmd[i]; - if (in_quote) { - if (c == in_quote) { - in_quote = 0; - } else if (clen < (int)sizeof(current) - SKIP_ONE) { - current[clen++] = c; - } - } else if (c == '"' || c == '\'') { - in_quote = c; - } else if (c == ' ' || c == '\t') { - count = emit_token(current, &clen, out, count, max_out); - } else if (clen < (int)sizeof(current) - SKIP_ONE) { - current[clen++] = c; - } - } - return emit_token(current, &clen, out, count, max_out); -} - -/* Resolve a path: if relative, join with directory. */ -static char *resolve_path(const char *path, const char *directory) { - if (!path) { - return NULL; - } - - /* Absolute path */ - if (path[0] == '/') { - return strdup(path); - } - - /* Relative — join with directory */ - if (directory && directory[0]) { - char buf[CBM_SZ_4K]; - snprintf(buf, sizeof(buf), "%s/%s", directory, path); - return strdup(buf); - } - - return strdup(path); -} - -/* Try to consume a -I or -isystem include path flag. Returns true if consumed. */ -static bool try_include_flag(cbm_compile_flags_t *f, const char **args, int argc, int *i, - const char *directory) { - const char *arg = args[*i]; - if (arg[0] == '-' && arg[CC_FLAG_IDX] == 'I') { - const char *path = arg + CC_FLAG_SKIP; - if (*path == '\0' && *i + SKIP_ONE < argc) { - (*i)++; - path = args[*i]; - } - if (path && *path) { - f->include_paths[f->include_count++] = resolve_path(path, directory); - } - return true; - } - if (strcmp(arg, "-isystem") == 0 && *i + SKIP_ONE < argc) { - (*i)++; - f->include_paths[f->include_count++] = resolve_path(args[*i], directory); - return true; - } - return false; -} - -/* Try to consume a -D define flag. Returns true if consumed. */ -static bool try_define_flag(cbm_compile_flags_t *f, const char **args, int argc, int *i) { - const char *arg = args[*i]; - if (arg[0] != '-' || arg[CC_FLAG_IDX] != 'D') { - return false; - } - const char *define = arg + CC_FLAG_SKIP; - if (*define == '\0' && *i + SKIP_ONE < argc) { - (*i)++; - define = args[*i]; - } - if (define && *define) { - f->defines[f->define_count++] = strdup(define); - } - return true; -} - -cbm_compile_flags_t *cbm_extract_flags(const char **args, int argc, const char *directory) { - cbm_compile_flags_t *f = calloc(CBM_ALLOC_ONE, sizeof(*f)); - if (!f) { - return NULL; - } - f->include_paths = calloc(argc, sizeof(char *)); - f->defines = calloc(argc, sizeof(char *)); - - for (int i = 0; i < argc; i++) { - if (try_include_flag(f, args, argc, &i, directory)) { - continue; - } - if (try_define_flag(f, args, argc, &i)) { - continue; - } - if (strncmp(args[i], "-std=", SLEN("-std=")) == 0) { - snprintf(f->standard, sizeof(f->standard), "%s", args[i] + 5); - } - } - return f; -} - -void cbm_compile_flags_free(cbm_compile_flags_t *f) { - if (!f) { - return; - } - for (int i = 0; i < f->include_count; i++) { - free(f->include_paths[i]); - } - free(f->include_paths); - for (int i = 0; i < f->define_count; i++) { - free(f->defines[i]); - } - free(f->defines); - free(f); -} - -/* Extract compiler flag args from either "arguments" array or "command" string. */ -static int extract_flag_args(yyjson_val *args_val, yyjson_val *cmd_val, const char **flag_args, - char **split_args, int max_args) { - if (args_val && yyjson_is_arr(args_val)) { - int n = 0; - yyjson_val *a; - yyjson_arr_iter aiter; - yyjson_arr_iter_init(args_val, &aiter); - while ((a = yyjson_arr_iter_next(&aiter)) && n < max_args) { - const char *s = yyjson_get_str(a); - if (s) { - flag_args[n++] = s; - } - } - return n; - } - if (cmd_val && yyjson_is_str(cmd_val)) { - int n = cbm_split_command(yyjson_get_str(cmd_val), split_args, max_args); - for (int j = 0; j < n; j++) { - flag_args[j] = split_args[j]; - } - return n; - } - return 0; -} - -/* Process a single compile_commands.json entry. Returns 1 if added, 0 otherwise. */ -static int process_compile_entry(yyjson_val *entry, const char *repo_path, char **out_path, - cbm_compile_flags_t **out_flag) { - yyjson_val *dir_val = yyjson_obj_get(entry, "directory"); - yyjson_val *file_val = yyjson_obj_get(entry, "file"); - yyjson_val *cmd_val = yyjson_obj_get(entry, "command"); - yyjson_val *args_val = yyjson_obj_get(entry, "arguments"); - - if (!file_val) { - return 0; - } - const char *directory = dir_val ? yyjson_get_str(dir_val) : ""; - const char *file_path = yyjson_get_str(file_val); - if (!file_path) { - return 0; - } - - char *split_args[CBM_SZ_256] = {NULL}; - const char *flag_args[CBM_SZ_256]; - int flag_argc = extract_flag_args(args_val, cmd_val, flag_args, split_args, CBM_SZ_256); - - if (flag_argc == 0) { - return 0; - } - - cbm_compile_flags_t *f = cbm_extract_flags(flag_args, flag_argc, directory); - - if (cmd_val && yyjson_is_str(cmd_val)) { - for (int j = 0; j < flag_argc; j++) { - free(split_args[j]); - } - } - - if (!f) { - return 0; - } - - char abs_path[CBM_SZ_4K]; - if (file_path[0] != '/' && directory && directory[0]) { - snprintf(abs_path, sizeof(abs_path), "%s/%s", directory, file_path); - } else { - snprintf(abs_path, sizeof(abs_path), "%s", file_path); - } - - size_t repo_len = strlen(repo_path); - if (strncmp(abs_path, repo_path, repo_len) != 0 || abs_path[repo_len] != '/') { - cbm_compile_flags_free(f); - return 0; - } - - *out_path = strdup(abs_path + repo_len + SKIP_ONE); - *out_flag = f; - return SKIP_ONE; -} - -int cbm_parse_compile_commands(const char *json_data, const char *repo_path, char ***out_paths, - cbm_compile_flags_t ***out_flags) { - if (!json_data || !repo_path || !out_paths || !out_flags) { - return CBM_NOT_FOUND; - } - *out_paths = NULL; - *out_flags = NULL; - - yyjson_doc *doc = yyjson_read(json_data, strlen(json_data), 0); - if (!doc) { - return CBM_NOT_FOUND; - } - - yyjson_val *root = yyjson_doc_get_root(doc); - if (!yyjson_is_arr(root)) { - yyjson_doc_free(doc); - return CBM_NOT_FOUND; - } - - int arr_len = (int)yyjson_arr_size(root); - if (arr_len == 0) { - yyjson_doc_free(doc); - return 0; - } - - char **paths = calloc(arr_len, sizeof(char *)); - cbm_compile_flags_t **flags = calloc(arr_len, sizeof(cbm_compile_flags_t *)); - int count = 0; - - yyjson_val *entry; - yyjson_arr_iter iter; - yyjson_arr_iter_init(root, &iter); - - while ((entry = yyjson_arr_iter_next(&iter))) { - char *p = NULL; - cbm_compile_flags_t *f = NULL; - if (process_compile_entry(entry, repo_path, &p, &f)) { - paths[count] = p; - flags[count] = f; - count++; - } - } - - yyjson_doc_free(doc); - *out_paths = paths; - *out_flags = flags; - return count; -} diff --git a/src/pipeline/pass_compile_commands.zig b/src/pipeline/pass_compile_commands.zig new file mode 100644 index 00000000..0e8fd14a --- /dev/null +++ b/src/pipeline/pass_compile_commands.zig @@ -0,0 +1,277 @@ +//! pass_compile_commands.zig — compile_commands.json parsing helpers. +//! +//! Parses compile_commands.json to extract per-file include paths, defines, +//! and C/C++ standard flags. + +const std = @import("std"); +const c = @import("c"); +const interop = @import("../foundation/interop.zig"); +const util = @import("util.zig"); + +const CStr = util.CStr; +const MutStr = util.MutStr; +const ix = util.ix; +const CompileFlags = c.cbm_compile_flags_t; + +const CC_FLAG_IDX = 1; +const CC_FLAG_SKIP = 2; + +/// Emit the current token if non-empty. Returns updated count. +fn emitToken(current: *[c.CBM_SZ_4K]u8, clen: *usize, out: [*]MutStr, count_in: c_int, max_out: c_int) c_int { + var count = count_in; + if (clen.* > 0 and count < max_out) { + current[clen.*] = 0; + out[ix(count)] = c.strdup(current); + count += 1; + clen.* = 0; + } + return count; +} + +pub export fn cbm_split_command(cmd: CStr, out_c: [*c]MutStr, max_out: c_int) c_int { + if (cmd == null or out_c == null or max_out <= 0) return 0; + const out: [*]MutStr = out_c; + + var count: c_int = 0; + var current: [c.CBM_SZ_4K]u8 = undefined; + var clen: usize = 0; + var in_quote: u8 = 0; + + var i: usize = 0; + while (cmd[i] != 0) : (i += 1) { + const ch = cmd[i]; + if (in_quote != 0) { + if (ch == in_quote) { + in_quote = 0; + } else if (clen < current.len - 1) { + current[clen] = ch; + clen += 1; + } + } else if (ch == '"' or ch == '\'') { + in_quote = ch; + } else if (ch == ' ' or ch == '\t') { + count = emitToken(¤t, &clen, out, count, max_out); + } else if (clen < current.len - 1) { + current[clen] = ch; + clen += 1; + } + } + return emitToken(¤t, &clen, out, count, max_out); +} + +/// Resolve a path: if relative, join with directory. +fn resolvePath(path: CStr, directory: CStr) MutStr { + if (path == null) return null; + + // Absolute path + if (path[0] == '/') return c.strdup(path); + + // Relative — join with directory + if (directory != null and directory[0] != 0) { + var buf: [c.CBM_SZ_4K]u8 = undefined; + _ = c.snprintf(&buf, buf.len, "%s/%s", directory, path); + return c.strdup(&buf); + } + + return c.strdup(path); +} + +/// Try to consume a -I or -isystem include path flag. Returns true if consumed. +fn tryIncludeFlag(f: *CompileFlags, args: [*]const CStr, argc: c_int, i: *c_int, directory: CStr) bool { + const arg = args[ix(i.*)]; + if (arg[0] == '-' and arg[CC_FLAG_IDX] == 'I') { + var path: CStr = arg + CC_FLAG_SKIP; + if (path[0] == 0 and i.* + 1 < argc) { + i.* += 1; + path = args[ix(i.*)]; + } + if (path != null and path[0] != 0) { + f.include_paths[ix(f.include_count)] = resolvePath(path, directory); + f.include_count += 1; + } + return true; + } + if (util.streq(arg, "-isystem") and i.* + 1 < argc) { + i.* += 1; + f.include_paths[ix(f.include_count)] = resolvePath(args[ix(i.*)], directory); + f.include_count += 1; + return true; + } + return false; +} + +/// Try to consume a -D define flag. Returns true if consumed. +fn tryDefineFlag(f: *CompileFlags, args: [*]const CStr, argc: c_int, i: *c_int) bool { + const arg = args[ix(i.*)]; + if (arg[0] != '-' or arg[CC_FLAG_IDX] != 'D') return false; + var define: CStr = arg + CC_FLAG_SKIP; + if (define[0] == 0 and i.* + 1 < argc) { + i.* += 1; + define = args[ix(i.*)]; + } + if (define != null and define[0] != 0) { + f.defines[ix(f.define_count)] = c.strdup(define); + f.define_count += 1; + } + return true; +} + +pub export fn cbm_extract_flags(args: [*c]CStr, argc: c_int, directory: CStr) ?*CompileFlags { + const f: *CompileFlags = @as(?*CompileFlags, interop.callocN(CompileFlags, 1)) orelse return null; + // (size_t)argc, as the C calloc(argc, ...) converted it. + const slots: usize = @bitCast(@as(isize, argc)); + f.include_paths = interop.callocN(MutStr, slots); + f.defines = interop.callocN(MutStr, slots); + + var i: c_int = 0; + while (i < argc) : (i += 1) { + if (tryIncludeFlag(f, args, argc, &i, directory)) continue; + if (tryDefineFlag(f, args, argc, &i)) continue; + if (c.strncmp(args[ix(i)], "-std=", "-std=".len) == 0) { + _ = c.snprintf(&f.standard, f.standard.len, "%s", args[ix(i)] + 5); + } + } + return f; +} + +pub export fn cbm_compile_flags_free(f_c: ?*CompileFlags) void { + const f = f_c orelse return; + var i: usize = 0; + while (i < f.include_count) : (i += 1) c.free(f.include_paths[i]); + c.free(@ptrCast(f.include_paths)); + i = 0; + while (i < f.define_count) : (i += 1) c.free(f.defines[i]); + c.free(@ptrCast(f.defines)); + c.free(f); +} + +/// Extract compiler flag args from either "arguments" array or "command" string. +fn extractFlagArgs( + args_val: ?*c.yyjson_val, + cmd_val: ?*c.yyjson_val, + flag_args: *[c.CBM_SZ_256]CStr, + split_args: *[c.CBM_SZ_256]MutStr, +) c_int { + @setEvalBranchQuota(10000); + const max_args: c_int = c.CBM_SZ_256; + if (args_val != null and c.yyjson_is_arr(args_val)) { + var n: c_int = 0; + var aiter: c.yyjson_arr_iter = undefined; + _ = c.yyjson_arr_iter_init(args_val, &aiter); + while (true) { + const a = c.yyjson_arr_iter_next(&aiter); + if (a == null or n >= max_args) break; + const s: CStr = c.yyjson_get_str(a); + if (s != null) { + flag_args[ix(n)] = s; + n += 1; + } + } + return n; + } + if (cmd_val != null and c.yyjson_is_str(cmd_val)) { + const n = cbm_split_command(c.yyjson_get_str(cmd_val), split_args, max_args); + for (0..ix(n)) |j| flag_args[j] = split_args[j]; + return n; + } + return 0; +} + +/// Process a single compile_commands.json entry. Returns true if added. +fn processCompileEntry(entry: ?*c.yyjson_val, repo_path: CStr, out_path: *MutStr, out_flag: *?*CompileFlags) bool { + @setEvalBranchQuota(10000); + const dir_val = c.yyjson_obj_get(entry, "directory"); + const file_val = c.yyjson_obj_get(entry, "file"); + const cmd_val = c.yyjson_obj_get(entry, "command"); + const args_val = c.yyjson_obj_get(entry, "arguments"); + + if (file_val == null) return false; + const directory: CStr = if (dir_val != null) c.yyjson_get_str(dir_val) else ""; + const file_path: CStr = c.yyjson_get_str(file_val); + if (file_path == null) return false; + + var split_args: [c.CBM_SZ_256]MutStr = @splat(null); + var flag_args: [c.CBM_SZ_256]CStr = undefined; + const flag_argc = extractFlagArgs(args_val, cmd_val, &flag_args, &split_args); + + if (flag_argc == 0) return false; + + const f = cbm_extract_flags(&flag_args, flag_argc, directory); + + if (cmd_val != null and c.yyjson_is_str(cmd_val)) { + for (0..ix(flag_argc)) |j| c.free(split_args[j]); + } + + if (f == null) return false; + + var abs_path: [c.CBM_SZ_4K]u8 = undefined; + if (file_path[0] != '/' and directory != null and directory[0] != 0) { + _ = c.snprintf(&abs_path, abs_path.len, "%s/%s", directory, file_path); + } else { + _ = c.snprintf(&abs_path, abs_path.len, "%s", file_path); + } + + const repo_len = c.strlen(repo_path); + if (c.strncmp(&abs_path, repo_path, repo_len) != 0 or abs_path[repo_len] != '/') { + cbm_compile_flags_free(f); + return false; + } + + out_path.* = c.strdup(@as(CStr, &abs_path) + repo_len + 1); + out_flag.* = f; + return true; +} + +pub export fn cbm_parse_compile_commands( + json_data: CStr, + repo_path: CStr, + out_paths: ?*[*c]MutStr, + out_flags: ?*[*c]?*CompileFlags, +) c_int { + @setEvalBranchQuota(10000); + if (json_data == null or repo_path == null) return c.CBM_NOT_FOUND; + const paths_out = out_paths orelse return c.CBM_NOT_FOUND; + const flags_out = out_flags orelse return c.CBM_NOT_FOUND; + paths_out.* = null; + flags_out.* = null; + + const doc = c.yyjson_read(json_data, c.strlen(json_data), 0); + if (doc == null) return c.CBM_NOT_FOUND; + + const root = c.yyjson_doc_get_root(doc); + if (!c.yyjson_is_arr(root)) { + c.yyjson_doc_free(doc); + return c.CBM_NOT_FOUND; + } + + const arr_len: c_int = @truncate(@as(isize, @bitCast(c.yyjson_arr_size(root)))); + if (arr_len == 0) { + c.yyjson_doc_free(doc); + return 0; + } + + const slots: usize = @bitCast(@as(isize, arr_len)); + const paths = interop.callocN(MutStr, slots); + const flags = interop.callocN(?*CompileFlags, slots); + var count: usize = 0; + + var iter: c.yyjson_arr_iter = undefined; + _ = c.yyjson_arr_iter_init(root, &iter); + + while (true) { + const entry = c.yyjson_arr_iter_next(&iter); + if (entry == null) break; + var p: MutStr = null; + var f: ?*CompileFlags = null; + if (processCompileEntry(entry, repo_path, &p, &f)) { + paths[count] = p; + flags[count] = f; + count += 1; + } + } + + c.yyjson_doc_free(doc); + paths_out.* = paths; + flags_out.* = flags; + return @intCast(count); +} diff --git a/src/pipeline/pass_complexity.c b/src/pipeline/pass_complexity.c deleted file mode 100644 index 33172046..00000000 --- a/src/pipeline/pass_complexity.c +++ /dev/null @@ -1,287 +0,0 @@ -/* - * pass_complexity.c — Interprocedural complexity propagation (Tier B). - * - * Tier A (in the extraction walk) stamps each Function/Method node with local - * structural metrics: complexity (cyclomatic), cognitive, loop_count, loop_depth. - * This pass propagates loop_depth along CALLS edges to estimate a worst-case - * *transitive* nested-loop degree: a function with a depth-1 loop that calls an - * O(n) helper is effectively O(n^2). The estimate assumes calls may occur inside - * loops (an upper bound) — it is a queryable bottleneck *candidate* signal, not a - * proof (true big-O is undecidable; cf. SPEED / Loopus). Cycles in the call graph - * are broken and flagged via a `recursive` property. - * - * Writes two extra node properties: transitive_loop_depth, recursive. - */ -#include "foundation/constants.h" -#include "pipeline/pipeline.h" -#include "pipeline/pipeline_internal.h" -#include "graph_buffer/graph_buffer.h" -#include "foundation/log.h" -#include "foundation/platform.h" -#include "foundation/compat.h" -#include "cbm.h" -#include "yyjson/yyjson.h" - -#include -#include -#include -#include -#include - -enum { CBM_TLD_MAX_DEPTH = 256 }; /* recursion-depth cap (cycle/stack guard) */ - -/* Parse an integer "key":N from a flat JSON object. Returns def if absent. */ -static int json_get_int(const char *json, const char *key, int dflt) { - if (!json) { - return dflt; - } - char pat[CBM_SZ_64]; - snprintf(pat, sizeof(pat), "\"%s\":", key); - const char *p = strstr(json, pat); - if (!p) { - return dflt; - } - p += strlen(pat); - while (*p == ' ' || *p == '\t') { - p++; - } - return (int)strtol(p, NULL, CBM_DECIMAL_BASE); -} - -/* Parse a boolean "key":true/false from a flat JSON object. */ -static bool json_get_bool(const char *json, const char *key) { - if (!json) { - return false; - } - char pat[CBM_SZ_64]; - snprintf(pat, sizeof(pat), "\"%s\":", key); - const char *p = strstr(json, pat); - if (!p) { - return false; - } - p += strlen(pat); - while (*p == ' ' || *p == '\t') { - p++; - } - return *p == 't'; -} - -/* Replace the derived complexity suffix when a graph buffer was loaded from a - * prior index, then append the current values. Full indexing calls this once; - * incremental indexing recomputes it over a mixed old/new graph, so blindly - * appending would leave duplicate JSON keys and stale values. */ -static void append_complexity_props(cbm_gbuf_node_t *node, int tld, bool recursive) { - const char *old = node->properties_json ? node->properties_json : "{}"; - yyjson_doc *doc = yyjson_read(old, strlen(old), 0); - yyjson_val *root = doc ? yyjson_doc_get_root(doc) : NULL; - if (!root || !yyjson_is_obj(root)) { - yyjson_doc_free(doc); - return; - } - yyjson_mut_doc *mdoc = yyjson_mut_doc_new(NULL); - if (!mdoc) { - yyjson_doc_free(doc); - return; - } - yyjson_mut_val *mroot = yyjson_val_mut_copy(mdoc, root); - if (!mroot) { - yyjson_mut_doc_free(mdoc); - yyjson_doc_free(doc); - return; - } - yyjson_mut_doc_set_root(mdoc, mroot); - /* Loaded incremental nodes may contain these fields before later-derived - * fields such as decorator_tags, and older generations may contain more - * than one copy. yyjson removes every matching key before current values - * are added exactly once. */ - yyjson_mut_obj_remove_key(mroot, "transitive_loop_depth"); - yyjson_mut_obj_remove_key(mroot, "recursive"); - yyjson_mut_obj_add_int(mdoc, mroot, "transitive_loop_depth", tld); - yyjson_mut_obj_add_bool(mdoc, mroot, "recursive", recursive); - char *neu = yyjson_mut_write(mdoc, 0, NULL); - yyjson_mut_doc_free(mdoc); - yyjson_doc_free(doc); - if (!neu) return; - free(node->properties_json); - node->properties_json = neu; -} - -static int compare_node_text(const char *left, const char *right) { - return strcmp(left ? left : "", right ? right : ""); -} - -static int compare_complexity_nodes(const void *left, const void *right) { - const cbm_gbuf_node_t *a = *(const cbm_gbuf_node_t *const *)left; - const cbm_gbuf_node_t *b = *(const cbm_gbuf_node_t *const *)right; - int cmp = compare_node_text(a->qualified_name, b->qualified_name); - if (cmp == 0) cmp = compare_node_text(a->label, b->label); - if (cmp == 0) cmp = compare_node_text(a->file_path, b->file_path); - if (cmp == 0 && a->start_line != b->start_line) - cmp = a->start_line < b->start_line ? -1 : 1; - if (cmp == 0 && a->end_line != b->end_line) - cmp = a->end_line < b->end_line ? -1 : 1; - if (cmp == 0) cmp = compare_node_text(a->name, b->name); - return cmp; -} - -typedef struct { - int64_t id; - int rank; -} complexity_callee_t; - -static int compare_complexity_callees(const void *left, const void *right) { - const complexity_callee_t *a = left; - const complexity_callee_t *b = right; - if (a->rank != b->rank) return a->rank < b->rank ? -1 : 1; - if (a->id != b->id) return a->id < b->id ? -1 : 1; - return 0; -} - -/* Memoized DFS: tld(id) = loop_depth(id) + max over CALLS-callees of tld(callee). - * state: 0=unvisited, 1=in-progress (back-edge → cycle), 2=done. */ -static int tld_dfs(const cbm_gbuf_t *gb, int64_t id, const int *loop_depth, int *tld, char *state, - bool *recursive, const int *rank, int64_t maxid, int depth) { - if (id < 1 || id > maxid) { - return 0; - } - if (state[id] == 2) { - return tld[id]; - } - if (state[id] == 1) { - recursive[id] = true; /* back edge → call-graph cycle */ - return 0; - } - if (depth > CBM_TLD_MAX_DEPTH) { - return loop_depth[id]; - } - state[id] = 1; - int best = 0; - const cbm_gbuf_edge_t **edges = NULL; - int ne = 0; - cbm_gbuf_find_edges_by_source_type(gb, id, "CALLS", &edges, &ne); - complexity_callee_t *callees = ne > 0 ? malloc((size_t)ne * sizeof(*callees)) : NULL; - if (ne > 0 && !callees) { - state[id] = 2; - tld[id] = loop_depth[id]; - return tld[id]; - } - for (int i = 0; i < ne; i++) { - int64_t target = edges[i]->target_id; - callees[i] = (complexity_callee_t){ - .id = target, - .rank = target >= 1 && target <= maxid && rank[target] > 0 - ? rank[target] - : INT_MAX, - }; - } - if (ne > 1) { - qsort(callees, (size_t)ne, sizeof(*callees), compare_complexity_callees); - } - for (int i = 0; i < ne; i++) { - int64_t c = callees[i].id; - if (c == id) { - recursive[id] = true; /* direct self-recursion */ - continue; - } - int ct = tld_dfs(gb, c, loop_depth, tld, state, recursive, rank, maxid, depth + 1); - if (ct > best) { - best = ct; - } - } - free(callees); - tld[id] = loop_depth[id] + best; - state[id] = 2; - return tld[id]; -} - -/* Seed each Function/Method node's loop_depth and self_recursive flag, and - * remember the node pointer for write-back. The self_recursive seed (set at - * extraction) feeds the final recursive flag; tld_dfs additionally ORs in - * mutual recursion discovered as a call-graph cycle. */ -static void seed_loop_depths(const cbm_gbuf_t *gb, const char *label, int *loop_depth, - bool *recursive, cbm_gbuf_node_t **nptr, int64_t maxid) { - const cbm_gbuf_node_t **nodes = NULL; - int count = 0; - if (cbm_gbuf_find_by_label(gb, label, &nodes, &count) != 0) { - return; - } - for (int i = 0; i < count; i++) { - const cbm_gbuf_node_t *n = nodes[i]; - if (n->id >= 1 && n->id <= maxid) { - /* CBM_WALK_METRIC_UNAVAILABLE (-1) marks a body whose metric walk - * could not grow its stack; propagate it as "no loops of its own" - * rather than letting it subtract from its callers' depth. */ - int ld = json_get_int(n->properties_json, "loop_depth", 0); - loop_depth[n->id] = ld < 0 ? 0 : ld; - recursive[n->id] = json_get_bool(n->properties_json, "self_recursive"); - nptr[n->id] = (cbm_gbuf_node_t *)n; - } - } -} - -void cbm_pipeline_pass_complexity(cbm_pipeline_ctx_t *ctx) { - cbm_gbuf_t *gb = ctx->gbuf; - /* Node and edge IDs are drawn from one shared counter, so node IDs are NOT - * contiguous 1..node_count — they interleave with edge IDs. Size the lookup - * arrays by the id ceiling (next_id) so every node id is addressable. */ - int64_t maxid = cbm_gbuf_next_id(gb) - 1; - if (maxid < 1) { - return; - } - size_t sz = (size_t)maxid + 1; - int *loop_depth = calloc(sz, sizeof(int)); - int *tld = calloc(sz, sizeof(int)); - char *state = calloc(sz, sizeof(char)); - bool *recursive = calloc(sz, sizeof(bool)); - cbm_gbuf_node_t **nptr = calloc(sz, sizeof(cbm_gbuf_node_t *)); - int *rank = calloc(sz, sizeof(int)); - if (!loop_depth || !tld || !state || !recursive || !nptr || !rank) { - free(loop_depth); - free(tld); - free(state); - free(recursive); - free(nptr); - free(rank); - return; - } - - seed_loop_depths(gb, "Function", loop_depth, recursive, nptr, maxid); - seed_loop_depths(gb, "Method", loop_depth, recursive, nptr, maxid); - - int node_count = 0; - for (int64_t id = 1; id <= maxid; id++) { - if (nptr[id]) node_count++; - } - cbm_gbuf_node_t **ordered = node_count > 0 - ? malloc((size_t)node_count * sizeof(*ordered)) - : NULL; - if (node_count > 0 && !ordered) { - free(loop_depth); free(tld); free(state); free(recursive); free(nptr); free(rank); - return; - } - int ordered_count = 0; - for (int64_t id = 1; id <= maxid; id++) { - if (nptr[id]) ordered[ordered_count++] = nptr[id]; - } - if (ordered_count > 1) { - qsort(ordered, (size_t)ordered_count, sizeof(*ordered), compare_complexity_nodes); - } - for (int i = 0; i < ordered_count; i++) rank[ordered[i]->id] = i + 1; - - for (int i = 0; i < ordered_count; i++) { - int64_t id = ordered[i]->id; - if (state[id] != 2) - tld_dfs(gb, id, loop_depth, tld, state, recursive, rank, maxid, 0); - append_complexity_props(ordered[i], tld[id], recursive[id]); - } - - cbm_log_info("pass.complexity", "functions", cbm_log_itoa(ordered_count)); - - free(ordered); - free(loop_depth); - free(tld); - free(state); - free(recursive); - free(nptr); - free(rank); -} diff --git a/src/pipeline/pass_complexity.zig b/src/pipeline/pass_complexity.zig new file mode 100644 index 00000000..04ba6fb0 --- /dev/null +++ b/src/pipeline/pass_complexity.zig @@ -0,0 +1,291 @@ +//! pass_complexity.zig — Interprocedural complexity propagation (Tier B). +//! +//! Tier A (in the extraction walk) stamps each Function/Method node with local +//! structural metrics: complexity (cyclomatic), cognitive, loop_count, loop_depth. +//! This pass propagates loop_depth along CALLS edges to estimate a worst-case +//! *transitive* nested-loop degree: a function with a depth-1 loop that calls an +//! O(n) helper is effectively O(n^2). The estimate assumes calls may occur inside +//! loops (an upper bound) — it is a queryable bottleneck *candidate* signal, not a +//! proof (true big-O is undecidable; cf. SPEED / Loopus). Cycles in the call graph +//! are broken and flagged via a `recursive` property. +//! +//! Writes two extra node properties: transitive_loop_depth, recursive. + +const std = @import("std"); +const c = @import("c"); +const interop = @import("../foundation/interop.zig"); +const util = @import("util.zig"); + +const CStr = util.CStr; +const Node = c.cbm_gbuf_node_t; +const Edge = c.cbm_gbuf_edge_t; + +const CBM_TLD_MAX_DEPTH = 256; // recursion-depth cap (cycle/stack guard) + +/// The text after "key": in a flat JSON object, past spaces and tabs; NULL +/// when the key is absent. +fn jsonValueOf(json: CStr, key: CStr) CStr { + if (json == null) return null; + var pat: [c.CBM_SZ_64]u8 = undefined; + _ = c.snprintf(&pat, pat.len, "\"%s\":", key); + var p: CStr = c.strstr(json, &pat); + if (p == null) return null; + p += c.strlen(&pat); + while (p[0] == ' ' or p[0] == '\t') p += 1; + return p; +} + +/// Parse an integer "key":N from a flat JSON object. Returns dflt if absent. +fn jsonGetInt(json: CStr, key: CStr, dflt: c_int) c_int { + const p = jsonValueOf(json, key); + if (p == null) return dflt; + return @truncate(c.strtol(p, null, c.CBM_DECIMAL_BASE)); +} + +/// Parse a boolean "key":true/false from a flat JSON object. +fn jsonGetBool(json: CStr, key: CStr) bool { + const p = jsonValueOf(json, key); + if (p == null) return false; + return p[0] == 't'; +} + +/// Replace the derived complexity suffix when a graph buffer was loaded from a +/// prior index, then append the current values. Full indexing calls this once; +/// incremental indexing recomputes it over a mixed old/new graph, so blindly +/// appending would leave duplicate JSON keys and stale values. +fn appendComplexityProps(node: *Node, tld: c_int, recursive: bool) void { + @setEvalBranchQuota(10000); + const old: CStr = if (node.properties_json != null) node.properties_json else "{}"; + const doc = c.yyjson_read(old, c.strlen(old), 0); + const root = if (doc != null) c.yyjson_doc_get_root(doc) else null; + if (root == null or !c.yyjson_is_obj(root)) { + c.yyjson_doc_free(doc); + return; + } + const mdoc = c.yyjson_mut_doc_new(null); + if (mdoc == null) { + c.yyjson_doc_free(doc); + return; + } + const mroot = c.yyjson_val_mut_copy(mdoc, root); + if (mroot == null) { + c.yyjson_mut_doc_free(mdoc); + c.yyjson_doc_free(doc); + return; + } + c.yyjson_mut_doc_set_root(mdoc, mroot); + // Loaded incremental nodes may contain these fields before later-derived + // fields such as decorator_tags, and older generations may contain more + // than one copy. yyjson removes every matching key before current values + // are added exactly once. + _ = c.yyjson_mut_obj_remove_key(mroot, "transitive_loop_depth"); + _ = c.yyjson_mut_obj_remove_key(mroot, "recursive"); + _ = c.yyjson_mut_obj_add_int(mdoc, mroot, "transitive_loop_depth", tld); + _ = c.yyjson_mut_obj_add_bool(mdoc, mroot, "recursive", recursive); + const neu = c.yyjson_mut_write(mdoc, 0, null); + c.yyjson_mut_doc_free(mdoc); + c.yyjson_doc_free(doc); + if (neu == null) return; + c.free(node.properties_json); + node.properties_json = neu; +} + +fn compareNodeText(left: CStr, right: CStr) c_int { + return c.strcmp(if (left != null) left else "", if (right != null) right else ""); +} + +fn compareComplexityNodes(left: ?*const anyopaque, right: ?*const anyopaque) callconv(.c) c_int { + const a: *const Node = @as(*const *const Node, @ptrCast(@alignCast(left))).*; + const b: *const Node = @as(*const *const Node, @ptrCast(@alignCast(right))).*; + var cmp = compareNodeText(a.qualified_name, b.qualified_name); + if (cmp == 0) cmp = compareNodeText(a.label, b.label); + if (cmp == 0) cmp = compareNodeText(a.file_path, b.file_path); + if (cmp == 0 and a.start_line != b.start_line) cmp = if (a.start_line < b.start_line) -1 else 1; + if (cmp == 0 and a.end_line != b.end_line) cmp = if (a.end_line < b.end_line) -1 else 1; + if (cmp == 0) cmp = compareNodeText(a.name, b.name); + return cmp; +} + +const ComplexityCallee = extern struct { + id: i64, + rank: c_int, +}; + +fn compareComplexityCallees(left: ?*const anyopaque, right: ?*const anyopaque) callconv(.c) c_int { + const a: *const ComplexityCallee = @ptrCast(@alignCast(left)); + const b: *const ComplexityCallee = @ptrCast(@alignCast(right)); + if (a.rank != b.rank) return if (a.rank < b.rank) -1 else 1; + if (a.id != b.id) return if (a.id < b.id) -1 else 1; + return 0; +} + +/// The per-node tables of the pass, indexed by node id. +const Tables = struct { + loop_depth: [*]c_int, + tld: [*]c_int, + state: [*]u8, + recursive: [*]bool, + rank: [*]c_int, + maxid: i64, +}; + +/// Memoized DFS: tld(id) = loop_depth(id) + max over CALLS-callees of tld(callee). +/// state: 0=unvisited, 1=in-progress (back-edge → cycle), 2=done. +fn tldDfs(gb: ?*const c.cbm_gbuf_t, id: i64, t: *const Tables, depth: c_int) c_int { + if (id < 1 or id > t.maxid) return 0; + const n: usize = @intCast(id); + if (t.state[n] == 2) return t.tld[n]; + if (t.state[n] == 1) { + t.recursive[n] = true; // back edge → call-graph cycle + return 0; + } + if (depth > CBM_TLD_MAX_DEPTH) return t.loop_depth[n]; + t.state[n] = 1; + var best: c_int = 0; + var edges: [*c][*c]const Edge = null; + var ne: c_int = 0; + _ = c.cbm_gbuf_find_edges_by_source_type(gb, id, "CALLS", &edges, &ne); + const callees: [*c]ComplexityCallee = if (ne > 0) interop.mallocN(ComplexityCallee, util.ix(ne)) else null; + if (ne > 0 and callees == null) { + t.state[n] = 2; + t.tld[n] = t.loop_depth[n]; + return t.tld[n]; + } + var i: usize = 0; + while (i < ne) : (i += 1) { + const target = edges[i].*.target_id; + callees[i] = .{ + .id = target, + .rank = if (target >= 1 and target <= t.maxid and t.rank[@intCast(target)] > 0) + t.rank[@intCast(target)] + else + std.math.maxInt(c_int), + }; + } + if (ne > 1) { + c.qsort(callees, util.ix(ne), @sizeOf(ComplexityCallee), compareComplexityCallees); + } + i = 0; + while (i < ne) : (i += 1) { + const callee = callees[i].id; + if (callee == id) { + t.recursive[n] = true; // direct self-recursion + continue; + } + const ct = tldDfs(gb, callee, t, depth + 1); + if (ct > best) best = ct; + } + c.free(callees); + t.tld[n] = t.loop_depth[n] + best; + t.state[n] = 2; + return t.tld[n]; +} + +/// Seed each Function/Method node's loop_depth and self_recursive flag, and +/// remember the node pointer for write-back. The self_recursive seed (set at +/// extraction) feeds the final recursive flag; tldDfs additionally ORs in +/// mutual recursion discovered as a call-graph cycle. +fn seedLoopDepths(gb: ?*const c.cbm_gbuf_t, label: CStr, t: *const Tables, nptr: [*]?*Node) void { + var nodes: [*c][*c]const Node = null; + var count: c_int = 0; + if (c.cbm_gbuf_find_by_label(gb, label, &nodes, &count) != 0) return; + var i: usize = 0; + while (i < count) : (i += 1) { + const n: *const Node = nodes[i]; + if (n.id >= 1 and n.id <= t.maxid) { + const id: usize = @intCast(n.id); + // CBM_WALK_METRIC_UNAVAILABLE (-1) marks a body whose metric walk + // could not grow its stack; propagate it as "no loops of its own" + // rather than letting it subtract from its callers' depth. + const ld = jsonGetInt(n.properties_json, "loop_depth", 0); + t.loop_depth[id] = if (ld < 0) 0 else ld; + t.recursive[id] = jsonGetBool(n.properties_json, "self_recursive"); + nptr[id] = @constCast(n); + } + } +} + +pub export fn cbm_pipeline_pass_complexity(ctx: *c.cbm_pipeline_ctx_t) void { + const gb = ctx.gbuf; + // Node and edge IDs are drawn from one shared counter, so node IDs are NOT + // contiguous 1..node_count — they interleave with edge IDs. Size the lookup + // arrays by the id ceiling (next_id) so every node id is addressable. + const maxid = c.cbm_gbuf_next_id(gb) - 1; + if (maxid < 1) return; + const sz: usize = @as(usize, @intCast(maxid)) + 1; + const loop_depth = interop.callocN(c_int, sz); + const tld = interop.callocN(c_int, sz); + const state = interop.callocN(u8, sz); + const recursive = interop.callocN(bool, sz); + const nptr = interop.callocN(?*Node, sz); + const rank = interop.callocN(c_int, sz); + if (loop_depth == null or tld == null or state == null or recursive == null or nptr == null or + rank == null) + { + c.free(loop_depth); + c.free(tld); + c.free(state); + c.free(recursive); + c.free(@ptrCast(nptr)); + c.free(rank); + return; + } + const t: Tables = .{ + .loop_depth = loop_depth, + .tld = tld, + .state = state, + .recursive = recursive, + .rank = rank, + .maxid = maxid, + }; + + seedLoopDepths(gb, "Function", &t, nptr); + seedLoopDepths(gb, "Method", &t, nptr); + + var node_count: usize = 0; + for (nptr[1..sz]) |node| { + if (node != null) node_count += 1; + } + const ordered: [*c]*Node = if (node_count > 0) interop.mallocN(*Node, node_count) else null; + if (node_count > 0 and ordered == null) { + c.free(loop_depth); + c.free(tld); + c.free(state); + c.free(recursive); + c.free(@ptrCast(nptr)); + c.free(rank); + return; + } + var ordered_count: usize = 0; + for (nptr[1..sz]) |node| { + if (node) |found| { + ordered[ordered_count] = found; + ordered_count += 1; + } + } + if (ordered_count > 1) { + c.qsort(@ptrCast(ordered), ordered_count, @sizeOf(*Node), compareComplexityNodes); + } + for (0..ordered_count) |i| { + rank[@intCast(ordered[i].id)] = @intCast(i + 1); + } + + for (0..ordered_count) |i| { + const id = ordered[i].id; + const n: usize = @intCast(id); + if (state[n] != 2) _ = tldDfs(gb, id, &t, 0); + appendComplexityProps(ordered[i], tld[n], recursive[n]); + } + + interop.logInfo("pass.complexity", .{ + @as(CStr, "functions"), @as(CStr, c.cbm_log_itoa(@intCast(ordered_count))), + }); + + c.free(@ptrCast(ordered)); + c.free(loop_depth); + c.free(tld); + c.free(state); + c.free(recursive); + c.free(@ptrCast(nptr)); + c.free(rank); +} diff --git a/src/pipeline/pass_configlink.c b/src/pipeline/pass_configlink.c deleted file mode 100644 index a9c6014b..00000000 --- a/src/pipeline/pass_configlink.c +++ /dev/null @@ -1,434 +0,0 @@ -/* - * pass_configlink.c — Config ↔ Code linking strategies (pre-dump pass). - * - * Three strategies link config files to code symbols: - * 1. Key→Symbol: normalized config key matches code function/variable name - * 2. Dep→Import: package manifest dependency matches IMPORTS edge target - * 3. File→Ref: source code string literal references config file path - * - * Operates on the graph buffer before dump to .db file. - */ -#include "pipeline/pipeline.h" -#include "pipeline/pipeline_internal.h" -#include "graph_buffer/graph_buffer.h" -#include "foundation/hash_table.h" -#include "foundation/log.h" -#include "foundation/compat.h" - -#include -#include -#include -#include -#include -#include "foundation/compat_regex.h" -#include "foundation/str_util.h" - -/* ── Config link confidence scores ───────────────────────────────── */ -/* Strategy 1: Key→Symbol matching */ -#define CONF_KEY_EXACT 0.85 -#define CONF_KEY_SUBSTRING 0.75 -/* Strategy 2: Dep→Import matching */ -#define CONF_DEP_EXACT 0.95 -#define CONF_DEP_QN_SUBSTR 0.80 -/* Strategy 3: File→Ref matching */ -#define CONF_FILE_FULLPATH 0.90 -#define CONF_FILE_BASENAME 0.70 - -/* ── Manifest / dep section tables ──────────────────────────────── */ - -static bool is_manifest_file(const char *basename) { - static const char *names[] = {"Cargo.toml", "package.json", "go.mod", - "requirements.txt", "Gemfile", "build.gradle", - "pom.xml", "composer.json", NULL}; - for (int i = 0; names[i]; i++) { - if (strcmp(basename, names[i]) == 0) { - return true; - } - } - return false; -} - -static bool is_dep_section(const char *s) { - static const char *secs[] = {"dependencies", "devdependencies", "peerdependencies", - "dev-dependencies", "build-dependencies", NULL}; - for (int i = 0; secs[i]; i++) { - if (cbm_strcasestr(s, secs[i]) != NULL) { - return true; - } - } - return false; -} - -/* ── Strategy 1: Config Key → Code Symbol ───────────────────────── */ - -typedef struct { - int64_t node_id; - char normalized[CBM_SZ_256]; - char name[CBM_SZ_256]; -} config_entry_t; - -static int node_ref_qualified_name_compare(const void *a, const void *b) { - const cbm_gbuf_node_t *left = *(const cbm_gbuf_node_t *const *)a; - const cbm_gbuf_node_t *right = *(const cbm_gbuf_node_t *const *)b; - return strcmp(left->qualified_name, right->qualified_name); -} - -static const cbm_gbuf_node_t **sorted_node_refs( - const cbm_gbuf_node_t *const *nodes, int count) { - if (!nodes || count <= 0) return NULL; - const cbm_gbuf_node_t **sorted = malloc((size_t)count * sizeof(*sorted)); - if (!sorted) return NULL; - memcpy(sorted, nodes, (size_t)count * sizeof(*sorted)); - if (count > 1) { - qsort(sorted, (size_t)count, sizeof(*sorted), node_ref_qualified_name_compare); - } - return sorted; -} - -/* Collect config Variable nodes with ≥2 tokens, each ≥3 chars. */ -static int collect_config_entries(const cbm_gbuf_node_t *const *vars, int var_count, - config_entry_t *out, int max_out) { - const cbm_gbuf_node_t **sorted = sorted_node_refs(vars, var_count); - if (var_count > 0 && !sorted) return 0; - int n = 0; - for (int i = 0; i < var_count && n < max_out; i++) { - if (!cbm_has_config_extension(sorted[i]->file_path)) { - continue; - } - - char norm[CBM_SZ_256]; - int tokens = cbm_normalize_config_key(sorted[i]->name, norm, sizeof(norm)); - if (tokens < PAIR_LEN) { - continue; - } - - /* Check all tokens ≥3 chars */ - bool all_long = true; - const char *p = norm; - while (*p) { - const char *end = strchr(p, '_'); - size_t tlen = end ? (size_t)(end - p) : strlen(p); - if (tlen < CBM_SZ_3) { - all_long = false; - break; - } - p = end ? end + SKIP_ONE : p + tlen; - } - if (!all_long) { - continue; - } - - out[n].node_id = sorted[i]->id; - snprintf(out[n].normalized, sizeof(out[n].normalized), "%s", norm); - snprintf(out[n].name, sizeof(out[n].name), "%s", sorted[i]->name); - n++; - } - free(sorted); - return n; -} - -/* Collect code nodes (Function/Variable/Class/Struct) not from config files. */ -typedef struct { - int64_t node_id; - char normalized[CBM_SZ_256]; -} code_entry_t; - -static int collect_code_entries(cbm_gbuf_t *gb, code_entry_t *out, int max_out) { - int n = 0; - /* "Struct" alongside "Class": a config key may name a Go/Rust/Swift/D struct - * type, which is now labelled "Struct" — keep it linkable. */ - static const char *labels[] = {"Function", "Variable", "Class", "Struct", NULL}; - - for (int li = 0; labels[li] && n < max_out; li++) { - const cbm_gbuf_node_t **nodes = NULL; - int count = 0; - if (cbm_gbuf_find_by_label(gb, labels[li], &nodes, &count) != 0) { - continue; - } - - const cbm_gbuf_node_t **sorted = sorted_node_refs(nodes, count); - if (count > 0 && !sorted) return n; - - for (int i = 0; i < count && n < max_out; i++) { - if (cbm_has_config_extension(sorted[i]->file_path)) { - continue; - } - - char norm[CBM_SZ_256]; - int tokens = cbm_normalize_config_key(sorted[i]->name, norm, sizeof(norm)); - if (tokens == 0 || norm[0] == '\0') { - continue; - } - - out[n].node_id = sorted[i]->id; - snprintf(out[n].normalized, sizeof(out[n].normalized), "%s", norm); - n++; - } - free(sorted); - } - return n; -} - -static int strategy_key_symbols(cbm_gbuf_t *gb) { - /* Get all Variable nodes */ - const cbm_gbuf_node_t **vars = NULL; - int var_count = 0; - if (cbm_gbuf_find_by_label(gb, "Variable", &vars, &var_count) != 0) { - return 0; - } - - config_entry_t config_entries[CBM_SZ_4K]; - int config_count = collect_config_entries(vars, var_count, config_entries, CBM_SZ_4K); - - if (config_count == 0) { - return 0; - } - - code_entry_t code_entries[CBM_SZ_8K]; - int code_count = collect_code_entries(gb, code_entries, CBM_SZ_8K); - - int edge_count = 0; - - for (int ci = 0; ci < config_count; ci++) { - for (int co = 0; co < code_count; co++) { - double confidence = 0.0; - - if (strcmp(config_entries[ci].normalized, code_entries[co].normalized) == 0) { - /* Exact match */ - confidence = CONF_KEY_EXACT; - } else if (strstr(code_entries[co].normalized, config_entries[ci].normalized) != NULL) { - /* Substring match */ - confidence = CONF_KEY_SUBSTRING; - } - - if (confidence > 0.0) { - char props[CBM_SZ_512]; - snprintf(props, sizeof(props), - "{\"strategy\":\"key_symbol\",\"confidence\":%.2f,\"config_key\":\"%s\"}", - confidence, config_entries[ci].name); - - cbm_gbuf_insert_edge(gb, code_entries[co].node_id, config_entries[ci].node_id, - "CONFIGURES", props); - edge_count++; - } - } - } - - return edge_count; -} - -/* ── Strategy 2: Dependency → Import ────────────────────────────── */ - -typedef struct { - int64_t node_id; - char name[CBM_SZ_256]; -} dep_entry_t; - -/* Check if a Cargo.toml QN contains a dependency section in any dotted part. */ -static bool is_cargo_dep_section(const char *qn) { - char qn_copy[CBM_SZ_512]; - snprintf(qn_copy, sizeof(qn_copy), "%s", qn); - char *saveptr = NULL; - char *part = strtok_r(qn_copy, ".", &saveptr); - while (part) { - char lower[CBM_SZ_128]; - size_t plen = strlen(part); - if (plen >= sizeof(lower)) { - plen = sizeof(lower) - SKIP_ONE; - } - for (size_t j = 0; j < plen; j++) { - lower[j] = (char)tolower((unsigned char)part[j]); - } - lower[plen] = '\0'; - - static const char *dep_secs[] = {"dependencies", "devdependencies", - "peerdependencies", "dev-dependencies", - "build-dependencies", NULL}; - for (int k = 0; dep_secs[k]; k++) { - if (strcmp(lower, dep_secs[k]) == 0) { - return true; - } - } - part = strtok_r(NULL, ".", &saveptr); - } - return false; -} - -static int collect_manifest_deps(const cbm_gbuf_node_t *const *vars, int var_count, - dep_entry_t *out, int max_out) { - int n = 0; - for (int i = 0; i < var_count && n < max_out; i++) { - const char *base = cbm_path_base(vars[i]->file_path); - if (!is_manifest_file(base)) { - continue; - } - - bool is_dep = vars[i]->qualified_name && is_dep_section(vars[i]->qualified_name); - - if (!is_dep && strcmp(base, "Cargo.toml") == 0 && vars[i]->qualified_name) { - is_dep = is_cargo_dep_section(vars[i]->qualified_name); - } - - if (is_dep) { - out[n].node_id = vars[i]->id; - snprintf(out[n].name, sizeof(out[n].name), "%s", vars[i]->name); - n++; - } - } - return n; -} - -/* Lowercase a string into buf. */ -static void lowercase_into(char *buf, size_t bufsize, const char *src) { - size_t len = src ? strlen(src) : 0; - for (size_t j = 0; j < len && j < bufsize - SKIP_ONE; j++) { - buf[j] = (char)tolower((unsigned char)src[j]); - } - buf[len < bufsize ? len : bufsize - SKIP_ONE] = '\0'; -} - -/* Match a dep name (lowercased) against an import target node. - * Returns confidence > 0 on match, 0 on no match. */ -static double match_dep_to_import(const cbm_gbuf_node_t *target, const char *dep_lower) { - char target_lower[CBM_SZ_256]; - lowercase_into(target_lower, sizeof(target_lower), target->name); - - if (strcmp(target_lower, dep_lower) == 0) { - return CONF_DEP_EXACT; - } - if (target->qualified_name) { - char qn_lower[CBM_SZ_512]; - lowercase_into(qn_lower, sizeof(qn_lower), target->qualified_name); - if (strstr(qn_lower, dep_lower) != NULL) { - return CONF_DEP_QN_SUBSTR; - } - } - return 0.0; -} - -static int strategy_dep_imports(cbm_gbuf_t *gb) { - const cbm_gbuf_node_t **vars = NULL; - int var_count = 0; - if (cbm_gbuf_find_by_label(gb, "Variable", &vars, &var_count) != 0) { - return 0; - } - - dep_entry_t deps[CBM_SZ_2K]; - int dep_count = collect_manifest_deps(vars, var_count, deps, CBM_SZ_2K); - - if (dep_count == 0) { - return 0; - } - - /* Get all IMPORTS edges */ - const cbm_gbuf_edge_t **imports = NULL; - int import_count = 0; - if (cbm_gbuf_find_edges_by_type(gb, "IMPORTS", &imports, &import_count) != 0) { - return 0; - } - - int edge_count = 0; - - for (int di = 0; di < dep_count; di++) { - char dep_lower[CBM_SZ_256]; - lowercase_into(dep_lower, sizeof(dep_lower), deps[di].name); - - for (int ii = 0; ii < import_count; ii++) { - const cbm_gbuf_node_t *target = cbm_gbuf_find_by_id(gb, imports[ii]->target_id); - if (!target) { - continue; - } - - const cbm_gbuf_node_t *source = cbm_gbuf_find_by_id(gb, imports[ii]->source_id); - if (!source) { - continue; - } - - double confidence = match_dep_to_import(target, dep_lower); - if (confidence > 0.0) { - char props[CBM_SZ_512]; - snprintf( - props, sizeof(props), - "{\"strategy\":\"dependency_import\",\"confidence\":%.2f,\"dep_name\":\"%s\"}", - confidence, deps[di].name); - - cbm_gbuf_insert_edge(gb, source->id, deps[di].node_id, "CONFIGURES", props); - edge_count++; - } - } - } - - /* gbuf data is borrowed — no free */ - return edge_count; -} - -/* ── Strategy 3: Config File Path → Code String Reference ───────── */ - -typedef struct { - const char *key; - int64_t node_id; -} path_map_t; - -/* Match a ref_path against config module maps. Returns target node_id (0 = no match). */ - -int cbm_pipeline_pass_configlink(cbm_pipeline_ctx_t *ctx) { - cbm_gbuf_t *gb = ctx->gbuf; - /* Early exit: check if any config files exist in the project. */ - bool has_config = false; - - const cbm_gbuf_node_t **vars_check = NULL; - int var_check_count = 0; - if (!has_config && cbm_gbuf_find_by_label(gb, "Variable", &vars_check, &var_check_count) == 0) { - for (int i = 0; i < var_check_count; i++) { - if (cbm_has_config_extension(vars_check[i]->file_path)) { - has_config = true; - break; - } - } - } - - if (!has_config) { - const cbm_gbuf_node_t **mods_check = NULL; - int mod_check_count = 0; - if (cbm_gbuf_find_by_label(gb, "Module", &mods_check, &mod_check_count) == 0) { - for (int i = 0; i < mod_check_count; i++) { - if (cbm_has_config_extension(mods_check[i]->file_path)) { - has_config = true; - break; - } - } - } - } - - if (!has_config) { - cbm_log_info("configlinker.skip", "reason", "no_config_files"); - return 0; - } - - char buf1[CBM_SZ_16]; - char buf2[CBM_SZ_16]; - char buf3[CBM_SZ_16]; - char buf4[CBM_SZ_16]; - - int key_edges = strategy_key_symbols(gb); - snprintf(buf1, sizeof(buf1), "%d", key_edges); - cbm_log_info("configlinker.strategy", "name", "key_symbol", "edges", buf1); - - int dep_edges = strategy_dep_imports(gb); - snprintf(buf2, sizeof(buf2), "%d", dep_edges); - cbm_log_info("configlinker.strategy", "name", "dep_import", "edges", buf2); - - int ref_edges = 0; - if (ctx->repo_path) { - /* File refs: no longer reads from disk — config file path matching - * is handled by CONFIGURES edges created during resolution. */ - ref_edges = 0; - } - snprintf(buf3, sizeof(buf3), "%d", ref_edges); - cbm_log_info("configlinker.strategy", "name", "file_ref", "edges", buf3); - - snprintf(buf4, sizeof(buf4), "%d", key_edges + dep_edges + ref_edges); - cbm_log_info("configlinker.done", "total", buf4); - - return key_edges + dep_edges + ref_edges; -} diff --git a/src/pipeline/pass_configlink.zig b/src/pipeline/pass_configlink.zig new file mode 100644 index 00000000..edf79af3 --- /dev/null +++ b/src/pipeline/pass_configlink.zig @@ -0,0 +1,378 @@ +//! pass_configlink.zig — Config ↔ Code linking strategies (pre-dump pass). +//! +//! Three strategies link config files to code symbols: +//! 1. Key→Symbol: normalized config key matches code function/variable name +//! 2. Dep→Import: package manifest dependency matches IMPORTS edge target +//! 3. File→Ref: source code string literal references config file path +//! +//! Operates on the graph buffer before dump to .db file. + +const std = @import("std"); +const c = @import("c"); +const interop = @import("../foundation/interop.zig"); +const util = @import("util.zig"); + +const CStr = util.CStr; +const MutStr = util.MutStr; +const ix = util.ix; +const streq = util.streq; +const Node = c.cbm_gbuf_node_t; +const Edge = c.cbm_gbuf_edge_t; +const NodeRefs = [*c]const [*c]const Node; + +// ── Config link confidence scores ─────────────────────────────────── +// Strategy 1: Key→Symbol matching +const CONF_KEY_EXACT: f64 = 0.85; +const CONF_KEY_SUBSTRING: f64 = 0.75; +// Strategy 2: Dep→Import matching +const CONF_DEP_EXACT: f64 = 0.95; +const CONF_DEP_QN_SUBSTR: f64 = 0.80; + +// ── Manifest / dep section tables ─────────────────────────────────── + +fn isManifestFile(basename: CStr) bool { + const names = [_][*:0]const u8{ + "Cargo.toml", "package.json", "go.mod", + "requirements.txt", "Gemfile", "build.gradle", + "pom.xml", "composer.json", + }; + for (names) |name| { + if (streq(basename, name)) return true; + } + return false; +} + +const DEP_SECTIONS = [_][*:0]const u8{ + "dependencies", "devdependencies", "peerdependencies", "dev-dependencies", "build-dependencies", +}; + +fn isDepSection(s: CStr) bool { + for (DEP_SECTIONS) |sec| { + if (c.cbm_strcasestr(s, sec) != null) return true; + } + return false; +} + +// ── Strategy 1: Config Key → Code Symbol ──────────────────────────── + +const ConfigEntry = extern struct { + node_id: i64, + normalized: [c.CBM_SZ_256]u8, + name: [c.CBM_SZ_256]u8, +}; + +fn nodeRefQualifiedNameCompare(a: ?*const anyopaque, b: ?*const anyopaque) callconv(.c) c_int { + const left: *const Node = @as(*const *const Node, @ptrCast(@alignCast(a))).*; + const right: *const Node = @as(*const *const Node, @ptrCast(@alignCast(b))).*; + return c.strcmp(left.qualified_name, right.qualified_name); +} + +fn sortedNodeRefs(nodes: NodeRefs, count: c_int) [*c][*c]const Node { + if (nodes == null or count <= 0) return null; + const sorted = interop.mallocN([*c]const Node, ix(count)); + if (sorted == null) return null; + @memcpy(sorted[0..ix(count)], nodes[0..ix(count)]); + if (count > 1) { + c.qsort(@ptrCast(sorted), ix(count), @sizeOf([*c]const Node), nodeRefQualifiedNameCompare); + } + return sorted; +} + +/// Collect config Variable nodes with ≥2 tokens, each ≥3 chars. +fn collectConfigEntries(vars: NodeRefs, var_count: c_int, out: []ConfigEntry) c_int { + const sorted = sortedNodeRefs(vars, var_count); + if (var_count > 0 and sorted == null) return 0; + var n: usize = 0; + var i: usize = 0; + while (i < var_count and n < out.len) : (i += 1) { + const node: *const Node = sorted[i]; + if (!c.cbm_has_config_extension(node.file_path)) continue; + + var norm: [c.CBM_SZ_256]u8 = undefined; + const tokens = c.cbm_normalize_config_key(node.name, &norm, norm.len); + if (tokens < 2) continue; + + // Check all tokens ≥3 chars + var all_long = true; + var p: [*]const u8 = &norm; + while (p[0] != 0) { + const end: CStr = c.strchr(p, '_'); + const tlen: usize = if (end != null) util.ptrDiff(end, p) else c.strlen(p); + if (tlen < c.CBM_SZ_3) { + all_long = false; + break; + } + p = if (end != null) end + 1 else p + tlen; + } + if (!all_long) continue; + + out[n].node_id = node.id; + _ = c.snprintf(&out[n].normalized, out[n].normalized.len, "%s", @as(CStr, &norm)); + _ = c.snprintf(&out[n].name, out[n].name.len, "%s", node.name); + n += 1; + } + c.free(@ptrCast(sorted)); + return @intCast(n); +} + +/// Collect code nodes (Function/Variable/Class/Struct) not from config files. +const CodeEntry = extern struct { + node_id: i64, + normalized: [c.CBM_SZ_256]u8, +}; + +fn collectCodeEntries(gb: ?*c.cbm_gbuf_t, out: []CodeEntry) c_int { + var n: usize = 0; + // "Struct" alongside "Class": a config key may name a Go/Rust/Swift/D struct + // type, which is now labelled "Struct" — keep it linkable. + const labels = [_][*:0]const u8{ "Function", "Variable", "Class", "Struct" }; + + for (labels) |label| { + if (n >= out.len) break; + var nodes: [*c][*c]const Node = null; + var count: c_int = 0; + if (c.cbm_gbuf_find_by_label(gb, label, &nodes, &count) != 0) continue; + + const sorted = sortedNodeRefs(nodes, count); + if (count > 0 and sorted == null) return @intCast(n); + + var i: usize = 0; + while (i < count and n < out.len) : (i += 1) { + const node: *const Node = sorted[i]; + if (c.cbm_has_config_extension(node.file_path)) continue; + + var norm: [c.CBM_SZ_256]u8 = undefined; + const tokens = c.cbm_normalize_config_key(node.name, &norm, norm.len); + if (tokens == 0 or norm[0] == 0) continue; + + out[n].node_id = node.id; + _ = c.snprintf(&out[n].normalized, out[n].normalized.len, "%s", @as(CStr, &norm)); + n += 1; + } + c.free(@ptrCast(sorted)); + } + return @intCast(n); +} + +fn strategyKeySymbols(gb: ?*c.cbm_gbuf_t) c_int { + // Get all Variable nodes + var vars: [*c][*c]const Node = null; + var var_count: c_int = 0; + if (c.cbm_gbuf_find_by_label(gb, "Variable", &vars, &var_count) != 0) return 0; + + var config_entries: [c.CBM_SZ_4K]ConfigEntry = undefined; + const config_count = collectConfigEntries(vars, var_count, &config_entries); + + if (config_count == 0) return 0; + + var code_entries: [c.CBM_SZ_8K]CodeEntry = undefined; + const code_count = collectCodeEntries(gb, &code_entries); + + var edge_count: c_int = 0; + + for (config_entries[0..ix(config_count)]) |*config| { + for (code_entries[0..ix(code_count)]) |*code| { + var confidence: f64 = 0.0; + + if (streq(&config.normalized, &code.normalized)) { + // Exact match + confidence = CONF_KEY_EXACT; + } else if (c.strstr(&code.normalized, &config.normalized) != null) { + // Substring match + confidence = CONF_KEY_SUBSTRING; + } + + if (confidence > 0.0) { + var props: [c.CBM_SZ_512]u8 = undefined; + _ = c.snprintf( + &props, + props.len, + "{\"strategy\":\"key_symbol\",\"confidence\":%.2f,\"config_key\":\"%s\"}", + confidence, + @as(CStr, &config.name), + ); + + _ = c.cbm_gbuf_insert_edge(gb, code.node_id, config.node_id, "CONFIGURES", &props); + edge_count += 1; + } + } + } + + return edge_count; +} + +// ── Strategy 2: Dependency → Import ───────────────────────────────── + +const DepEntry = extern struct { + node_id: i64, + name: [c.CBM_SZ_256]u8, +}; + +/// Check if a Cargo.toml QN contains a dependency section in any dotted part. +fn isCargoDepSection(qn: CStr) bool { + var qn_copy: [c.CBM_SZ_512]u8 = undefined; + _ = c.snprintf(&qn_copy, qn_copy.len, "%s", qn); + var saveptr: MutStr = null; + var part: MutStr = c.strtok_r(&qn_copy, ".", &saveptr); + while (part != null) { + var lower: [c.CBM_SZ_128]u8 = undefined; + var plen = c.strlen(part); + if (plen >= lower.len) plen = lower.len - 1; + for (0..plen) |j| lower[j] = std.ascii.toLower(part[j]); + lower[plen] = 0; + + for (DEP_SECTIONS) |sec| { + if (streq(&lower, sec)) return true; + } + part = c.strtok_r(null, ".", &saveptr); + } + return false; +} + +fn collectManifestDeps(vars: NodeRefs, var_count: c_int, out: []DepEntry) c_int { + var n: usize = 0; + var i: usize = 0; + while (i < var_count and n < out.len) : (i += 1) { + const node: *const Node = vars[i]; + const base: CStr = c.cbm_path_base(node.file_path); + if (!isManifestFile(base)) continue; + + var is_dep = node.qualified_name != null and isDepSection(node.qualified_name); + + if (!is_dep and streq(base, "Cargo.toml") and node.qualified_name != null) { + is_dep = isCargoDepSection(node.qualified_name); + } + + if (is_dep) { + out[n].node_id = node.id; + _ = c.snprintf(&out[n].name, out[n].name.len, "%s", node.name); + n += 1; + } + } + return @intCast(n); +} + +/// Lowercase a string into buf. +fn lowercaseInto(buf: []u8, src: CStr) void { + const len: usize = if (src != null) c.strlen(src) else 0; + var j: usize = 0; + while (j < len and j < buf.len - 1) : (j += 1) { + buf[j] = std.ascii.toLower(src[j]); + } + buf[if (len < buf.len) len else buf.len - 1] = 0; +} + +/// Match a dep name (lowercased) against an import target node. +/// Returns confidence > 0 on match, 0 on no match. +fn matchDepToImport(target: *const Node, dep_lower: CStr) f64 { + var target_lower: [c.CBM_SZ_256]u8 = undefined; + lowercaseInto(&target_lower, target.name); + + if (streq(&target_lower, dep_lower)) return CONF_DEP_EXACT; + if (target.qualified_name != null) { + var qn_lower: [c.CBM_SZ_512]u8 = undefined; + lowercaseInto(&qn_lower, target.qualified_name); + if (c.strstr(&qn_lower, dep_lower) != null) return CONF_DEP_QN_SUBSTR; + } + return 0.0; +} + +fn strategyDepImports(gb: ?*c.cbm_gbuf_t) c_int { + var vars: [*c][*c]const Node = null; + var var_count: c_int = 0; + if (c.cbm_gbuf_find_by_label(gb, "Variable", &vars, &var_count) != 0) return 0; + + var deps: [c.CBM_SZ_2K]DepEntry = undefined; + const dep_count = collectManifestDeps(vars, var_count, &deps); + + if (dep_count == 0) return 0; + + // Get all IMPORTS edges + var imports: [*c][*c]const Edge = null; + var import_count: c_int = 0; + if (c.cbm_gbuf_find_edges_by_type(gb, "IMPORTS", &imports, &import_count) != 0) return 0; + + var edge_count: c_int = 0; + + for (deps[0..ix(dep_count)]) |*dep| { + var dep_lower: [c.CBM_SZ_256]u8 = undefined; + lowercaseInto(&dep_lower, &dep.name); + + var ii: usize = 0; + while (ii < import_count) : (ii += 1) { + const target: *const Node = @as(?*const Node, c.cbm_gbuf_find_by_id(gb, imports[ii].*.target_id)) orelse continue; + const source: *const Node = @as(?*const Node, c.cbm_gbuf_find_by_id(gb, imports[ii].*.source_id)) orelse continue; + + const confidence = matchDepToImport(target, &dep_lower); + if (confidence > 0.0) { + var props: [c.CBM_SZ_512]u8 = undefined; + _ = c.snprintf( + &props, + props.len, + "{\"strategy\":\"dependency_import\",\"confidence\":%.2f,\"dep_name\":\"%s\"}", + confidence, + @as(CStr, &dep.name), + ); + + _ = c.cbm_gbuf_insert_edge(gb, source.id, dep.node_id, "CONFIGURES", &props); + edge_count += 1; + } + } + } + + // gbuf data is borrowed — no free + return edge_count; +} + +// ── Strategy 3: Config File Path → Code String Reference ──────────── + +/// True when a node of this label sits in a config file. +fn labelHasConfigFile(gb: ?*c.cbm_gbuf_t, label: CStr) bool { + var nodes: [*c][*c]const Node = null; + var count: c_int = 0; + if (c.cbm_gbuf_find_by_label(gb, label, &nodes, &count) != 0) return false; + var i: usize = 0; + while (i < count) : (i += 1) { + if (c.cbm_has_config_extension(nodes[i].*.file_path)) return true; + } + return false; +} + +fn logStrategy(name: CStr, edges: c_int, buf: *[c.CBM_SZ_16]u8) void { + _ = c.snprintf(buf, buf.len, "%d", edges); + interop.logInfo("configlinker.strategy", .{ + @as(CStr, "name"), name, @as(CStr, "edges"), @as(CStr, buf), + }); +} + +pub export fn cbm_pipeline_pass_configlink(ctx: *c.cbm_pipeline_ctx_t) c_int { + const gb = ctx.gbuf; + // Early exit: check if any config files exist in the project. + const has_config = labelHasConfigFile(gb, "Variable") or labelHasConfigFile(gb, "Module"); + + if (!has_config) { + interop.logInfo("configlinker.skip", .{ @as(CStr, "reason"), @as(CStr, "no_config_files") }); + return 0; + } + + var buf1: [c.CBM_SZ_16]u8 = undefined; + var buf2: [c.CBM_SZ_16]u8 = undefined; + var buf3: [c.CBM_SZ_16]u8 = undefined; + var buf4: [c.CBM_SZ_16]u8 = undefined; + + const key_edges = strategyKeySymbols(gb); + logStrategy("key_symbol", key_edges, &buf1); + + const dep_edges = strategyDepImports(gb); + logStrategy("dep_import", dep_edges, &buf2); + + // File refs: no longer reads from disk — config file path matching + // is handled by CONFIGURES edges created during resolution. + const ref_edges: c_int = 0; + logStrategy("file_ref", ref_edges, &buf3); + + _ = c.snprintf(&buf4, buf4.len, "%d", key_edges + dep_edges + ref_edges); + interop.logInfo("configlinker.done", .{ @as(CStr, "total"), @as(CStr, &buf4) }); + + return key_edges + dep_edges + ref_edges; +} diff --git a/src/pipeline/pass_configures.c b/src/pipeline/pass_configures.c deleted file mode 100644 index 938f9bb8..00000000 --- a/src/pipeline/pass_configures.c +++ /dev/null @@ -1,131 +0,0 @@ -/* - * pass_configures.c — Config key and env var helpers. - * - * Pure helper functions for detecting environment variable names, - * normalizing config keys, and identifying config file extensions. - */ -#include "pipeline/pipeline.h" -#include "pipeline/pipeline_internal.h" - -#include -#include - -bool cbm_is_env_var_name(const char *s) { - if (!s) { - return false; - } - size_t len = strlen(s); - if (len < PAIR_LEN) { - return false; - } - - bool has_upper = false; - for (size_t i = 0; i < len; i++) { - char c = s[i]; - if (c >= 'A' && c <= 'Z') { - has_upper = true; - } else if (c == '_' || (c >= '0' && c <= '9')) { - /* ok */ - } else { - return false; - } - } - return has_upper; -} - -/* Emit a camelCase-split word (lowercased) into the output buffer. - * Prepends '_' if out_pos > 0. Returns updated out_pos and token_count. */ -static void emit_camel_words(const char *part, size_t plen, char *norm_out, size_t norm_sz, - size_t *out_pos, int *token_count) { - size_t start = 0; - for (size_t i = SKIP_ONE; i < plen; i++) { - if (part[i] >= 'A' && part[i] <= 'Z' && part[i - SKIP_ONE] >= 'a' && - part[i - SKIP_ONE] <= 'z') { - if (*out_pos > 0 && *out_pos < norm_sz - SKIP_ONE) { - norm_out[(*out_pos)++] = '_'; - } - for (size_t j = start; j < i && *out_pos < norm_sz - SKIP_ONE; j++) { - norm_out[(*out_pos)++] = (char)tolower((unsigned char)part[j]); - } - (*token_count)++; - start = i; - } - } - /* Emit remaining */ - if (*out_pos > 0 && *out_pos < norm_sz - SKIP_ONE) { - norm_out[(*out_pos)++] = '_'; - } - for (size_t j = start; j < plen && *out_pos < norm_sz - SKIP_ONE; j++) { - norm_out[(*out_pos)++] = (char)tolower((unsigned char)part[j]); - } - (*token_count)++; -} - -int cbm_normalize_config_key(const char *key, char *norm_out, size_t norm_sz) { - if (!key || !norm_out || norm_sz == 0) { - return 0; - } - norm_out[0] = '\0'; - - char buf[CBM_SZ_512]; - size_t klen = strlen(key); - if (klen >= sizeof(buf)) { - klen = sizeof(buf) - SKIP_ONE; - } - memcpy(buf, key, klen); - buf[klen] = '\0'; - - /* Replace delimiters with spaces */ - for (size_t i = 0; i < klen; i++) { - if (buf[i] == '_' || buf[i] == '-' || buf[i] == '.') { - buf[i] = ' '; - } - } - - int token_count = 0; - size_t out_pos = 0; - - char *saveptr = NULL; - char *part = strtok_r(buf, " ", &saveptr); - while (part) { - emit_camel_words(part, strlen(part), norm_out, norm_sz, &out_pos, &token_count); - part = strtok_r(NULL, " ", &saveptr); - } - - norm_out[out_pos] = '\0'; - return token_count; -} - -bool cbm_has_config_extension(const char *path) { - if (!path) { - return false; - } - - /* Find last dot */ - const char *dot = strrchr(path, '.'); - const char *basename = strrchr(path, '/'); - if (!basename) { - basename = path; - } else { - basename++; - } - - /* Special case: .env files */ - if (strcmp(basename, ".env") == 0) { - return true; - } - - if (!dot) { - return false; - } - - static const char *exts[] = {".toml", ".ini", ".yaml", ".yml", ".cfg", ".properties", - ".json", ".xml", ".conf", ".env", NULL}; - - for (int i = 0; exts[i]; i++) { - if (strcmp(dot, exts[i]) == 0) { - return true; - } - } - return false; -} diff --git a/src/pipeline/pass_configures.zig b/src/pipeline/pass_configures.zig new file mode 100644 index 00000000..59719022 --- /dev/null +++ b/src/pipeline/pass_configures.zig @@ -0,0 +1,117 @@ +//! pass_configures.zig — Config key and env var helpers. +//! +//! Pure helper functions for detecting environment variable names, +//! normalizing config keys, and identifying config file extensions. + +const std = @import("std"); +const c = @import("c"); +const util = @import("util.zig"); + +const CStr = util.CStr; +const MutStr = util.MutStr; + +pub export fn cbm_is_env_var_name(s: CStr) bool { + if (s == null) return false; + const len = c.strlen(s); + if (len < 2) return false; + + var has_upper = false; + for (s[0..len]) |ch| { + if (ch >= 'A' and ch <= 'Z') { + has_upper = true; + } else if (ch == '_' or (ch >= '0' and ch <= '9')) { + // ok + } else { + return false; + } + } + return has_upper; +} + +/// Emit a camelCase-split word (lowercased) into the output buffer. +/// Prepends '_' if out_pos > 0. Updates out_pos and token_count. +fn emitCamelWords(part: []const u8, norm_out: [*]u8, norm_sz: usize, out_pos: *usize, token_count: *c_int) void { + var start: usize = 0; + var i: usize = 1; + while (i < part.len) : (i += 1) { + if (part[i] >= 'A' and part[i] <= 'Z' and part[i - 1] >= 'a' and part[i - 1] <= 'z') { + if (out_pos.* > 0 and out_pos.* < norm_sz - 1) { + norm_out[out_pos.*] = '_'; + out_pos.* += 1; + } + var j = start; + while (j < i and out_pos.* < norm_sz - 1) : (j += 1) { + norm_out[out_pos.*] = std.ascii.toLower(part[j]); + out_pos.* += 1; + } + token_count.* += 1; + start = i; + } + } + // Emit remaining + if (out_pos.* > 0 and out_pos.* < norm_sz - 1) { + norm_out[out_pos.*] = '_'; + out_pos.* += 1; + } + var j = start; + while (j < part.len and out_pos.* < norm_sz - 1) : (j += 1) { + norm_out[out_pos.*] = std.ascii.toLower(part[j]); + out_pos.* += 1; + } + token_count.* += 1; +} + +pub export fn cbm_normalize_config_key(key: CStr, norm_out: MutStr, norm_sz: usize) c_int { + if (key == null or norm_out == null or norm_sz == 0) return 0; + norm_out[0] = 0; + + var buf: [c.CBM_SZ_512]u8 = undefined; + var klen = c.strlen(key); + if (klen >= buf.len) klen = buf.len - 1; + @memcpy(buf[0..klen], key[0..klen]); + buf[klen] = 0; + + // Replace delimiters with spaces + for (buf[0..klen]) |*ch| { + if (ch.* == '_' or ch.* == '-' or ch.* == '.') ch.* = ' '; + } + + var token_count: c_int = 0; + var out_pos: usize = 0; + + var saveptr: MutStr = null; + var part: MutStr = c.strtok_r(&buf, " ", &saveptr); + while (part != null) { + emitCamelWords(part[0..c.strlen(part)], norm_out, norm_sz, &out_pos, &token_count); + part = c.strtok_r(null, " ", &saveptr); + } + + norm_out[out_pos] = 0; + return token_count; +} + +pub export fn cbm_has_config_extension(path: CStr) bool { + if (path == null) return false; + + // Find last dot + const dot: CStr = c.strrchr(path, '.'); + var basename: CStr = c.strrchr(path, '/'); + if (basename == null) { + basename = path; + } else { + basename += 1; + } + + // Special case: .env files + if (util.streq(basename, ".env")) return true; + + if (dot == null) return false; + + const exts = [_][*:0]const u8{ + ".toml", ".ini", ".yaml", ".yml", ".cfg", ".properties", ".json", ".xml", ".conf", ".env", + }; + for (exts) |ext| { + if (util.streq(dot, ext)) return true; + } + return false; +} diff --git a/src/pipeline/pass_enrichment.c b/src/pipeline/pass_enrichment.c deleted file mode 100644 index ee7579c1..00000000 --- a/src/pipeline/pass_enrichment.c +++ /dev/null @@ -1,407 +0,0 @@ -/* - * pass_enrichment.c — Decorator tokenization, camelCase splitting, - * and decorator_tags pass (post-flush enrichment). - * - * Pure helper functions + a store-level pass that classifies decorators - * into semantic tags via auto-discovery (words on 2+ nodes become tags). - */ -#include "foundation/constants.h" - -enum { ENRICH_ATTR_SKIP = 2, ENRICH_MAX_CAMEL = 16 }; -#include "pipeline/pipeline.h" -#include -#include "pipeline/pipeline_internal.h" -#include "graph_buffer/graph_buffer.h" -#include "foundation/log.h" -#include "foundation/hash_table.h" -#include "foundation/platform.h" -#include "foundation/compat.h" -#include "yyjson/yyjson.h" - -#include -#include -#include - -static bool is_decorator_stopword(const char *w) { - static const char *stopwords[] = {"get", "set", "new", "class", "method", "function", - "value", "type", "param", "return", "public", "private", - "for", "if", "the", "and", "or", "not", - "with", "from", "app", "router", NULL}; - for (int i = 0; stopwords[i]; i++) { - if (strcmp(w, stopwords[i]) == 0) { - return true; - } - } - return false; -} - -int cbm_split_camel_case(const char *s, char **out, int max_out) { - if (!s || !out || max_out <= 0) { - return 0; - } - size_t len = strlen(s); - if (len == 0) { - return 0; - } - - int count = 0; - size_t start = 0; - - for (size_t i = SKIP_ONE; i < len; i++) { - if (s[i] >= 'A' && s[i] <= 'Z' && s[i - SKIP_ONE] >= 'a' && s[i - SKIP_ONE] <= 'z') { - if (count < max_out) { - out[count] = cbm_strndup(s + start, i - start); - count++; - } - start = i; - } - } - /* Emit remaining */ - if (count < max_out) { - out[count] = cbm_strndup(s + start, len - start); - count++; - } - return count; -} - -/* Strip decorator syntax: leading @, #[...], and arguments (...). - * Operates in-place on buf. Returns pointer to the cleaned token start. */ -static char *strip_decorator_syntax(char *buf) { - char *p = buf; - if (*p == '@') { - p++; - } - if (p[0] == '#' && p[SKIP_ONE] == '[') { - p += ENRICH_ATTR_SKIP; - size_t plen = strlen(p); - if (plen > 0 && p[plen - SKIP_ONE] == ']') { - p[plen - SKIP_ONE] = '\0'; - } - } - char *paren = strchr(p, '('); - if (paren) { - *paren = '\0'; - } - for (char *c = p; *c; c++) { - if (*c == '.' || *c == '_' || *c == '-' || *c == ':' || *c == '/') { - *c = ' '; - } - } - return p; -} - -int cbm_tokenize_decorator(const char *dec, char **out, int max_out) { - if (!dec || !out || max_out <= 0) { - return 0; - } - - char buf[CBM_SZ_256]; - size_t len = strlen(dec); - if (len >= sizeof(buf)) { - len = sizeof(buf) - SKIP_ONE; - } - memcpy(buf, dec, len); - buf[len] = '\0'; - - char *p = strip_decorator_syntax(buf); - - int count = 0; - char *saveptr = NULL; - char *part = strtok_r(p, " ", &saveptr); - - while (part && count < max_out) { - char *camel_parts[ENRICH_MAX_CAMEL]; - int camel_count = cbm_split_camel_case(part, camel_parts, ENRICH_MAX_CAMEL); - - for (int i = 0; i < camel_count && count < max_out; i++) { - for (char *c = camel_parts[i]; *c; c++) { - *c = (char)tolower((unsigned char)*c); - } - if (strlen(camel_parts[i]) >= PAIR_LEN && !is_decorator_stopword(camel_parts[i])) { - out[count++] = camel_parts[i]; - } else { - free(camel_parts[i]); - } - } - part = strtok_r(NULL, " ", &saveptr); - } - - return count; -} - -/* ══════════════════════════════════════════════════════════════════ - * Decorator Tags Pass (post-flush, operates on store) - * - * Algorithm: - * 1. Load all Function/Method/Class nodes from the store - * 2. For each node with "decorators" property, tokenize decorators - * 3. Count word frequency across all nodes - * 4. Words on 2+ distinct nodes become candidates - * 5. Update each node's properties_json with "decorator_tags" array - * ══════════════════════════════════════════════════════════════════ */ - -/* Per-node tokenization state */ -typedef struct { - int64_t node_id; - char *qualified_name; - char **words; - int word_count; -} tagged_node_t; - -/* Extract the "decorators" array from a properties_json string. - * Returns a NULL-terminated array of strings. Caller must free array and strings. */ -static char **extract_decorators_from_json(const char *json) { - if (!json) { - return NULL; - } - - yyjson_doc *doc = yyjson_read(json, strlen(json), 0); - if (!doc) { - return NULL; - } - - yyjson_val *root = yyjson_doc_get_root(doc); - yyjson_val *decs = yyjson_obj_get(root, "decorators"); - if (!decs || !yyjson_is_arr(decs)) { - yyjson_doc_free(doc); - return NULL; - } - - size_t cnt = yyjson_arr_size(decs); - if (cnt == 0) { - yyjson_doc_free(doc); - return NULL; - } - - char **out = calloc(cnt + SKIP_ONE, sizeof(char *)); - size_t idx = 0; - yyjson_val *item; - yyjson_arr_iter iter; - yyjson_arr_iter_init(decs, &iter); - while ((item = yyjson_arr_iter_next(&iter))) { - if (yyjson_is_str(item)) { - out[idx++] = strdup(yyjson_get_str(item)); - } - } - out[idx] = NULL; - - yyjson_doc_free(doc); - if (idx > 0) { - return out; - } - free(out); - return NULL; -} - -/* Tokenize all decorators on a node into unique words. - * Returns heap-allocated array, caller frees each word and the array. */ -static int extract_decorator_words(const char *json, char ***out_words) { - char **decorators = extract_decorators_from_json(json); - if (!decorators) { - *out_words = NULL; - return 0; - } - - /* Collect unique words from all decorators */ - char *all_words[CBM_SZ_256]; - int total = 0; - CBMHashTable *seen = cbm_ht_create(CBM_SZ_32); - - for (int i = 0; decorators[i]; i++) { - char *tokens[CBM_SZ_32]; - int tc = cbm_tokenize_decorator(decorators[i], tokens, CBM_SZ_32); - for (int j = 0; j < tc; j++) { - if (!cbm_ht_get(seen, tokens[j]) && total < CBM_SZ_256) { - cbm_ht_set(seen, tokens[j], cbm_intptr_to_ptr(SKIP_ONE)); - all_words[total++] = tokens[j]; - } else { - free(tokens[j]); - } - } - free(decorators[i]); - } - free(decorators); - cbm_ht_free(seen); - - if (total == 0) { - *out_words = NULL; - return 0; - } - - *out_words = malloc(sizeof(char *) * total); - memcpy(*out_words, all_words, sizeof(char *) * total); - return total; -} - -/* Insert "decorator_tags" into a properties_json string. - * Returns a newly allocated JSON string. Caller must free(). */ -static char *inject_decorator_tags(const char *json, char **tags, int tag_count) { - yyjson_doc *doc = yyjson_read(json, strlen(json), 0); - yyjson_val *root = doc ? yyjson_doc_get_root(doc) : NULL; - - yyjson_mut_doc *mdoc = yyjson_mut_doc_new(NULL); - yyjson_mut_val *mroot; - - if (root && yyjson_is_obj(root)) { - mroot = yyjson_val_mut_copy(mdoc, root); - } else { - mroot = yyjson_mut_obj(mdoc); - } - yyjson_mut_doc_set_root(mdoc, mroot); - - /* Remove existing decorator_tags if any */ - yyjson_mut_obj_remove_key(mroot, "decorator_tags"); - - /* Add sorted tag array */ - yyjson_mut_val *arr = yyjson_mut_arr(mdoc); - for (int i = 0; i < tag_count; i++) { - yyjson_mut_arr_add_str(mdoc, arr, tags[i]); - } - yyjson_mut_obj_add_val(mdoc, mroot, "decorator_tags", arr); - - char *result = yyjson_mut_write(mdoc, 0, NULL); - yyjson_mut_doc_free(mdoc); - yyjson_doc_free(doc); - return result; -} - -/* Simple string comparison for qsort */ -static int cmp_str(const void *a, const void *b) { - return strcmp(*(const char **)a, *(const char **)b); -} - -/* Free tagged_node_t array. */ -static void free_tagged_nodes(tagged_node_t *nodes, int count) { - for (int n = 0; n < count; n++) { - free(nodes[n].qualified_name); - for (int w = 0; w < nodes[n].word_count; w++) { - free(nodes[n].words[w]); - } - free(nodes[n].words); - } - free(nodes); -} - -/* Phase 1: Collect decorated nodes and count word frequency. */ -static int collect_decorated_nodes(cbm_gbuf_t *gbuf, tagged_node_t **out_nodes, - CBMHashTable *word_counts) { - /* "Struct" alongside "Class" so Go/Rust/Swift/D struct names keep - * contributing to / receiving auto-tags as they did when structs were - * labelled "Class". */ - static const char *labels[] = {"Function", "Method", "Class", "Struct"}; - static const int nlabels = 4; - tagged_node_t *nodes = NULL; - int node_count = 0; - int node_cap = 0; - - for (int l = 0; l < nlabels; l++) { - const cbm_gbuf_node_t **found = NULL; - int fc = 0; - cbm_gbuf_find_by_label(gbuf, labels[l], &found, &fc); - if (!found || fc <= 0) { - continue; - } - - for (int i = 0; i < fc; i++) { - char **words = NULL; - int wc = extract_decorator_words(found[i]->properties_json, &words); - if (wc <= 0) { - continue; - } - if (node_count >= node_cap) { - node_cap = node_cap ? node_cap * PAIR_LEN : CBM_SZ_64; - nodes = safe_realloc(nodes, sizeof(tagged_node_t) * node_cap); - } - tagged_node_t *tn = &nodes[node_count++]; - tn->node_id = found[i]->id; - tn->qualified_name = strdup(found[i]->qualified_name); - tn->words = words; - tn->word_count = wc; - for (int w = 0; w < wc; w++) { - intptr_t cnt = (intptr_t)cbm_ht_get(word_counts, words[w]); - cbm_ht_set(word_counts, words[w], cbm_intptr_to_ptr(cnt + SKIP_ONE)); - } - } - } - - *out_nodes = nodes; - return node_count; -} - -/* Phase 3: Apply candidate tags to nodes in the gbuf. */ -static int apply_decorator_tags(cbm_gbuf_t *gbuf, tagged_node_t *nodes, int node_count, - CBMHashTable *candidates) { - int tagged = 0; - for (int n = 0; n < node_count; n++) { - char *tag_words[CBM_SZ_256]; - int tag_count = 0; - for (int w = 0; w < nodes[n].word_count; w++) { - if (cbm_ht_get(candidates, nodes[n].words[w]) && tag_count < CBM_SZ_256) { - tag_words[tag_count++] = nodes[n].words[w]; - } - } - if (tag_count == 0) { - continue; - } - if (tag_count > 1) { - qsort(tag_words, tag_count, sizeof(char *), cmp_str); - } - - const cbm_gbuf_node_t *gn = cbm_gbuf_find_by_qn(gbuf, nodes[n].qualified_name); - if (!gn) { - continue; - } - - const char *props = gn->properties_json ? gn->properties_json : "{}"; - char *new_props = inject_decorator_tags(props, tag_words, tag_count); - if (new_props) { - cbm_gbuf_upsert_node(gbuf, gn->label, gn->name, gn->qualified_name, gn->file_path, - gn->start_line, gn->end_line, new_props); - free(new_props); - tagged++; - } - } - return tagged; -} - -int cbm_pipeline_pass_decorator_tags(cbm_gbuf_t *gbuf, const char *project) { - if (!gbuf || !project) { - return 0; - } - - CBMHashTable *word_counts = cbm_ht_create(CBM_SZ_128); - tagged_node_t *nodes = NULL; - int node_count = collect_decorated_nodes(gbuf, &nodes, word_counts); - if (node_count == 0) { - cbm_ht_free(word_counts); - free(nodes); - return 0; - } - - /* Phase 2: Determine candidates (words on 2+ nodes) */ - CBMHashTable *candidates = cbm_ht_create(CBM_SZ_64); - int candidate_count = 0; - for (int n = 0; n < node_count; n++) { - for (int w = 0; w < nodes[n].word_count; w++) { - const char *word = nodes[n].words[w]; - intptr_t cnt = (intptr_t)cbm_ht_get(word_counts, word); - if (cnt >= PAIR_LEN && !cbm_ht_get(candidates, word)) { - cbm_ht_set(candidates, word, cbm_intptr_to_ptr(SKIP_ONE)); - candidate_count++; - } - } - } - - int tagged = 0; - if (candidate_count > 0) { - tagged = apply_decorator_tags(gbuf, nodes, node_count, candidates); - } - - free_tagged_nodes(nodes, node_count); - cbm_ht_free(word_counts); - cbm_ht_free(candidates); - - cbm_log_info("pass.decorator_tags", "candidates", candidate_count > 0 ? "yes" : "0", "tagged", - tagged > 0 ? "yes" : "0"); - return tagged; -} diff --git a/src/pipeline/pass_enrichment.zig b/src/pipeline/pass_enrichment.zig new file mode 100644 index 00000000..7e62060c --- /dev/null +++ b/src/pipeline/pass_enrichment.zig @@ -0,0 +1,391 @@ +//! pass_enrichment.zig — Decorator tokenization, camelCase splitting, +//! and decorator_tags pass (post-flush enrichment). +//! +//! Pure helper functions + a store-level pass that classifies decorators +//! into semantic tags via auto-discovery (words on 2+ nodes become tags). + +const std = @import("std"); +const c = @import("c"); +const interop = @import("../foundation/interop.zig"); +const util = @import("util.zig"); + +const CStr = util.CStr; +const MutStr = util.MutStr; +const ix = util.ix; +const Node = c.cbm_gbuf_node_t; +const HashTable = c.CBMHashTable; + +const ENRICH_ATTR_SKIP = 2; +const ENRICH_MAX_CAMEL = 16; + +/// cbm_intptr_to_ptr: a small integer stored as a hash table value. +fn intToPtr(v: isize) ?*anyopaque { + return @ptrFromInt(@as(usize, @bitCast(v))); +} + +fn ptrToInt(p: ?*anyopaque) isize { + return @bitCast(@intFromPtr(p)); +} + +fn isDecoratorStopword(w: CStr) bool { + const stopwords = [_][*:0]const u8{ + "get", "set", "new", "class", "method", "function", + "value", "type", "param", "return", "public", "private", + "for", "if", "the", "and", "or", "not", + "with", "from", "app", "router", + }; + for (stopwords) |stopword| { + if (util.streq(w, stopword)) return true; + } + return false; +} + +pub export fn cbm_split_camel_case(s: CStr, out: [*c]MutStr, max_out: c_int) c_int { + if (s == null or out == null or max_out <= 0) return 0; + const len = c.strlen(s); + if (len == 0) return 0; + + var count: c_int = 0; + var start: usize = 0; + + var i: usize = 1; + while (i < len) : (i += 1) { + if (s[i] >= 'A' and s[i] <= 'Z' and s[i - 1] >= 'a' and s[i - 1] <= 'z') { + if (count < max_out) { + out[ix(count)] = c.cbm_strndup(s + start, i - start); + count += 1; + } + start = i; + } + } + // Emit remaining + if (count < max_out) { + out[ix(count)] = c.cbm_strndup(s + start, len - start); + count += 1; + } + return count; +} + +/// Strip decorator syntax: leading @, #[...], and arguments (...). +/// Operates in-place on buf. Returns pointer to the cleaned token start. +fn stripDecoratorSyntax(buf: [*:0]u8) [*:0]u8 { + var p = buf; + if (p[0] == '@') p += 1; + if (p[0] == '#' and p[1] == '[') { + p += ENRICH_ATTR_SKIP; + const plen = c.strlen(p); + if (plen > 0 and p[plen - 1] == ']') p[plen - 1] = 0; + } + const paren: MutStr = c.strchr(p, '('); + if (paren != null) paren[0] = 0; + var ch = p; + while (ch[0] != 0) : (ch += 1) { + if (ch[0] == '.' or ch[0] == '_' or ch[0] == '-' or ch[0] == ':' or ch[0] == '/') { + ch[0] = ' '; + } + } + return p; +} + +pub export fn cbm_tokenize_decorator(dec: CStr, out: [*c]MutStr, max_out: c_int) c_int { + if (dec == null or out == null or max_out <= 0) return 0; + + var buf: [c.CBM_SZ_256:0]u8 = undefined; + var len = c.strlen(dec); + if (len >= c.CBM_SZ_256) len = c.CBM_SZ_256 - 1; + @memcpy(buf[0..len], dec[0..len]); + buf[len] = 0; + + const p = stripDecoratorSyntax(&buf); + + var count: c_int = 0; + var saveptr: MutStr = null; + var part: MutStr = c.strtok_r(p, " ", &saveptr); + + while (part != null and count < max_out) { + var camel_parts: [ENRICH_MAX_CAMEL]MutStr = undefined; + const camel_count = cbm_split_camel_case(part, &camel_parts, ENRICH_MAX_CAMEL); + + var i: usize = 0; + while (i < camel_count and count < max_out) : (i += 1) { + const word = camel_parts[i]; + const word_len = c.strlen(word); + for (word[0..word_len]) |*ch| ch.* = std.ascii.toLower(ch.*); + if (word_len >= 2 and !isDecoratorStopword(word)) { + out[ix(count)] = word; + count += 1; + } else { + c.free(word); + } + } + part = c.strtok_r(null, " ", &saveptr); + } + + return count; +} + +// ════════════════════════════════════════════════════════════════════ +// Decorator Tags Pass (post-flush, operates on store) +// +// Algorithm: +// 1. Load all Function/Method/Class nodes from the store +// 2. For each node with "decorators" property, tokenize decorators +// 3. Count word frequency across all nodes +// 4. Words on 2+ distinct nodes become candidates +// 5. Update each node's properties_json with "decorator_tags" array +// ════════════════════════════════════════════════════════════════════ + +/// Per-node tokenization state +const TaggedNode = extern struct { + node_id: i64, + qualified_name: MutStr, + words: [*c]MutStr, + word_count: c_int, +}; + +/// Extract the "decorators" array from a properties_json string. +/// Returns a NULL-terminated array of strings. Caller must free array and strings. +fn extractDecoratorsFromJson(json: CStr) [*c]MutStr { + @setEvalBranchQuota(10000); + if (json == null) return null; + + const doc = c.yyjson_read(json, c.strlen(json), 0); + if (doc == null) return null; + + const root = c.yyjson_doc_get_root(doc); + const decs = c.yyjson_obj_get(root, "decorators"); + if (decs == null or !c.yyjson_is_arr(decs)) { + c.yyjson_doc_free(doc); + return null; + } + + const cnt = c.yyjson_arr_size(decs); + if (cnt == 0) { + c.yyjson_doc_free(doc); + return null; + } + + const out = interop.callocN(MutStr, cnt + 1); + var idx: usize = 0; + var iter: c.yyjson_arr_iter = undefined; + _ = c.yyjson_arr_iter_init(decs, &iter); + while (true) { + const item = c.yyjson_arr_iter_next(&iter); + if (item == null) break; + if (c.yyjson_is_str(item)) { + out[idx] = c.strdup(c.yyjson_get_str(item)); + idx += 1; + } + } + out[idx] = null; + + c.yyjson_doc_free(doc); + if (idx > 0) return out; + c.free(@ptrCast(out)); + return null; +} + +/// Tokenize all decorators on a node into unique words. +/// Returns heap-allocated array, caller frees each word and the array. +fn extractDecoratorWords(json: CStr, out_words: *[*c]MutStr) c_int { + const decorators = extractDecoratorsFromJson(json); + if (decorators == null) { + out_words.* = null; + return 0; + } + + // Collect unique words from all decorators + var all_words: [c.CBM_SZ_256]MutStr = undefined; + var total: usize = 0; + const seen = c.cbm_ht_create(c.CBM_SZ_32); + + var i: usize = 0; + while (decorators[i] != null) : (i += 1) { + var tokens: [c.CBM_SZ_32]MutStr = undefined; + const tc = cbm_tokenize_decorator(decorators[i], &tokens, c.CBM_SZ_32); + for (tokens[0..ix(tc)]) |token| { + if (c.cbm_ht_get(seen, token) == null and total < all_words.len) { + _ = c.cbm_ht_set(seen, token, intToPtr(1)); + all_words[total] = token; + total += 1; + } else { + c.free(token); + } + } + c.free(decorators[i]); + } + c.free(@ptrCast(decorators)); + c.cbm_ht_free(seen); + + if (total == 0) { + out_words.* = null; + return 0; + } + + out_words.* = interop.mallocN(MutStr, total); + @memcpy(out_words.*[0..total], all_words[0..total]); + return @intCast(total); +} + +/// Insert "decorator_tags" into a properties_json string. +/// Returns a newly allocated JSON string. Caller must free(). +fn injectDecoratorTags(json: CStr, tags: []const MutStr) MutStr { + @setEvalBranchQuota(10000); + const doc = c.yyjson_read(json, c.strlen(json), 0); + const root = if (doc != null) c.yyjson_doc_get_root(doc) else null; + + const mdoc = c.yyjson_mut_doc_new(null); + const mroot = if (root != null and c.yyjson_is_obj(root)) + c.yyjson_val_mut_copy(mdoc, root) + else + c.yyjson_mut_obj(mdoc); + c.yyjson_mut_doc_set_root(mdoc, mroot); + + // Remove existing decorator_tags if any + _ = c.yyjson_mut_obj_remove_key(mroot, "decorator_tags"); + + // Add sorted tag array + const arr = c.yyjson_mut_arr(mdoc); + for (tags) |tag| { + _ = c.yyjson_mut_arr_add_str(mdoc, arr, tag); + } + _ = c.yyjson_mut_obj_add_val(mdoc, mroot, "decorator_tags", arr); + + const result = c.yyjson_mut_write(mdoc, 0, null); + c.yyjson_mut_doc_free(mdoc); + c.yyjson_doc_free(doc); + return result; +} + +/// Simple string comparison for qsort +fn cmpStr(a: ?*const anyopaque, b: ?*const anyopaque) callconv(.c) c_int { + const sa: *const CStr = @ptrCast(@alignCast(a)); + const sb: *const CStr = @ptrCast(@alignCast(b)); + return c.strcmp(sa.*, sb.*); +} + +/// Free TaggedNode array. +fn freeTaggedNodes(nodes: [*c]TaggedNode, count: c_int) void { + for (0..ix(count)) |n| { + c.free(nodes[n].qualified_name); + for (0..ix(nodes[n].word_count)) |w| { + c.free(nodes[n].words[w]); + } + c.free(@ptrCast(nodes[n].words)); + } + c.free(nodes); +} + +/// Phase 1: Collect decorated nodes and count word frequency. +fn collectDecoratedNodes(gbuf: *c.cbm_gbuf_t, out_nodes: *[*c]TaggedNode, word_counts: ?*HashTable) c_int { + // "Struct" alongside "Class" so Go/Rust/Swift/D struct names keep + // contributing to / receiving auto-tags as they did when structs were + // labelled "Class". + const labels = [_][*:0]const u8{ "Function", "Method", "Class", "Struct" }; + var nodes: [*c]TaggedNode = null; + var node_count: c_int = 0; + var node_cap: c_int = 0; + + for (labels) |label| { + var found: [*c][*c]const Node = null; + var fc: c_int = 0; + _ = c.cbm_gbuf_find_by_label(gbuf, label, &found, &fc); + if (found == null or fc <= 0) continue; + + for (0..ix(fc)) |i| { + var words: [*c]MutStr = null; + const wc = extractDecoratorWords(found[i].*.properties_json, &words); + if (wc <= 0) continue; + if (node_count >= node_cap) { + node_cap = if (node_cap != 0) node_cap * 2 else c.CBM_SZ_64; + nodes = interop.safeReallocN(TaggedNode, nodes, ix(node_cap)); + } + const tn = util.at(nodes, ix(node_count)); + node_count += 1; + tn.node_id = found[i].*.id; + tn.qualified_name = c.strdup(found[i].*.qualified_name); + tn.words = words; + tn.word_count = wc; + for (0..ix(wc)) |w| { + const cnt = ptrToInt(c.cbm_ht_get(word_counts, words[w])); + _ = c.cbm_ht_set(word_counts, words[w], intToPtr(cnt + 1)); + } + } + } + + out_nodes.* = nodes; + return node_count; +} + +/// Phase 3: Apply candidate tags to nodes in the gbuf. +fn applyDecoratorTags(gbuf: *c.cbm_gbuf_t, nodes: [*c]TaggedNode, node_count: c_int, candidates: ?*HashTable) c_int { + var tagged: c_int = 0; + for (0..ix(node_count)) |n| { + var tag_words: [c.CBM_SZ_256]MutStr = undefined; + var tag_count: usize = 0; + for (0..ix(nodes[n].word_count)) |w| { + if (c.cbm_ht_get(candidates, nodes[n].words[w]) != null and tag_count < tag_words.len) { + tag_words[tag_count] = nodes[n].words[w]; + tag_count += 1; + } + } + if (tag_count == 0) continue; + if (tag_count > 1) { + c.qsort(&tag_words, tag_count, @sizeOf(MutStr), cmpStr); + } + + const gn: *const Node = @as(?*const Node, c.cbm_gbuf_find_by_qn(gbuf, nodes[n].qualified_name)) orelse continue; + + const props: CStr = if (gn.properties_json != null) gn.properties_json else "{}"; + const new_props = injectDecoratorTags(props, tag_words[0..tag_count]); + if (new_props != null) { + _ = c.cbm_gbuf_upsert_node(gbuf, gn.label, gn.name, gn.qualified_name, gn.file_path, gn.start_line, gn.end_line, new_props); + c.free(new_props); + tagged += 1; + } + } + return tagged; +} + +pub export fn cbm_pipeline_pass_decorator_tags(gbuf_c: ?*c.cbm_gbuf_t, project: CStr) c_int { + const gbuf = gbuf_c orelse return 0; + if (project == null) return 0; + + const word_counts = c.cbm_ht_create(c.CBM_SZ_128); + var nodes: [*c]TaggedNode = null; + const node_count = collectDecoratedNodes(gbuf, &nodes, word_counts); + if (node_count == 0) { + c.cbm_ht_free(word_counts); + c.free(nodes); + return 0; + } + + // Phase 2: Determine candidates (words on 2+ nodes) + const candidates = c.cbm_ht_create(c.CBM_SZ_64); + var candidate_count: c_int = 0; + for (0..ix(node_count)) |n| { + for (0..ix(nodes[n].word_count)) |w| { + const word: CStr = nodes[n].words[w]; + const cnt = ptrToInt(c.cbm_ht_get(word_counts, word)); + if (cnt >= 2 and c.cbm_ht_get(candidates, word) == null) { + _ = c.cbm_ht_set(candidates, word, intToPtr(1)); + candidate_count += 1; + } + } + } + + var tagged: c_int = 0; + if (candidate_count > 0) { + tagged = applyDecoratorTags(gbuf, nodes, node_count, candidates); + } + + freeTaggedNodes(nodes, node_count); + c.cbm_ht_free(word_counts); + c.cbm_ht_free(candidates); + + interop.logInfo("pass.decorator_tags", .{ + @as(CStr, "candidates"), @as(CStr, if (candidate_count > 0) "yes" else "0"), + @as(CStr, "tagged"), @as(CStr, if (tagged > 0) "yes" else "0"), + }); + return tagged; +} diff --git a/src/pipeline/pass_envscan.c b/src/pipeline/pass_envscan.c deleted file mode 100644 index 8989959a..00000000 --- a/src/pipeline/pass_envscan.c +++ /dev/null @@ -1,433 +0,0 @@ -/* - * pass_envscan.c — Environment URL scanner. - * - * Walks a project directory, scans config files (Dockerfile, .env, shell, - * YAML, TOML, Terraform, .properties) for environment variable assignments - * where the value is a URL. Filters out secrets. - * - * Port of internal/pipeline/envscan.go:ScanProjectEnvURLs(). - */ -#include "foundation/constants.h" - -enum { - ENV_REGEX_MAX = 5, - ENV_GRP_1 = 1, - ENV_GRP_2 = 2, - ENV_GRP_3 = 3, - ENV_GRP_4 = 4, - ENV_GRP_5 = 5, - ENV_EXT_LEN = 4, /* strlen(".env") */ -}; - -#define SLEN(s) (sizeof(s) - 1) -#include "pipeline/pipeline.h" -#include "pipeline/pipeline_internal.h" -#include "foundation/log.h" - -#include -#include "foundation/compat_fs.h" -#include "foundation/compat_regex.h" -#include "foundation/compat_thread.h" -#include -#include -#include -#include - -/* ── Regex patterns (compiled lazily) ──────────────────────────── */ - -static cbm_regex_t dockerfile_re; /* ENV|ARG KEY=VALUE or KEY VALUE */ -static cbm_regex_t yaml_kv_re; /* key: "https://..." */ -static cbm_regex_t yaml_setenv_re; /* --set-env-vars KEY=VALUE */ -static cbm_regex_t terraform_re; /* default|value = "https://..." */ -static cbm_regex_t shell_re; /* [export] KEY=https://... */ -static cbm_regex_t envfile_re; /* KEY=https://... */ -static cbm_regex_t toml_re; /* key = "https://..." */ -static cbm_regex_t properties_re; /* key=https://... */ -static cbm_once_t patterns_once = CBM_ONCE_INIT; - -/* POSIX ERE doesn't support \w or \S — use bracket expressions */ -#define W "[A-Za-z0-9_]" /* word char */ -#define NW "[^ \t\"']" /* non-whitespace, non-quote */ - -/* Compiled once per process; concurrent pipelines then only read them. */ -static void compile_patterns_once(void) { - cbm_regcomp(&dockerfile_re, "^(ENV|ARG)[[:space:]]+(" W "+)[= ](.*)", CBM_REG_EXTENDED); - cbm_regcomp(&yaml_kv_re, "(" W "+):[[:space:]]*[\"']?(https?://" NW "+)", CBM_REG_EXTENDED); - cbm_regcomp(&yaml_setenv_re, "--set-env-vars[[:space:]]+(" W "+)=([^ \t]+)", CBM_REG_EXTENDED); - cbm_regcomp(&terraform_re, "(default|value)[[:space:]]*=[[:space:]]*\"(https?://[^\"]+)\"", - CBM_REG_EXTENDED); - cbm_regcomp(&shell_re, "(export[[:space:]]+)?(" W "+)=[\"']?(https?://" NW "+)", - CBM_REG_EXTENDED); - cbm_regcomp(&envfile_re, "^(" W "+)=(https?://[^ \t]+)", CBM_REG_EXTENDED); - cbm_regcomp(&toml_re, "(" W "+)[[:space:]]*=[[:space:]]*\"(https?://[^\"]+)\"", - CBM_REG_EXTENDED); - cbm_regcomp(&properties_re, "(" W "+)[[:space:]]*=[[:space:]]*(https?://[^ \t]+)", - CBM_REG_EXTENDED); -} - -static void compile_patterns(void) { - cbm_once(&patterns_once, compile_patterns_once); -} - -#undef W -#undef NW - -/* ── File type detection ───────────────────────────────────────── */ - -static int is_dockerfile_name(const char *name) { - /* Case-insensitive check */ - char lower[CBM_SZ_256]; - size_t len = strlen(name); - if (len >= sizeof(lower)) { - return 0; - } - for (size_t i = 0; i <= len; i++) { - lower[i] = (char)tolower((unsigned char)name[i]); - } - - if (strcmp(lower, "dockerfile") == 0) { - return SKIP_ONE; - } -#define DOCKERFILE_SUFFIX_LEN 11 /* strlen("dockerfile.") == strlen(".dockerfile") */ - if (strncmp(lower, "dockerfile.", DOCKERFILE_SUFFIX_LEN) == 0) { - return SKIP_ONE; - } - if (len > DOCKERFILE_SUFFIX_LEN && - strcmp(lower + len - DOCKERFILE_SUFFIX_LEN, ".dockerfile") == 0) { - return SKIP_ONE; - } - return 0; -} - -static int is_env_file_name(const char *name) { - char lower[CBM_SZ_256]; - size_t len = strlen(name); - if (len >= sizeof(lower)) { - return 0; - } - for (size_t i = 0; i <= len; i++) { - lower[i] = (char)tolower((unsigned char)name[i]); - } - - if (strcmp(lower, ".env") == 0) { - return SKIP_ONE; - } - if (strncmp(lower, ".env.", SLEN(".env.")) == 0) { - return SKIP_ONE; - } - if (len > ENV_EXT_LEN && strcmp(lower + len - ENV_EXT_LEN, ".env") == 0) { - return SKIP_ONE; - } - return 0; -} - -static int is_secret_file(const char *name) { - char lower[CBM_SZ_256]; - size_t len = strlen(name); - if (len >= sizeof(lower)) { - return 0; - } - for (size_t i = 0; i <= len; i++) { - lower[i] = (char)tolower((unsigned char)name[i]); - } - - static const char *patterns[] = { - "service_account", "credentials", "key.json", "key.pem", "id_rsa", - "id_ed25519", ".pem", ".key", NULL}; - for (int i = 0; patterns[i]; i++) { - if (strstr(lower, patterns[i])) { - return SKIP_ONE; - } - } - return 0; -} - -/* ── Ignored directories ───────────────────────────────────────── */ - -static int is_ignored_dir(const char *name) { - static const char *dirs[] = { - ".git", "node_modules", ".svn", ".hg", "__pycache__", "vendor", ".terraform", ".cache", - ".idea", ".vscode", "dist", "build", ".next", ".nuxt", "target", NULL}; - for (int i = 0; dirs[i]; i++) { - if (strcmp(name, dirs[i]) == 0) { - return SKIP_ONE; - } - } - return 0; -} - -/* ── File type enum ────────────────────────────────────────────── */ - -typedef enum { - FT_UNKNOWN = 0, - FT_DOCKERFILE, - FT_YAML, - FT_TERRAFORM, - FT_SHELL, - FT_ENVFILE, - FT_TOML, - FT_PROPERTIES, -} file_type_t; - -static file_type_t detect_file_type(const char *name) { - if (is_dockerfile_name(name)) { - return FT_DOCKERFILE; - } - if (is_env_file_name(name)) { - return FT_ENVFILE; - } - - const char *ext = strrchr(name, '.'); - if (!ext) { - return FT_UNKNOWN; - } - - if (strcmp(ext, ".yaml") == 0 || strcmp(ext, ".yml") == 0) { - return FT_YAML; - } - if (strcmp(ext, ".tf") == 0 || strcmp(ext, ".hcl") == 0) { - return FT_TERRAFORM; - } - if (strcmp(ext, ".sh") == 0 || strcmp(ext, ".bash") == 0 || strcmp(ext, ".zsh") == 0) { - return FT_SHELL; - } - if (strcmp(ext, ".toml") == 0) { - return FT_TOML; - } - if (strcmp(ext, ".properties") == 0 || strcmp(ext, ".cfg") == 0 || strcmp(ext, ".ini") == 0) { - return FT_PROPERTIES; - } - - return FT_UNKNOWN; -} - -/* ── Line scanner ──────────────────────────────────────────────── */ - -/* Extract key/value from a regex match with two capture groups. - * Returns 1 on success, 0 if groups are empty or too large. */ -static int extract_kv_groups(const char *trimmed, const cbm_regmatch_t *m, int key_grp, int val_grp, - char *key_out, size_t key_sz, char *val_out, size_t val_sz) { - int klen = (m[key_grp].rm_eo - m[key_grp].rm_so); - int vlen = (m[val_grp].rm_eo - m[val_grp].rm_so); - if (klen <= 0 || klen >= (int)key_sz || vlen <= 0 || vlen >= (int)val_sz) { - return 0; - } - memcpy(key_out, trimmed + m[key_grp].rm_so, klen); - key_out[klen] = '\0'; - memcpy(val_out, trimmed + m[val_grp].rm_so, vlen); - val_out[vlen] = '\0'; - return SKIP_ONE; -} - -/* Try to scan a Dockerfile line. */ -static int scan_dockerfile_line(const char *line, char *key, size_t ksz, char *val, size_t vsz) { - cbm_regmatch_t m[ENV_REGEX_MAX]; - if (cbm_regexec(&dockerfile_re, line, ENV_GRP_4, m, 0) != 0) { - return 0; - } - if (!extract_kv_groups(line, m, ENV_GRP_2, ENV_GRP_3, key, ksz, val, vsz)) { - return 0; - } - size_t vl = strlen(val); - while (vl > 0 && (val[vl - SKIP_ONE] == '"' || val[vl - SKIP_ONE] == '\'')) { - val[--vl] = '\0'; - } - return SKIP_ONE; -} - -/* Try to scan a YAML line. */ -static int scan_yaml_line(const char *line, char *key, size_t ksz, char *val, size_t vsz) { - cbm_regmatch_t m[ENV_REGEX_MAX]; - if (cbm_regexec(&yaml_kv_re, line, ENV_GRP_3, m, 0) == 0 && - extract_kv_groups(line, m, ENV_GRP_1, ENV_GRP_2, key, ksz, val, vsz)) { - return SKIP_ONE; - } - if (cbm_regexec(&yaml_setenv_re, line, ENV_GRP_3, m, 0) == 0 && - extract_kv_groups(line, m, ENV_GRP_1, ENV_GRP_2, key, ksz, val, vsz)) { - return SKIP_ONE; - } - return 0; -} - -/* Try to scan a Terraform line. */ -static int scan_terraform_line(const char *line, char *key, size_t ksz, char *val, size_t vsz) { - cbm_regmatch_t m[ENV_REGEX_MAX]; - if (cbm_regexec(&terraform_re, line, ENV_GRP_3, m, 0) != 0) { - return 0; - } - int vlen = (m[ENV_GRP_2].rm_eo - m[ENV_GRP_2].rm_so); - if (vlen <= 0 || vlen >= (int)vsz) { - return 0; - } - strncpy(key, "_tf_default", ksz - SKIP_ONE); - key[ksz - SKIP_ONE] = '\0'; - memcpy(val, line + m[ENV_GRP_2].rm_so, vlen); - val[vlen] = '\0'; - return SKIP_ONE; -} - -/* Try single-regex scan (shell, envfile, toml, properties). */ -static int scan_regex_line(cbm_regex_t *re, const char *line, int kg, int vg, char *key, size_t ksz, - char *val, size_t vsz) { - cbm_regmatch_t m[ENV_REGEX_MAX]; - if (cbm_regexec(re, line, ENV_GRP_5, m, 0) == 0 && - extract_kv_groups(line, m, kg, vg, key, ksz, val, vsz)) { - return SKIP_ONE; - } - return 0; -} - -static int scan_line(const char *line, file_type_t ft, char *key_out, size_t key_sz, char *val_out, - size_t val_sz) { - const char *trimmed = line; - while (*trimmed == ' ' || *trimmed == '\t') { - trimmed++; - } - if (*trimmed == '#' || (trimmed[0] == '/' && trimmed[SKIP_ONE] == '/')) { - return 0; - } - - switch (ft) { - case FT_DOCKERFILE: - return scan_dockerfile_line(trimmed, key_out, key_sz, val_out, val_sz); - case FT_YAML: - return scan_yaml_line(trimmed, key_out, key_sz, val_out, val_sz); - case FT_TERRAFORM: - return scan_terraform_line(trimmed, key_out, key_sz, val_out, val_sz); - case FT_SHELL: - return scan_regex_line(&shell_re, trimmed, ENV_GRP_2, ENV_GRP_3, key_out, key_sz, val_out, - val_sz); - case FT_ENVFILE: - return scan_regex_line(&envfile_re, trimmed, ENV_GRP_1, ENV_GRP_2, key_out, key_sz, val_out, - val_sz); - case FT_TOML: - return scan_regex_line(&toml_re, trimmed, ENV_GRP_1, ENV_GRP_2, key_out, key_sz, val_out, - val_sz); - case FT_PROPERTIES: - return scan_regex_line(&properties_re, trimmed, ENV_GRP_1, ENV_GRP_2, key_out, key_sz, - val_out, val_sz); - default: - return 0; - } -} - -/* ── Public API ────────────────────────────────────────────────── */ - -/* Scan a single file for env URL bindings. Returns number of bindings added. */ -static int scan_env_file(const char *full_path, const char *rel, file_type_t ft, - cbm_env_binding_t *out, int max_out) { - FILE *f = cbm_fopen(full_path, "r"); - if (!f) { - return 0; - } - - struct stat fst; - if (fstat(fileno(f), &fst) != 0 || fst.st_size > (long)CBM_SZ_1K * CBM_SZ_1K) { - (void)fclose(f); - return 0; - } - - int count = 0; - char line[CBM_SZ_2K]; - while (fgets(line, sizeof(line), f) && count < max_out) { - size_t ll = strlen(line); - while (ll > 0 && (line[ll - SKIP_ONE] == '\n' || line[ll - SKIP_ONE] == '\r')) { - line[--ll] = '\0'; - } - - char key[CBM_SZ_128]; - char value[CBM_SZ_512]; - if (!scan_line(line, ft, key, sizeof(key), value, sizeof(value))) { - continue; - } - if (strncmp(value, "http://", SLEN("http://")) != 0 && - strncmp(value, "https://", SLEN("https://")) != 0) { - continue; - } - if (cbm_is_secret_binding(key, value) || cbm_is_secret_value(value)) { - continue; - } - - strncpy(out[count].key, key, sizeof(out[count].key) - 1); - out[count].key[sizeof(out[count].key) - SKIP_ONE] = '\0'; - strncpy(out[count].value, value, sizeof(out[count].value) - 1); - out[count].value[sizeof(out[count].value) - SKIP_ONE] = '\0'; - strncpy(out[count].file_path, rel, sizeof(out[count].file_path) - 1); - out[count].file_path[sizeof(out[count].file_path) - SKIP_ONE] = '\0'; - count++; - } - (void)fclose(f); - return count; -} - -/* Process a single directory entry for env scanning. Returns bindings added. */ -static int process_env_entry(cbm_dirent_t *ent, const char *dir_path, const char *root_path, - cbm_env_binding_t *out, int max_out, char path_stack[][CBM_SZ_512], - int *stack_top, char **excluded_dirs, int excluded_count) { - char full_path[CBM_SZ_512]; - snprintf(full_path, sizeof(full_path), "%s/%s", dir_path, ent->name); - const char *rel = full_path + strlen(root_path); - while (*rel == '/') { - rel++; - } - - struct stat st; - if (stat(full_path, &st) != 0) { - return 0; - } - if (S_ISDIR(st.st_mode)) { - if (!is_ignored_dir(ent->name) && - !cbm_pipeline_relpath_is_excluded(rel, excluded_dirs, excluded_count) && - *stack_top < CBM_SZ_256) { - strncpy(path_stack[*stack_top], full_path, sizeof(path_stack[0]) - 1); - path_stack[*stack_top][sizeof(path_stack[0]) - SKIP_ONE] = '\0'; - (*stack_top)++; - } - return 0; - } - if (is_secret_file(ent->name)) { - return 0; - } - file_type_t ft = detect_file_type(ent->name); - if (ft == FT_UNKNOWN) { - return 0; - } - return scan_env_file(full_path, rel, ft, out, max_out); -} - -int cbm_scan_project_env_urls_excluded(const char *root_path, cbm_env_binding_t *out, int max_out, - char **excluded_dirs, int excluded_count) { - if (!root_path || !out || max_out <= 0) { - return 0; - } - compile_patterns(); - - int count = 0; - char path_stack[CBM_SZ_256][CBM_SZ_512]; - int stack_top = SKIP_ONE; - strncpy(path_stack[0], root_path, sizeof(path_stack[0]) - 1); - path_stack[0][sizeof(path_stack[0]) - SKIP_ONE] = '\0'; - - while (stack_top > 0 && count < max_out) { - stack_top--; - char dir_path[CBM_SZ_512]; - strncpy(dir_path, path_stack[stack_top], sizeof(dir_path) - SKIP_ONE); - dir_path[sizeof(dir_path) - SKIP_ONE] = '\0'; - - cbm_dir_t *d = cbm_opendir(dir_path); - if (!d) { - continue; - } - cbm_dirent_t *ent; - while ((ent = cbm_readdir(d)) && count < max_out) { - count += process_env_entry(ent, dir_path, root_path, out + count, max_out - count, - path_stack, &stack_top, excluded_dirs, excluded_count); - } - cbm_closedir(d); - } - return count; -} - -int cbm_scan_project_env_urls(const char *root_path, cbm_env_binding_t *out, int max_out) { - return cbm_scan_project_env_urls_excluded(root_path, out, max_out, NULL, 0); -} diff --git a/src/pipeline/pass_envscan.zig b/src/pipeline/pass_envscan.zig new file mode 100644 index 00000000..1d7be7cb --- /dev/null +++ b/src/pipeline/pass_envscan.zig @@ -0,0 +1,354 @@ +//! pass_envscan.zig — Environment URL scanner. +//! +//! Walks a project directory, scans config files (Dockerfile, .env, shell, +//! YAML, TOML, Terraform, .properties) for environment variable assignments +//! where the value is a URL. Filters out secrets. +//! +//! Port of internal/pipeline/envscan.go:ScanProjectEnvURLs(). + +const std = @import("std"); +const c = @import("c"); +const util = @import("util.zig"); + +const CStr = util.CStr; +const MutStr = util.MutStr; +const ix = util.ix; +const streq = util.streq; +const Regex = c.cbm_regex_t; +const Regmatch = c.cbm_regmatch_t; +const EnvBinding = c.cbm_env_binding_t; + +const ENV_REGEX_MAX = 5; +const ENV_EXT_LEN = 4; // strlen(".env") + +// ── Regex patterns (compiled lazily) ──────────────────────────────── + +var dockerfile_re: Regex = undefined; // ENV|ARG KEY=VALUE or KEY VALUE +var yaml_kv_re: Regex = undefined; // key: "https://..." +var yaml_setenv_re: Regex = undefined; // --set-env-vars KEY=VALUE +var terraform_re: Regex = undefined; // default|value = "https://..." +var shell_re: Regex = undefined; // [export] KEY=https://... +var envfile_re: Regex = undefined; // KEY=https://... +var toml_re: Regex = undefined; // key = "https://..." +var properties_re: Regex = undefined; // key=https://... +var patterns_once: util.Once = .{}; + +// POSIX ERE doesn't support \w or \S — use bracket expressions +const W = "[A-Za-z0-9_]"; // word char +const NW = "[^ \t\"']"; // non-whitespace, non-quote + +/// Compiled once per process; concurrent pipelines then only read them. +fn compilePatternsOnce() void { + const ext = c.CBM_REG_EXTENDED; + _ = c.cbm_regcomp(&dockerfile_re, "^(ENV|ARG)[[:space:]]+(" ++ W ++ "+)[= ](.*)", ext); + _ = c.cbm_regcomp(&yaml_kv_re, "(" ++ W ++ "+):[[:space:]]*[\"']?(https?://" ++ NW ++ "+)", ext); + _ = c.cbm_regcomp(&yaml_setenv_re, "--set-env-vars[[:space:]]+(" ++ W ++ "+)=([^ \t]+)", ext); + _ = c.cbm_regcomp(&terraform_re, "(default|value)[[:space:]]*=[[:space:]]*\"(https?://[^\"]+)\"", ext); + _ = c.cbm_regcomp(&shell_re, "(export[[:space:]]+)?(" ++ W ++ "+)=[\"']?(https?://" ++ NW ++ "+)", ext); + _ = c.cbm_regcomp(&envfile_re, "^(" ++ W ++ "+)=(https?://[^ \t]+)", ext); + _ = c.cbm_regcomp(&toml_re, "(" ++ W ++ "+)[[:space:]]*=[[:space:]]*\"(https?://[^\"]+)\"", ext); + _ = c.cbm_regcomp(&properties_re, "(" ++ W ++ "+)[[:space:]]*=[[:space:]]*(https?://[^ \t]+)", ext); +} + +fn compilePatterns() void { + patterns_once.call(compilePatternsOnce); +} + +// ── File type detection ───────────────────────────────────────────── + +/// name lowercased into buf (with its NUL); null when it does not fit. +fn lowerName(name: CStr, buf: *[c.CBM_SZ_256]u8) ?[:0]const u8 { + const len = c.strlen(name); + if (len >= buf.len) return null; + for (0..len + 1) |i| buf[i] = std.ascii.toLower(name[i]); + return buf[0..len :0]; +} + +const DOCKERFILE_SUFFIX_LEN = 11; // strlen("dockerfile.") == strlen(".dockerfile") + +fn isDockerfileName(name: CStr) bool { + // Case-insensitive check + var buf: [c.CBM_SZ_256]u8 = undefined; + const lower = lowerName(name, &buf) orelse return false; + + if (streq(lower, "dockerfile")) return true; + if (c.strncmp(lower, "dockerfile.", DOCKERFILE_SUFFIX_LEN) == 0) return true; + if (lower.len > DOCKERFILE_SUFFIX_LEN and + streq(lower.ptr + lower.len - DOCKERFILE_SUFFIX_LEN, ".dockerfile")) + { + return true; + } + return false; +} + +fn isEnvFileName(name: CStr) bool { + var buf: [c.CBM_SZ_256]u8 = undefined; + const lower = lowerName(name, &buf) orelse return false; + + if (streq(lower, ".env")) return true; + if (c.strncmp(lower, ".env.", ".env.".len) == 0) return true; + if (lower.len > ENV_EXT_LEN and streq(lower.ptr + lower.len - ENV_EXT_LEN, ".env")) return true; + return false; +} + +fn isSecretFile(name: CStr) bool { + var buf: [c.CBM_SZ_256]u8 = undefined; + const lower = lowerName(name, &buf) orelse return false; + + const patterns = [_][*:0]const u8{ + "service_account", "credentials", "key.json", "key.pem", "id_rsa", + "id_ed25519", ".pem", ".key", + }; + for (patterns) |pattern| { + if (c.strstr(lower, pattern) != null) return true; + } + return false; +} + +// ── Ignored directories ───────────────────────────────────────────── + +fn isIgnoredDir(name: CStr) bool { + const dirs = [_][*:0]const u8{ + ".git", "node_modules", ".svn", ".hg", "__pycache__", "vendor", ".terraform", ".cache", + ".idea", ".vscode", "dist", "build", ".next", ".nuxt", "target", + }; + for (dirs) |dir| { + if (streq(name, dir)) return true; + } + return false; +} + +// ── File type enum ────────────────────────────────────────────────── + +const FileType = enum { + unknown, + dockerfile, + yaml, + terraform, + shell, + envfile, + toml, + properties, +}; + +fn detectFileType(name: CStr) FileType { + if (isDockerfileName(name)) return .dockerfile; + if (isEnvFileName(name)) return .envfile; + + const ext: CStr = c.strrchr(name, '.'); + if (ext == null) return .unknown; + + if (streq(ext, ".yaml") or streq(ext, ".yml")) return .yaml; + if (streq(ext, ".tf") or streq(ext, ".hcl")) return .terraform; + if (streq(ext, ".sh") or streq(ext, ".bash") or streq(ext, ".zsh")) return .shell; + if (streq(ext, ".toml")) return .toml; + if (streq(ext, ".properties") or streq(ext, ".cfg") or streq(ext, ".ini")) return .properties; + + return .unknown; +} + +// ── Line scanner ──────────────────────────────────────────────────── + +/// Extract key/value from a regex match with two capture groups. +/// Returns true on success, false if groups are empty or too large. +fn extractKvGroups( + trimmed: [*:0]const u8, + m: *const [ENV_REGEX_MAX]Regmatch, + key_grp: usize, + val_grp: usize, + key_out: []u8, + val_out: []u8, +) bool { + const klen: c_int = @intCast(m[key_grp].rm_eo - m[key_grp].rm_so); + const vlen: c_int = @intCast(m[val_grp].rm_eo - m[val_grp].rm_so); + if (klen <= 0 or klen >= key_out.len or vlen <= 0 or vlen >= val_out.len) return false; + const key_start: usize = @intCast(m[key_grp].rm_so); + const val_start: usize = @intCast(m[val_grp].rm_so); + @memcpy(key_out[0..ix(klen)], trimmed[key_start..][0..ix(klen)]); + key_out[ix(klen)] = 0; + @memcpy(val_out[0..ix(vlen)], trimmed[val_start..][0..ix(vlen)]); + val_out[ix(vlen)] = 0; + return true; +} + +/// Try to scan a Dockerfile line. +fn scanDockerfileLine(line: [*:0]const u8, key: []u8, val: []u8) bool { + var m: [ENV_REGEX_MAX]Regmatch = undefined; + if (c.cbm_regexec(&dockerfile_re, line, 4, &m, 0) != 0) return false; + if (!extractKvGroups(line, &m, 2, 3, key, val)) return false; + var vl = c.strlen(val.ptr); + while (vl > 0 and (val[vl - 1] == '"' or val[vl - 1] == '\'')) { + vl -= 1; + val[vl] = 0; + } + return true; +} + +/// Try to scan a YAML line. +fn scanYamlLine(line: [*:0]const u8, key: []u8, val: []u8) bool { + var m: [ENV_REGEX_MAX]Regmatch = undefined; + if (c.cbm_regexec(&yaml_kv_re, line, 3, &m, 0) == 0 and extractKvGroups(line, &m, 1, 2, key, val)) { + return true; + } + if (c.cbm_regexec(&yaml_setenv_re, line, 3, &m, 0) == 0 and extractKvGroups(line, &m, 1, 2, key, val)) { + return true; + } + return false; +} + +/// Try to scan a Terraform line. +fn scanTerraformLine(line: [*:0]const u8, key: []u8, val: []u8) bool { + var m: [ENV_REGEX_MAX]Regmatch = undefined; + if (c.cbm_regexec(&terraform_re, line, 3, &m, 0) != 0) return false; + const vlen: c_int = @intCast(m[2].rm_eo - m[2].rm_so); + if (vlen <= 0 or vlen >= val.len) return false; + _ = c.strncpy(key.ptr, "_tf_default", key.len - 1); + key[key.len - 1] = 0; + const val_start: usize = @intCast(m[2].rm_so); + @memcpy(val[0..ix(vlen)], line[val_start..][0..ix(vlen)]); + val[ix(vlen)] = 0; + return true; +} + +/// Try single-regex scan (shell, envfile, toml, properties). +fn scanRegexLine(re: *Regex, line: [*:0]const u8, kg: usize, vg: usize, key: []u8, val: []u8) bool { + var m: [ENV_REGEX_MAX]Regmatch = undefined; + return c.cbm_regexec(re, line, 5, &m, 0) == 0 and extractKvGroups(line, &m, kg, vg, key, val); +} + +fn scanLine(line: [*:0]const u8, ft: FileType, key_out: []u8, val_out: []u8) bool { + var trimmed = line; + while (trimmed[0] == ' ' or trimmed[0] == '\t') trimmed += 1; + if (trimmed[0] == '#' or (trimmed[0] == '/' and trimmed[1] == '/')) return false; + + return switch (ft) { + .dockerfile => scanDockerfileLine(trimmed, key_out, val_out), + .yaml => scanYamlLine(trimmed, key_out, val_out), + .terraform => scanTerraformLine(trimmed, key_out, val_out), + .shell => scanRegexLine(&shell_re, trimmed, 2, 3, key_out, val_out), + .envfile => scanRegexLine(&envfile_re, trimmed, 1, 2, key_out, val_out), + .toml => scanRegexLine(&toml_re, trimmed, 1, 2, key_out, val_out), + .properties => scanRegexLine(&properties_re, trimmed, 1, 2, key_out, val_out), + .unknown => false, + }; +} + +// ── Public API ────────────────────────────────────────────────────── + +/// strncpy into a fixed buffer, always NUL-terminated. +fn copyField(dst: []u8, src: CStr) void { + _ = c.strncpy(dst.ptr, src, dst.len - 1); + dst[dst.len - 1] = 0; +} + +/// Scan a single file for env URL bindings. Returns number of bindings added. +fn scanEnvFile(full_path: CStr, rel: CStr, ft: FileType, out: [*]EnvBinding, max_out: c_int) c_int { + const f = c.cbm_fopen(full_path, "r") orelse return 0; + + var fst: c.struct_stat = undefined; + if (c.fstat(c.fileno(f), &fst) != 0 or fst.st_size > c.CBM_SZ_1K * c.CBM_SZ_1K) { + _ = c.fclose(f); + return 0; + } + + var count: c_int = 0; + var line: [c.CBM_SZ_2K:0]u8 = undefined; + while (count < max_out and c.fgets(&line, c.CBM_SZ_2K, f) != null) { + var ll = c.strlen(&line); + while (ll > 0 and (line[ll - 1] == '\n' or line[ll - 1] == '\r')) { + ll -= 1; + line[ll] = 0; + } + + var key: [c.CBM_SZ_128]u8 = undefined; + var value: [c.CBM_SZ_512]u8 = undefined; + if (!scanLine(&line, ft, &key, &value)) continue; + if (c.strncmp(&value, "http://", "http://".len) != 0 and + c.strncmp(&value, "https://", "https://".len) != 0) + { + continue; + } + if (c.cbm_is_secret_binding(&key, &value) or c.cbm_is_secret_value(&value)) continue; + + const binding = &out[ix(count)]; + copyField(&binding.key, &key); + copyField(&binding.value, &value); + copyField(&binding.file_path, rel); + count += 1; + } + _ = c.fclose(f); + return count; +} + +const PathStack = [c.CBM_SZ_256][c.CBM_SZ_512]u8; + +/// Process a single directory entry for env scanning. Returns bindings added. +fn processEnvEntry( + ent: *c.cbm_dirent_t, + dir_path: CStr, + root_path: CStr, + out: [*]EnvBinding, + max_out: c_int, + path_stack: *PathStack, + stack_top: *usize, + excluded_dirs: [*c]MutStr, + excluded_count: c_int, +) c_int { + var full_path: [c.CBM_SZ_512]u8 = undefined; + const name: CStr = &ent.name; + _ = c.snprintf(&full_path, full_path.len, "%s/%s", dir_path, name); + var rel: [*]const u8 = @as([*]const u8, &full_path) + c.strlen(root_path); + while (rel[0] == '/') rel += 1; + + var st: c.struct_stat = undefined; + if (c.stat(&full_path, &st) != 0) return 0; + if (util.isDirMode(&st)) { + if (!isIgnoredDir(name) and + c.cbm_pipeline_relpath_is_excluded(rel, excluded_dirs, excluded_count) == 0 and + stack_top.* < c.CBM_SZ_256) + { + copyField(&path_stack[stack_top.*], &full_path); + stack_top.* += 1; + } + return 0; + } + if (isSecretFile(name)) return 0; + const ft = detectFileType(name); + if (ft == .unknown) return 0; + return scanEnvFile(&full_path, rel, ft, out, max_out); +} + +pub export fn cbm_scan_project_env_urls_excluded( + root_path: CStr, + out: [*c]EnvBinding, + max_out: c_int, + excluded_dirs: [*c]MutStr, + excluded_count: c_int, +) c_int { + if (root_path == null or out == null or max_out <= 0) return 0; + compilePatterns(); + + var count: c_int = 0; + var path_stack: PathStack = undefined; + var stack_top: usize = 1; + copyField(&path_stack[0], root_path); + + while (stack_top > 0 and count < max_out) { + stack_top -= 1; + var dir_path: [c.CBM_SZ_512]u8 = undefined; + copyField(&dir_path, &path_stack[stack_top]); + + const d = c.cbm_opendir(&dir_path) orelse continue; + while (true) { + const ent = c.cbm_readdir(d) orelse break; + if (count >= max_out) break; + count += processEnvEntry(ent, &dir_path, root_path, @as([*]EnvBinding, out) + ix(count), max_out - count, &path_stack, &stack_top, excluded_dirs, excluded_count); + } + _ = c.cbm_closedir(d); + } + return count; +} + +pub export fn cbm_scan_project_env_urls(root_path: CStr, out: [*c]EnvBinding, max_out: c_int) c_int { + return cbm_scan_project_env_urls_excluded(root_path, out, max_out, null, 0); +} diff --git a/src/pipeline/pass_gitdiff.c b/src/pipeline/pass_gitdiff.c deleted file mode 100644 index f2b8fac6..00000000 --- a/src/pipeline/pass_gitdiff.c +++ /dev/null @@ -1,190 +0,0 @@ -/* - * pass_gitdiff.c — Git diff output parsing helpers. - * - * Pure string parsers for git diff --name-status and --unified=0 output. - * No git execution — just parsing pre-captured output strings. - */ -#include "foundation/constants.h" - -enum { GD_STATUS_IDX = 1, GD_PLUS_PREFIX = 6 }; - -#define SLEN(s) (sizeof(s) - 1) -#include "pipeline/pipeline.h" -#include "pipeline/pipeline_internal.h" - -#include -#include -#include - -void cbm_parse_range(const char *s, int *out_start, int *out_count) { - *out_start = 0; - *out_count = SKIP_ONE; - - const char *comma = strchr(s, ','); - if (!comma) { - *out_start = (int)strtol(s, NULL, CBM_DECIMAL_BASE); - return; - } - - /* Parse start */ - char buf[CBM_SZ_32]; - size_t len = (size_t)(comma - s); - if (len >= sizeof(buf)) { - len = sizeof(buf) - SKIP_ONE; - } - memcpy(buf, s, len); - buf[len] = '\0'; - *out_start = (int)strtol(buf, NULL, CBM_DECIMAL_BASE); - - /* Parse count */ - *out_count = (int)strtol(comma + SKIP_ONE, NULL, CBM_DECIMAL_BASE); -} - -#define HUNK_LINE_BUF 1536 - -/* Parse a single name-status line into a cbm_changed_file_t. - * Returns true if a valid entry was produced. */ -static bool parse_one_name_status(const char *line, size_t line_len, cbm_changed_file_t *out_f) { - char tmp[HUNK_LINE_BUF]; - if (line_len >= sizeof(tmp)) { - line_len = sizeof(tmp) - SKIP_ONE; - } - memcpy(tmp, line, line_len); - tmp[line_len] = '\0'; - - char *status_str = tmp; - char *tab1 = strchr(tmp, '\t'); - if (!tab1) { - return false; - } - *tab1 = '\0'; - char *path1 = tab1 + SKIP_ONE; - - char *tab2 = strchr(path1, '\t'); - char *path2 = NULL; - if (tab2) { - *tab2 = '\0'; - path2 = tab2 + SKIP_ONE; - } - - memset(out_f, 0, sizeof(*out_f)); - - if (status_str[0] == 'R') { - out_f->status[0] = 'R'; - out_f->status[GD_STATUS_IDX] = '\0'; - snprintf(out_f->old_path, sizeof(out_f->old_path), "%s", path1); - snprintf(out_f->path, sizeof(out_f->path), "%s", path2 ? path2 : path1); - } else { - out_f->status[0] = status_str[0]; - out_f->status[GD_STATUS_IDX] = '\0'; - snprintf(out_f->path, sizeof(out_f->path), "%s", path1); - out_f->old_path[0] = '\0'; - } - - return cbm_is_trackable_file(out_f->path); -} - -int cbm_parse_name_status(const char *output, cbm_changed_file_t *out, int max_out) { - if (!output || !out || max_out <= 0) { - return 0; - } - - int count = 0; - const char *line = output; - - while (*line && count < max_out) { - const char *eol = strchr(line, '\n'); - size_t line_len = eol ? (size_t)(eol - line) : strlen(line); - - if (line_len > 0) { - cbm_changed_file_t f; - if (parse_one_name_status(line, line_len, &f)) { - out[count++] = f; - } - } - - line = eol ? eol + SKIP_ONE : line + line_len; - } - return count; -} - -/* Parse a single @@ hunk header line and emit a hunk entry if valid. - * Returns true if a hunk was added. */ -static bool parse_hunk_line(const char *line, size_t line_len, const char *current_file, - cbm_changed_hunk_t *out_h) { - const char *plus = memchr(line, '+', line_len); - if (!plus || plus <= line) { - return false; - } - - const char *end_at = strstr(plus, " @@"); - size_t range_len; - if (end_at) { - range_len = (size_t)(end_at - plus - SKIP_ONE); - } else { - range_len = (size_t)(line + line_len - plus - SKIP_ONE); - } - - char range_str[CBM_SZ_64]; - if (range_len >= sizeof(range_str)) { - range_len = sizeof(range_str) - SKIP_ONE; - } - memcpy(range_str, plus + SKIP_ONE, range_len); - range_str[range_len] = '\0'; - - int start; - int cnt; - cbm_parse_range(range_str, &start, &cnt); - - if (start <= 0 || !cbm_is_trackable_file(current_file)) { - return false; - } - - int end = start + cnt - SKIP_ONE; - if (end < start) { - end = start; - } - - snprintf(out_h->path, sizeof(out_h->path), "%s", current_file); - out_h->start_line = start; - out_h->end_line = end; - return true; -} - -int cbm_parse_hunks(const char *output, cbm_changed_hunk_t *out, int max_out) { - if (!output || !out || max_out <= 0) { - return 0; - } - - int count = 0; - char current_file[CBM_SZ_512] = {0}; - const char *line = output; - - while (*line && count < max_out) { - const char *eol = strchr(line, '\n'); - size_t line_len = eol ? (size_t)(eol - line) : strlen(line); - - if (line_len == 0) { - line = eol ? eol + SKIP_ONE : line + line_len; - continue; - } - - if (line_len > GD_PLUS_PREFIX && strncmp(line, "+++ b/", SLEN("+++ b/")) == 0) { - size_t flen = line_len - GD_PLUS_PREFIX; - if (flen >= sizeof(current_file)) { - flen = sizeof(current_file) - SKIP_ONE; - } - memcpy(current_file, line + 6, flen); - current_file[flen] = '\0'; - } else if (line_len >= PAIR_LEN && line[0] == '@' && line[GD_STATUS_IDX] == '@' && - current_file[0]) { - cbm_changed_hunk_t h; - if (parse_hunk_line(line, line_len, current_file, &h)) { - out[count++] = h; - } - } - - line = eol ? eol + SKIP_ONE : line + line_len; - } - return count; -} diff --git a/src/pipeline/pass_gitdiff.zig b/src/pipeline/pass_gitdiff.zig new file mode 100644 index 00000000..a502bb37 --- /dev/null +++ b/src/pipeline/pass_gitdiff.zig @@ -0,0 +1,173 @@ +//! pass_gitdiff.zig — Git diff output parsing helpers. +//! +//! Pure string parsers for git diff --name-status and --unified=0 output. +//! No git execution — just parsing pre-captured output strings. + +const std = @import("std"); +const c = @import("c"); +const util = @import("util.zig"); + +const CStr = util.CStr; +const MutStr = util.MutStr; +const ChangedFile = c.cbm_changed_file_t; +const ChangedHunk = c.cbm_changed_hunk_t; + +const GD_STATUS_IDX = 1; +const GD_PLUS_PREFIX = 6; +const DECIMAL_BASE = c.CBM_DECIMAL_BASE; + +pub export fn cbm_parse_range(s: CStr, out_start: *c_int, out_count: *c_int) void { + out_start.* = 0; + out_count.* = 1; + + const comma: CStr = c.strchr(s, ','); + if (comma == null) { + out_start.* = @truncate(c.strtol(s, null, DECIMAL_BASE)); + return; + } + + // Parse start + var buf: [c.CBM_SZ_32]u8 = undefined; + var len = util.ptrDiff(comma, s); + if (len >= buf.len) len = buf.len - 1; + @memcpy(buf[0..len], s[0..len]); + buf[len] = 0; + out_start.* = @truncate(c.strtol(&buf, null, DECIMAL_BASE)); + + // Parse count + out_count.* = @truncate(c.strtol(comma + 1, null, DECIMAL_BASE)); +} + +const HUNK_LINE_BUF = 1536; + +/// Parse a single name-status line into a cbm_changed_file_t. +/// Returns true if a valid entry was produced. +fn parseOneNameStatus(line: [*]const u8, line_len_in: usize, out_f: *ChangedFile) bool { + var tmp: [HUNK_LINE_BUF]u8 = undefined; + var line_len = line_len_in; + if (line_len >= tmp.len) line_len = tmp.len - 1; + @memcpy(tmp[0..line_len], line[0..line_len]); + tmp[line_len] = 0; + + const status_str: [*]u8 = &tmp; + const tab1: MutStr = c.strchr(&tmp, '\t'); + if (tab1 == null) return false; + tab1[0] = 0; + const path1: MutStr = tab1 + 1; + + const tab2: MutStr = c.strchr(path1, '\t'); + var path2: MutStr = null; + if (tab2 != null) { + tab2[0] = 0; + path2 = tab2 + 1; + } + + out_f.* = std.mem.zeroes(ChangedFile); + + if (status_str[0] == 'R') { + out_f.status[0] = 'R'; + out_f.status[GD_STATUS_IDX] = 0; + _ = c.snprintf(&out_f.old_path, out_f.old_path.len, "%s", path1); + _ = c.snprintf(&out_f.path, out_f.path.len, "%s", if (path2 != null) path2 else path1); + } else { + out_f.status[0] = status_str[0]; + out_f.status[GD_STATUS_IDX] = 0; + _ = c.snprintf(&out_f.path, out_f.path.len, "%s", path1); + out_f.old_path[0] = 0; + } + + return c.cbm_is_trackable_file(&out_f.path); +} + +pub export fn cbm_parse_name_status(output: CStr, out: [*c]ChangedFile, max_out: c_int) c_int { + if (output == null or out == null or max_out <= 0) return 0; + + var count: c_int = 0; + var line: [*]const u8 = output; + + while (line[0] != 0 and count < max_out) { + const eol: CStr = c.strchr(line, '\n'); + const line_len: usize = if (eol != null) util.ptrDiff(eol, line) else c.strlen(line); + + if (line_len > 0) { + var f: ChangedFile = undefined; + if (parseOneNameStatus(line, line_len, &f)) { + out[util.ix(count)] = f; + count += 1; + } + } + + line = if (eol != null) eol + 1 else line + line_len; + } + return count; +} + +/// Parse a single @@ hunk header line and emit a hunk entry if valid. +/// Returns true if a hunk was added. +fn parseHunkLine(line: [*]const u8, line_len: usize, current_file: CStr, out_h: *ChangedHunk) bool { + const plus: CStr = @ptrCast(c.memchr(line, '+', line_len)); + if (plus == null or @intFromPtr(plus) <= @intFromPtr(line)) return false; + + const end_at: CStr = c.strstr(plus, " @@"); + var range_len: usize = undefined; + if (end_at != null) { + range_len = util.ptrDiff(end_at, plus) - 1; + } else { + range_len = util.ptrDiff(line + line_len, plus) - 1; + } + + var range_str: [c.CBM_SZ_64]u8 = undefined; + if (range_len >= range_str.len) range_len = range_str.len - 1; + @memcpy(range_str[0..range_len], (plus + 1)[0..range_len]); + range_str[range_len] = 0; + + var start: c_int = undefined; + var cnt: c_int = undefined; + cbm_parse_range(&range_str, &start, &cnt); + + if (start <= 0 or !c.cbm_is_trackable_file(current_file)) return false; + + var end = start +% cnt -% 1; + if (end < start) end = start; + + _ = c.snprintf(&out_h.path, out_h.path.len, "%s", current_file); + out_h.start_line = start; + out_h.end_line = end; + return true; +} + +pub export fn cbm_parse_hunks(output: CStr, out: [*c]ChangedHunk, max_out: c_int) c_int { + if (output == null or out == null or max_out <= 0) return 0; + + var count: c_int = 0; + var current_file: [c.CBM_SZ_512]u8 = @splat(0); + var line: [*]const u8 = output; + + while (line[0] != 0 and count < max_out) { + const eol: CStr = c.strchr(line, '\n'); + const line_len: usize = if (eol != null) util.ptrDiff(eol, line) else c.strlen(line); + + if (line_len == 0) { + line = if (eol != null) eol + 1 else line + line_len; + continue; + } + + if (line_len > GD_PLUS_PREFIX and c.strncmp(line, "+++ b/", "+++ b/".len) == 0) { + var flen = line_len - GD_PLUS_PREFIX; + if (flen >= current_file.len) flen = current_file.len - 1; + @memcpy(current_file[0..flen], (line + 6)[0..flen]); + current_file[flen] = 0; + } else if (line_len >= 2 and line[0] == '@' and line[GD_STATUS_IDX] == '@' and + current_file[0] != 0) + { + var h: ChangedHunk = undefined; + if (parseHunkLine(line, line_len, ¤t_file, &h)) { + out[util.ix(count)] = h; + count += 1; + } + } + + line = if (eol != null) eol + 1 else line + line_len; + } + return count; +} diff --git a/src/pipeline/pass_githistory.c b/src/pipeline/pass_githistory.c deleted file mode 100644 index e4d8fe85..00000000 --- a/src/pipeline/pass_githistory.c +++ /dev/null @@ -1,512 +0,0 @@ -/* - * pass_githistory.c — Analyze git log to find change coupling. - * - * Runs `git log --name-only --since=6 months ago` and computes - * file pairs that change together frequently. Creates FILE_CHANGES_WITH - * edges between File nodes with coupling_score properties. - * - * Skips commits with >20 files (refactoring/merge noise). - * Requires minimum 3 co-changes for an edge. - * - * Depends on: pass_structure having created File nodes - */ -#include "foundation/constants.h" - -enum { GH_INIT_CAP = 16, GH_MIN_COMMITS = 3, GH_MAX_FILES = 20 }; - -#define SLEN(s) (sizeof(s) - 1) -#include "pipeline/pipeline.h" -#include "pipeline/pipeline_internal.h" -#include "graph_buffer/graph_buffer.h" -#include "foundation/hash_table.h" -#include "foundation/log.h" -#include "foundation/platform.h" -#include "foundation/compat.h" -#include "foundation/compat_fs.h" -#include "foundation/str_util.h" -#include "yyjson/yyjson.h" - -/* Minimum coupling score to create an edge */ -#define MIN_COUPLING_SCORE 0.3 - -#include -#include -#include - -bool cbm_is_trackable_file(const char *path) { - if (!path) { - return false; - } - /* Skip directory prefixes */ -#define LEN_NODE_MODULES_SLASH 13 /* strlen("node_modules/") */ - if (strncmp(path, ".git/", SLEN(".git/")) == 0 || - strncmp(path, "node_modules/", LEN_NODE_MODULES_SLASH) == 0 || - strncmp(path, "vendor/", SLEN("vendor/")) == 0 || - strncmp(path, "__pycache__/", SLEN("__pycache__/")) == 0 || - strncmp(path, ".cache/", SLEN(".cache/")) == 0) { - return false; - } - /* Skip lock/generated file names */ - const char *base = strrchr(path, '/'); - base = base ? base + SKIP_ONE : path; - if (strcmp(base, "package-lock.json") == 0 || strcmp(base, "yarn.lock") == 0 || - strcmp(base, "pnpm-lock.yaml") == 0 || strcmp(base, "Cargo.lock") == 0 || - strcmp(base, "poetry.lock") == 0 || strcmp(base, "composer.lock") == 0 || - strcmp(base, "Gemfile.lock") == 0 || strcmp(base, "Pipfile.lock") == 0) { - return false; - } - /* Skip non-source file extensions */ - if (cbm_str_ends_with(path, ".lock") || cbm_str_ends_with(path, ".sum") || - cbm_str_ends_with(path, ".min.js") || cbm_str_ends_with(path, ".min.css") || - cbm_str_ends_with(path, ".map") || cbm_str_ends_with(path, ".wasm") || - cbm_str_ends_with(path, ".png") || cbm_str_ends_with(path, ".jpg") || - cbm_str_ends_with(path, ".gif") || cbm_str_ends_with(path, ".ico") || - cbm_str_ends_with(path, ".svg")) { - return false; - } - return true; -} - -/* ── Commit parsing ───────────────────────────────────────────────── */ - -typedef struct { - char **files; - int count; - int cap; - long long timestamp; /* unix epoch of this commit; 0 when unknown */ -} commit_t; - -static void commit_add_file(commit_t *c, const char *file) { - if (c->count >= c->cap) { - c->cap = c->cap ? c->cap * PAIR_LEN : GH_INIT_CAP; - c->files = safe_realloc(c->files, c->cap * sizeof(char *)); - } - c->files[c->count++] = strdup(file); -} - -static void commit_free(commit_t *c) { - for (int i = 0; i < c->count; i++) { - free(c->files[i]); - } - free(c->files); -} - -/* ── git log parsing (popen "git log") ────────────────────────────── */ - -static int parse_git_log(const char *repo_path, commit_t **out, int *out_count) { - *out = NULL; - *out_count = 0; - - if (!cbm_validate_shell_arg(repo_path)) { - return CBM_NOT_FOUND; - } - - char cmd[CBM_SZ_1K]; -#ifdef _WIN32 - /* cmd.exe does not recognize single quotes, and '/dev/null' is a POSIX path. */ - const char *null_dev = "NUL"; -#else - const char *null_dev = "/dev/null"; -#endif - /* git -C "" works on both cmd.exe and POSIX shells. Double quotes are - * safe here because cbm_validate_shell_arg (above) rejects ", $, `, \ and the - * other shell metacharacters that would otherwise be active inside them. */ - snprintf(cmd, sizeof(cmd), - "git -C \"%s\" log --name-only --pretty=format:COMMIT:%%H:%%ct " - "--since=\"1 year ago\" --max-count=10000 2>%s", - repo_path, null_dev); - - FILE *fp = cbm_popen_git(cmd); - if (!fp) { - return CBM_NOT_FOUND; - } - - int cap = CBM_SZ_64; - commit_t *commits = malloc(cap * sizeof(commit_t)); - int count = 0; - commit_t current = {0}; - - char line[CBM_SZ_1K]; - while (fgets(line, sizeof(line), fp)) { - size_t len = strlen(line); - while (len > 0 && (line[len - SKIP_ONE] == '\n' || line[len - SKIP_ONE] == '\r')) { - line[--len] = '\0'; - } - if (len == 0) { - continue; - } - - if (strncmp(line, "COMMIT:", SLEN("COMMIT:")) == 0) { - if (current.count > 0) { - if (count >= cap) { - cap *= PAIR_LEN; - commits = safe_realloc(commits, cap * sizeof(commit_t)); - } - commits[count++] = current; - memset(¤t, 0, sizeof(current)); - } - /* Parse the unix timestamp from "COMMIT::". - * Older callers / stripped-down git output without %ct land on 0. */ - const char *hash_end = strchr(line + SLEN("COMMIT:"), ':'); - if (hash_end) { - current.timestamp = strtoll(hash_end + 1, NULL, 10); - } - continue; - } - - if (cbm_is_trackable_file(line)) { - commit_add_file(¤t, line); - } - } - if (current.count > 0) { - if (count >= cap) { - cap *= PAIR_LEN; - commits = safe_realloc(commits, cap * sizeof(commit_t)); - } - commits[count++] = current; - } else { - commit_free(¤t); - } - - cbm_pclose(fp); - *out = commits; - *out_count = count; - return 0; -} - -/* Callback to free hash table entries. */ -static void free_counter(const char *key, void *val, void *ud) { - (void)ud; - safe_str_free(&key); - free(val); -} - -/* ── Standalone coupling computation (testable) ──────────────────── */ - -/* Context for collect_coupling_result callback. */ -typedef struct { - CBMHashTable *file_counts; - CBMHashTable *pair_timestamps; /* pair_key → long long*: max commit ts */ - cbm_change_coupling_t *out; - int out_count; - int max_out; -} collect_coupling_ctx_t; - -static void collect_coupling_cb(const char *pair_key, void *val, void *ud) { - collect_coupling_ctx_t *cctx = ud; - int co_count = *(int *)val; - if (co_count < GH_MIN_COMMITS) { - return; - } - if (cctx->out_count >= cctx->max_out) { - return; - } - - const char *sep = strchr(pair_key, '\x01'); - if (!sep) { - return; - } - size_t la = sep - pair_key; - const char *file_b = sep + SKIP_ONE; - - char file_a_buf[CBM_SZ_512]; - if (la >= sizeof(file_a_buf)) { - return; - } - memcpy(file_a_buf, pair_key, la); - file_a_buf[la] = '\0'; - - int *count_a = cbm_ht_get(cctx->file_counts, file_a_buf); - int *count_b = cbm_ht_get(cctx->file_counts, file_b); - if (!count_a || !count_b) { - return; - } - - int min_total = *count_a < *count_b ? *count_a : *count_b; - if (min_total == 0) { - return; - } - - double score = (double)co_count / (double)min_total; - if (score < MIN_COUPLING_SCORE) { - return; - } - - cbm_change_coupling_t *cc = &cctx->out[cctx->out_count++]; - snprintf(cc->file_a, sizeof(cc->file_a), "%s", file_a_buf); - snprintf(cc->file_b, sizeof(cc->file_b), "%s", file_b); - cc->co_change_count = co_count; - cc->coupling_score = score; - long long *ts = cbm_ht_get(cctx->pair_timestamps, pair_key); - cc->last_co_change = ts ? *ts : 0; -} - -int cbm_compute_change_coupling(const cbm_commit_files_t *commits, int commit_count, - cbm_change_coupling_t *out, int max_out) { - CBMHashTable *file_counts = cbm_ht_create(CBM_SZ_1K); - CBMHashTable *pair_counts = cbm_ht_create(CBM_SZ_2K); - /* Parallel table mapping pair_key → max commit timestamp seen for that - * pair, so the resulting edge can carry last_co_change. The pair_counts - * table consumes its key on insert; pair_timestamps gets its own copy. */ - CBMHashTable *pair_timestamps = cbm_ht_create(CBM_SZ_2K); - - for (int c = 0; c < commit_count; c++) { - if (commits[c].count > GH_MAX_FILES) { - continue; - } - - for (int i = 0; i < commits[c].count; i++) { - int *val = cbm_ht_get(file_counts, commits[c].files[i]); - if (val) { - (*val)++; - } else { - int *nv = malloc(sizeof(int)); - *nv = SKIP_ONE; - cbm_ht_set(file_counts, strdup(commits[c].files[i]), nv); - } - } - - for (int i = 0; i < commits[c].count; i++) { - for (int j = i + SKIP_ONE; j < commits[c].count; j++) { - const char *a = commits[c].files[i]; - const char *b = commits[c].files[j]; - if (strcmp(a, b) > 0) { - const char *t = a; - a = b; - b = t; - } - size_t la = strlen(a); - size_t lb = strlen(b); - size_t pk_len = la + SKIP_ONE + lb + SKIP_ONE; - char *pk = malloc(pk_len); - memcpy(pk, a, la); - pk[la] = '\x01'; - memcpy(pk + la + SKIP_ONE, b, lb + SKIP_ONE); - - int *val = cbm_ht_get(pair_counts, pk); - if (val) { - (*val)++; - long long *ts = cbm_ht_get(pair_timestamps, pk); - if (ts && commits[c].timestamp > *ts) { - *ts = commits[c].timestamp; - } - free(pk); - } else { - int *nv = malloc(sizeof(int)); - *nv = SKIP_ONE; - /* pair_counts takes ownership of pk; pair_timestamps - * needs its own copy. */ - char *pk2 = malloc(pk_len); - memcpy(pk2, pk, pk_len); - cbm_ht_set(pair_counts, pk, nv); - long long *nts = malloc(sizeof(long long)); - *nts = commits[c].timestamp; - cbm_ht_set(pair_timestamps, pk2, nts); - } - } - } - } - - collect_coupling_ctx_t cctx = { - .file_counts = file_counts, - .pair_timestamps = pair_timestamps, - .out = out, - .out_count = 0, - .max_out = max_out, - }; - cbm_ht_foreach(pair_counts, collect_coupling_cb, &cctx); - - cbm_ht_foreach(pair_counts, free_counter, NULL); - cbm_ht_free(pair_counts); - cbm_ht_foreach(pair_timestamps, free_counter, NULL); - cbm_ht_free(pair_timestamps); - cbm_ht_foreach(file_counts, free_counter, NULL); - cbm_ht_free(file_counts); - - return cctx.out_count; -} - -/* ── Split pass: compute (I/O-bound) + apply (gbuf writes) ───────── */ - -/* Pre-computed coupling result buffer for fused post-pass parallelism. */ -#define MAX_COUPLINGS 8192 -#define MAX_FILE_TEMPORAL 16384 - -/* Compute change couplings without touching the graph buffer. - * Can run on a separate thread while other passes use the gbuf. */ -int cbm_pipeline_githistory_compute(const char *repo_path, cbm_githistory_result_t *result) { - result->couplings = NULL; - result->count = 0; - result->commit_count = 0; - result->file_temporal = NULL; - result->file_temporal_count = 0; - - commit_t *commits = NULL; - int commit_count = 0; - int rc = parse_git_log(repo_path, &commits, &commit_count); - if (rc != 0 || commit_count == 0) { - free(commits); - return 0; - } - - result->commit_count = commit_count; - - /* Convert to testable format */ - cbm_commit_files_t *cf = calloc((size_t)commit_count, sizeof(cbm_commit_files_t)); - if (!cf) { - for (int c = 0; c < commit_count; c++) { - commit_free(&commits[c]); - } - free(commits); - return 0; - } - for (int c = 0; c < commit_count; c++) { - cf[c].files = commits[c].files; - cf[c].count = commits[c].count; - cf[c].timestamp = commits[c].timestamp; - } - - cbm_change_coupling_t *couplings = malloc(MAX_COUPLINGS * sizeof(cbm_change_coupling_t)); - int coupling_count = cbm_compute_change_coupling(cf, commit_count, couplings, MAX_COUPLINGS); - - /* Per-file temporal aggregation: change_count + last_modified. - * Single hash-table pass over the same commit set used for coupling so - * we don't re-scan history. NULL on OOM is fine — the caller still - * gets the couplings. */ - cbm_file_temporal_t *ft_arr = malloc(MAX_FILE_TEMPORAL * sizeof(cbm_file_temporal_t)); - if (ft_arr) { - int ft_count = 0; - CBMHashTable *file_idx = cbm_ht_create(CBM_SZ_1K); - for (int c = 0; c < commit_count; c++) { - if (cf[c].count > GH_MAX_FILES) { - continue; - } - for (int f = 0; f < cf[c].count; f++) { - const char *fp = cf[c].files[f]; - int *idx = cbm_ht_get(file_idx, fp); - if (idx) { - ft_arr[*idx].change_count++; - if (cf[c].timestamp > ft_arr[*idx].last_modified) { - ft_arr[*idx].last_modified = cf[c].timestamp; - } - } else if (ft_count < MAX_FILE_TEMPORAL) { - int new_idx = ft_count++; - snprintf(ft_arr[new_idx].file_path, sizeof(ft_arr[new_idx].file_path), "%s", - fp); - ft_arr[new_idx].change_count = 1; - ft_arr[new_idx].last_modified = cf[c].timestamp; - int *nidx = malloc(sizeof(int)); - *nidx = new_idx; - cbm_ht_set(file_idx, strdup(fp), nidx); - } - } - } - cbm_ht_foreach(file_idx, free_counter, NULL); - cbm_ht_free(file_idx); - result->file_temporal = ft_arr; - result->file_temporal_count = ft_count; - } - - free(cf); - for (int c = 0; c < commit_count; c++) { - commit_free(&commits[c]); - } - free(commits); - - result->couplings = couplings; - result->count = coupling_count; - return 0; -} - -/* Apply pre-computed couplings to the graph buffer (must be on main thread). */ -int cbm_pipeline_githistory_apply(cbm_pipeline_ctx_t *ctx, const cbm_githistory_result_t *result) { - int edge_count = 0; - - for (int i = 0; i < result->count; i++) { - const cbm_change_coupling_t *cc = &result->couplings[i]; - - char *qn_a = cbm_pipeline_fqn_compute(ctx->project_name, cc->file_a, "__file__"); - char *qn_b = cbm_pipeline_fqn_compute(ctx->project_name, cc->file_b, "__file__"); - - const cbm_gbuf_node_t *node_a = cbm_gbuf_find_by_qn(ctx->gbuf, qn_a); - const cbm_gbuf_node_t *node_b = cbm_gbuf_find_by_qn(ctx->gbuf, qn_b); - - free(qn_a); - free(qn_b); - - if (!node_a || !node_b || node_a->id == node_b->id) { - continue; - } - - char props[CBM_SZ_128]; - snprintf(props, sizeof(props), - "{\"co_changes\":%d,\"coupling_score\":%.2f,\"last_co_change\":%lld}", - cc->co_change_count, cc->coupling_score, cc->last_co_change); - - cbm_gbuf_insert_edge(ctx->gbuf, node_a->id, node_b->id, "FILE_CHANGES_WITH", props); - edge_count++; - } - - /* Apply per-file temporal metadata to existing File nodes so callers - * can query change_count / last_modified for hotspot analysis. The keys - * are merged into the node's properties: the coverage stamp written - * after extraction lives there too, and replacing the blob dropped it. */ - for (int i = 0; i < result->file_temporal_count; i++) { - const cbm_file_temporal_t *ft = &result->file_temporal[i]; - char *qn = cbm_pipeline_fqn_compute(ctx->project_name, ft->file_path, "__file__"); - const cbm_gbuf_node_t *node = cbm_gbuf_find_by_qn(ctx->gbuf, qn); - if (!node) { - free(qn); - continue; - } - - const char *base = strrchr(ft->file_path, '/'); - base = base ? base + SKIP_ONE : ft->file_path; - const char *ext = strrchr(base, '.'); - - const char *old = node->properties_json ? node->properties_json : "{}"; - yyjson_doc *odoc = yyjson_read(old, strlen(old), 0); - yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); - yyjson_mut_val *root = odoc && yyjson_is_obj(yyjson_doc_get_root(odoc)) - ? yyjson_val_mut_copy(doc, yyjson_doc_get_root(odoc)) - : yyjson_mut_obj(doc); - yyjson_mut_doc_set_root(doc, root); - yyjson_mut_obj_remove_key(root, "extension"); - yyjson_mut_obj_remove_key(root, "last_modified"); - yyjson_mut_obj_remove_key(root, "change_count"); - yyjson_mut_obj_add_strcpy(doc, root, "extension", ext ? ext : ""); - yyjson_mut_obj_add_sint(doc, root, "last_modified", (int64_t)ft->last_modified); - yyjson_mut_obj_add_int(doc, root, "change_count", ft->change_count); - char *props = yyjson_mut_write(doc, 0, NULL); - yyjson_mut_doc_free(doc); - yyjson_doc_free(odoc); - if (props) { - cbm_gbuf_set_node_props(ctx->gbuf, qn, props); - free(props); - } - free(qn); - } - - return edge_count; -} - -/* ── Main pass (original serial interface) ───────────────────────── */ - -int cbm_pipeline_pass_githistory(cbm_pipeline_ctx_t *ctx) { - cbm_log_info("pass.start", "pass", "githistory"); - - cbm_githistory_result_t result = {0}; - cbm_pipeline_githistory_compute(ctx->repo_path, &result); - - int edge_count = 0; - if (result.count > 0 || result.file_temporal_count > 0) { - edge_count = cbm_pipeline_githistory_apply(ctx, &result); - } - - free(result.couplings); - free(result.file_temporal); - - cbm_log_info("pass.done", "pass", "githistory", "commits", cbm_log_itoa(result.commit_count), - "edges", cbm_log_itoa(edge_count)); - return 0; -} diff --git a/src/pipeline/pass_githistory.zig b/src/pipeline/pass_githistory.zig new file mode 100644 index 00000000..50e526b2 --- /dev/null +++ b/src/pipeline/pass_githistory.zig @@ -0,0 +1,507 @@ +//! pass_githistory.zig — Analyze git log to find change coupling. +//! +//! Runs `git log --name-only --since=1 year ago` and computes +//! file pairs that change together frequently. Creates FILE_CHANGES_WITH +//! edges between File nodes with coupling_score properties. +//! +//! Skips commits with >20 files (refactoring/merge noise). +//! Requires minimum 3 co-changes for an edge. +//! +//! Depends on: pass_structure having created File nodes + +const std = @import("std"); +const builtin = @import("builtin"); +const c = @import("c"); +const interop = @import("../foundation/interop.zig"); +const util = @import("util.zig"); + +const CStr = util.CStr; +const MutStr = util.MutStr; +const ix = util.ix; +const streq = util.streq; +const Node = c.cbm_gbuf_node_t; +const HashTable = c.CBMHashTable; +const Coupling = c.cbm_change_coupling_t; +const FileTemporal = c.cbm_file_temporal_t; + +const GH_INIT_CAP = 16; +const GH_MIN_COMMITS = 3; +const GH_MAX_FILES = 20; + +/// Minimum coupling score to create an edge +const MIN_COUPLING_SCORE: f64 = 0.3; + +fn hasPrefix(s: CStr, comptime prefix: [:0]const u8) bool { + return c.strncmp(s, prefix, prefix.len) == 0; +} + +pub export fn cbm_is_trackable_file(path: CStr) bool { + if (path == null) return false; + // Skip directory prefixes + if (hasPrefix(path, ".git/") or hasPrefix(path, "node_modules/") or + hasPrefix(path, "vendor/") or hasPrefix(path, "__pycache__/") or + hasPrefix(path, ".cache/")) + { + return false; + } + // Skip lock/generated file names + var base: CStr = c.strrchr(path, '/'); + base = if (base != null) base + 1 else path; + if (streq(base, "package-lock.json") or streq(base, "yarn.lock") or + streq(base, "pnpm-lock.yaml") or streq(base, "Cargo.lock") or + streq(base, "poetry.lock") or streq(base, "composer.lock") or + streq(base, "Gemfile.lock") or streq(base, "Pipfile.lock")) + { + return false; + } + // Skip non-source file extensions + const skipped = [_][*:0]const u8{ + ".lock", ".sum", ".min.js", ".min.css", ".map", ".wasm", ".png", ".jpg", ".gif", ".ico", ".svg", + }; + for (skipped) |suffix| { + if (c.cbm_str_ends_with(path, suffix)) return false; + } + return true; +} + +// ── Commit parsing ────────────────────────────────────────────────── + +const Commit = extern struct { + files: [*c]MutStr, + count: c_int, + cap: c_int, + timestamp: c_longlong, // unix epoch of this commit; 0 when unknown +}; + +fn commitAddFile(commit: *Commit, file: CStr) void { + if (commit.count >= commit.cap) { + commit.cap = if (commit.cap != 0) commit.cap * 2 else GH_INIT_CAP; + commit.files = interop.safeReallocN(MutStr, commit.files, ix(commit.cap)); + } + commit.files[ix(commit.count)] = c.strdup(file); + commit.count += 1; +} + +fn commitFree(commit: *Commit) void { + for (0..ix(commit.count)) |i| c.free(commit.files[i]); + c.free(@ptrCast(commit.files)); +} + +// ── git log parsing (popen "git log") ─────────────────────────────── + +/// Appends `current` to the commit array, doubling it when full. +fn pushCommit(commits: *[*c]Commit, count: *c_int, cap: *c_int, current: Commit) void { + if (count.* >= cap.*) { + cap.* *= 2; + commits.* = interop.safeReallocN(Commit, commits.*, ix(cap.*)); + } + commits.*[ix(count.*)] = current; + count.* += 1; +} + +fn parseGitLog(repo_path: CStr, out: *[*c]Commit, out_count: *c_int) c_int { + out.* = null; + out_count.* = 0; + + if (!c.cbm_validate_shell_arg(repo_path)) return c.CBM_NOT_FOUND; + + var cmd: [c.CBM_SZ_1K]u8 = undefined; + // cmd.exe does not recognize single quotes, and '/dev/null' is a POSIX path. + const null_dev: CStr = if (builtin.os.tag == .windows) "NUL" else "/dev/null"; + // git -C "" works on both cmd.exe and POSIX shells. Double quotes are + // safe here because cbm_validate_shell_arg (above) rejects ", $, `, \ and the + // other shell metacharacters that would otherwise be active inside them. + _ = c.snprintf( + &cmd, + cmd.len, + "git -C \"%s\" log --name-only --pretty=format:COMMIT:%%H:%%ct " ++ + "--since=\"1 year ago\" --max-count=10000 2>%s", + repo_path, + null_dev, + ); + + const fp = c.cbm_popen_git(&cmd) orelse return c.CBM_NOT_FOUND; + + var cap: c_int = c.CBM_SZ_64; + var commits: [*c]Commit = interop.mallocN(Commit, ix(cap)); + var count: c_int = 0; + var current = std.mem.zeroes(Commit); + + var line: [c.CBM_SZ_1K]u8 = undefined; + while (c.fgets(&line, line.len, fp) != null) { + var len = c.strlen(&line); + while (len > 0 and (line[len - 1] == '\n' or line[len - 1] == '\r')) { + len -= 1; + line[len] = 0; + } + if (len == 0) continue; + + if (hasPrefix(&line, "COMMIT:")) { + if (current.count > 0) { + pushCommit(&commits, &count, &cap, current); + current = std.mem.zeroes(Commit); + } + // Parse the unix timestamp from "COMMIT::". + // Older callers / stripped-down git output without %ct land on 0. + const hash_end: CStr = c.strchr(@as(CStr, &line) + "COMMIT:".len, ':'); + if (hash_end != null) { + current.timestamp = c.strtoll(hash_end + 1, null, 10); + } + continue; + } + + if (cbm_is_trackable_file(&line)) { + commitAddFile(¤t, &line); + } + } + if (current.count > 0) { + pushCommit(&commits, &count, &cap, current); + } else { + commitFree(¤t); + } + + _ = c.cbm_pclose(fp); + out.* = commits; + out_count.* = count; + return 0; +} + +/// Callback to free hash table entries. +fn freeCounter(key: CStr, val: ?*anyopaque, ud: ?*anyopaque) callconv(.c) void { + _ = ud; + c.free(@constCast(key)); + c.free(val); +} + +// ── Standalone coupling computation (testable) ────────────────────── + +/// Context for collectCouplingCb callback. +const CollectCouplingCtx = struct { + file_counts: ?*HashTable, + pair_timestamps: ?*HashTable, // pair_key → long long*: max commit ts + out: [*c]Coupling, + out_count: c_int, + max_out: c_int, +}; + +fn intAt(table: ?*HashTable, key: CStr) ?*c_int { + return @ptrCast(@alignCast(c.cbm_ht_get(table, key))); +} + +fn timestampAt(table: ?*HashTable, key: CStr) ?*c_longlong { + return @ptrCast(@alignCast(c.cbm_ht_get(table, key))); +} + +/// A malloc'd int holding `value`, as the hash tables here store them. +fn newInt(value: c_int) *c_int { + const nv: *c_int = interop.mallocN(c_int, 1); + nv.* = value; + return nv; +} + +fn collectCouplingCb(pair_key: CStr, val: ?*anyopaque, ud: ?*anyopaque) callconv(.c) void { + const cctx: *CollectCouplingCtx = @ptrCast(@alignCast(ud)); + const co_count = @as(*c_int, @ptrCast(@alignCast(val))).*; + if (co_count < GH_MIN_COMMITS) return; + if (cctx.out_count >= cctx.max_out) return; + + const sep: CStr = c.strchr(pair_key, 0x01); + if (sep == null) return; + const la = util.ptrDiff(sep, pair_key); + const file_b: CStr = sep + 1; + + var file_a_buf: [c.CBM_SZ_512]u8 = undefined; + if (la >= file_a_buf.len) return; + @memcpy(file_a_buf[0..la], pair_key[0..la]); + file_a_buf[la] = 0; + + const count_a = intAt(cctx.file_counts, &file_a_buf) orelse return; + const count_b = intAt(cctx.file_counts, file_b) orelse return; + + const min_total = @min(count_a.*, count_b.*); + if (min_total == 0) return; + + const score = @as(f64, @floatFromInt(co_count)) / @as(f64, @floatFromInt(min_total)); + if (score < MIN_COUPLING_SCORE) return; + + const cc = util.at(cctx.out, ix(cctx.out_count)); + cctx.out_count += 1; + _ = c.snprintf(&cc.file_a, cc.file_a.len, "%s", @as(CStr, &file_a_buf)); + _ = c.snprintf(&cc.file_b, cc.file_b.len, "%s", file_b); + cc.co_change_count = co_count; + cc.coupling_score = score; + const ts = timestampAt(cctx.pair_timestamps, pair_key); + cc.last_co_change = if (ts) |t| t.* else 0; +} + +pub export fn cbm_compute_change_coupling( + commits: [*c]const c.cbm_commit_files_t, + commit_count: c_int, + out: [*c]Coupling, + max_out: c_int, +) c_int { + const file_counts = c.cbm_ht_create(c.CBM_SZ_1K); + const pair_counts = c.cbm_ht_create(c.CBM_SZ_2K); + // Parallel table mapping pair_key → max commit timestamp seen for that + // pair, so the resulting edge can carry last_co_change. The pair_counts + // table consumes its key on insert; pair_timestamps gets its own copy. + const pair_timestamps = c.cbm_ht_create(c.CBM_SZ_2K); + + var ci: usize = 0; + while (ci < commit_count) : (ci += 1) { + const commit = &commits[ci]; + if (commit.count > GH_MAX_FILES) continue; + const n = ix(commit.count); + + for (0..n) |i| { + if (intAt(file_counts, commit.files[i])) |val| { + val.* += 1; + } else { + _ = c.cbm_ht_set(file_counts, c.strdup(commit.files[i]), newInt(1)); + } + } + + for (0..n) |i| { + for (i + 1..n) |j| { + var a: CStr = commit.files[i]; + var b: CStr = commit.files[j]; + if (c.strcmp(a, b) > 0) { + const t = a; + a = b; + b = t; + } + const la = c.strlen(a); + const lb = c.strlen(b); + const pk_len = la + 1 + lb + 1; + const pk: MutStr = @ptrCast(c.malloc(pk_len)); + @memcpy(pk[0..la], a[0..la]); + pk[la] = 0x01; + @memcpy(pk[la + 1 ..][0 .. lb + 1], b[0 .. lb + 1]); + + if (intAt(pair_counts, pk)) |val| { + val.* += 1; + if (timestampAt(pair_timestamps, pk)) |ts| { + if (commit.timestamp > ts.*) ts.* = commit.timestamp; + } + c.free(pk); + } else { + // pair_counts takes ownership of pk; pair_timestamps + // needs its own copy. + const pk2: MutStr = @ptrCast(c.malloc(pk_len)); + @memcpy(pk2[0..pk_len], pk[0..pk_len]); + _ = c.cbm_ht_set(pair_counts, pk, newInt(1)); + const nts: *c_longlong = interop.mallocN(c_longlong, 1); + nts.* = commit.timestamp; + _ = c.cbm_ht_set(pair_timestamps, pk2, nts); + } + } + } + } + + var cctx: CollectCouplingCtx = .{ + .file_counts = file_counts, + .pair_timestamps = pair_timestamps, + .out = out, + .out_count = 0, + .max_out = max_out, + }; + c.cbm_ht_foreach(pair_counts, collectCouplingCb, &cctx); + + c.cbm_ht_foreach(pair_counts, freeCounter, null); + c.cbm_ht_free(pair_counts); + c.cbm_ht_foreach(pair_timestamps, freeCounter, null); + c.cbm_ht_free(pair_timestamps); + c.cbm_ht_foreach(file_counts, freeCounter, null); + c.cbm_ht_free(file_counts); + + return cctx.out_count; +} + +// ── Split pass: compute (I/O-bound) + apply (gbuf writes) ─────────── + +/// Pre-computed coupling result buffer for fused post-pass parallelism. +const MAX_COUPLINGS = 8192; +const MAX_FILE_TEMPORAL = 16384; + +/// Compute change couplings without touching the graph buffer. +/// Can run on a separate thread while other passes use the gbuf. +pub export fn cbm_pipeline_githistory_compute(repo_path: CStr, result: *c.cbm_githistory_result_t) c_int { + result.couplings = null; + result.count = 0; + result.commit_count = 0; + result.file_temporal = null; + result.file_temporal_count = 0; + + var commits: [*c]Commit = null; + var commit_count: c_int = 0; + const rc = parseGitLog(repo_path, &commits, &commit_count); + if (rc != 0 or commit_count == 0) { + c.free(commits); + return 0; + } + const n_commits = ix(commit_count); + + result.commit_count = commit_count; + + // Convert to testable format + const cf = interop.callocN(c.cbm_commit_files_t, n_commits); + if (cf == null) { + for (0..n_commits) |i| commitFree(util.at(commits, i)); + c.free(commits); + return 0; + } + for (0..n_commits) |i| { + cf[i].files = commits[i].files; + cf[i].count = commits[i].count; + cf[i].timestamp = commits[i].timestamp; + } + + const couplings = interop.mallocN(Coupling, MAX_COUPLINGS); + const coupling_count = cbm_compute_change_coupling(cf, commit_count, couplings, MAX_COUPLINGS); + + // Per-file temporal aggregation: change_count + last_modified. + // Single hash-table pass over the same commit set used for coupling so + // we don't re-scan history. NULL on OOM is fine — the caller still + // gets the couplings. + const ft_arr = interop.mallocN(FileTemporal, MAX_FILE_TEMPORAL); + if (ft_arr != null) { + var ft_count: c_int = 0; + const file_idx = c.cbm_ht_create(c.CBM_SZ_1K); + for (0..n_commits) |i| { + if (cf[i].count > GH_MAX_FILES) continue; + for (0..ix(cf[i].count)) |f| { + const fp: CStr = cf[i].files[f]; + if (intAt(file_idx, fp)) |idx| { + const entry = util.at(ft_arr, ix(idx.*)); + entry.change_count += 1; + if (cf[i].timestamp > entry.last_modified) { + entry.last_modified = cf[i].timestamp; + } + } else if (ft_count < MAX_FILE_TEMPORAL) { + const new_idx = ft_count; + ft_count += 1; + const entry = util.at(ft_arr, ix(new_idx)); + _ = c.snprintf(&entry.file_path, entry.file_path.len, "%s", fp); + entry.change_count = 1; + entry.last_modified = cf[i].timestamp; + _ = c.cbm_ht_set(file_idx, c.strdup(fp), newInt(new_idx)); + } + } + } + c.cbm_ht_foreach(file_idx, freeCounter, null); + c.cbm_ht_free(file_idx); + result.file_temporal = ft_arr; + result.file_temporal_count = ft_count; + } + + c.free(cf); + for (0..n_commits) |i| commitFree(util.at(commits, i)); + c.free(commits); + + result.couplings = couplings; + result.count = coupling_count; + return 0; +} + +/// Apply pre-computed couplings to the graph buffer (must be on main thread). +pub export fn cbm_pipeline_githistory_apply(ctx: *c.cbm_pipeline_ctx_t, result: *const c.cbm_githistory_result_t) c_int { + @setEvalBranchQuota(10000); + var edge_count: c_int = 0; + + for (0..ix(result.count)) |i| { + const cc = util.at(result.couplings, i); + + const qn_a = c.cbm_pipeline_fqn_compute(ctx.project_name, &cc.file_a, "__file__"); + const qn_b = c.cbm_pipeline_fqn_compute(ctx.project_name, &cc.file_b, "__file__"); + + const node_a: ?*const Node = c.cbm_gbuf_find_by_qn(ctx.gbuf, qn_a); + const node_b: ?*const Node = c.cbm_gbuf_find_by_qn(ctx.gbuf, qn_b); + + c.free(qn_a); + c.free(qn_b); + + const a = node_a orelse continue; + const b = node_b orelse continue; + if (a.id == b.id) continue; + + var props: [c.CBM_SZ_128]u8 = undefined; + _ = c.snprintf( + &props, + props.len, + "{\"co_changes\":%d,\"coupling_score\":%.2f,\"last_co_change\":%lld}", + cc.co_change_count, + cc.coupling_score, + cc.last_co_change, + ); + + _ = c.cbm_gbuf_insert_edge(ctx.gbuf, a.id, b.id, "FILE_CHANGES_WITH", &props); + edge_count += 1; + } + + // Apply per-file temporal metadata to existing File nodes so callers + // can query change_count / last_modified for hotspot analysis. The keys + // are merged into the node's properties: the coverage stamp written + // after extraction lives there too, and replacing the blob dropped it. + for (0..ix(result.file_temporal_count)) |i| { + const ft = util.at(result.file_temporal, i); + const file_path: CStr = &ft.file_path; + const qn = c.cbm_pipeline_fqn_compute(ctx.project_name, file_path, "__file__"); + const node: *const Node = @as(?*const Node, c.cbm_gbuf_find_by_qn(ctx.gbuf, qn)) orelse { + c.free(qn); + continue; + }; + + var base: CStr = c.strrchr(file_path, '/'); + base = if (base != null) base + 1 else file_path; + const ext: CStr = c.strrchr(base, '.'); + + const old: CStr = if (node.properties_json != null) node.properties_json else "{}"; + const odoc = c.yyjson_read(old, c.strlen(old), 0); + const doc = c.yyjson_mut_doc_new(null); + const root = if (odoc != null and c.yyjson_is_obj(c.yyjson_doc_get_root(odoc))) + c.yyjson_val_mut_copy(doc, c.yyjson_doc_get_root(odoc)) + else + c.yyjson_mut_obj(doc); + c.yyjson_mut_doc_set_root(doc, root); + _ = c.yyjson_mut_obj_remove_key(root, "extension"); + _ = c.yyjson_mut_obj_remove_key(root, "last_modified"); + _ = c.yyjson_mut_obj_remove_key(root, "change_count"); + _ = c.yyjson_mut_obj_add_strcpy(doc, root, "extension", if (ext != null) ext else ""); + _ = c.yyjson_mut_obj_add_sint(doc, root, "last_modified", ft.last_modified); + _ = c.yyjson_mut_obj_add_int(doc, root, "change_count", ft.change_count); + const props = c.yyjson_mut_write(doc, 0, null); + c.yyjson_mut_doc_free(doc); + c.yyjson_doc_free(odoc); + if (props != null) { + _ = c.cbm_gbuf_set_node_props(ctx.gbuf, qn, props); + c.free(props); + } + c.free(qn); + } + + return edge_count; +} + +// ── Main pass (original serial interface) ─────────────────────────── + +pub export fn cbm_pipeline_pass_githistory(ctx: *c.cbm_pipeline_ctx_t) c_int { + interop.logInfo("pass.start", .{ @as(CStr, "pass"), @as(CStr, "githistory") }); + + var result = std.mem.zeroes(c.cbm_githistory_result_t); + _ = cbm_pipeline_githistory_compute(ctx.repo_path, &result); + + var edge_count: c_int = 0; + if (result.count > 0 or result.file_temporal_count > 0) { + edge_count = cbm_pipeline_githistory_apply(ctx, &result); + } + + c.free(result.couplings); + c.free(result.file_temporal); + + interop.logInfo("pass.done", .{ + @as(CStr, "pass"), @as(CStr, "githistory"), + @as(CStr, "commits"), @as(CStr, c.cbm_log_itoa(result.commit_count)), + @as(CStr, "edges"), @as(CStr, c.cbm_log_itoa(edge_count)), + }); + return 0; +} diff --git a/src/pipeline/pass_importance.c b/src/pipeline/pass_importance.c deleted file mode 100644 index 031924a5..00000000 --- a/src/pipeline/pass_importance.c +++ /dev/null @@ -1,438 +0,0 @@ -/* - * pass_importance.c — Index-time per-symbol importance score (weighted degree). - * - * Predump pass. Computes an importance score for every Function/Method/Class - * node and stores it as a numeric "importance" key inside the node's EXISTING - * properties_json blob — no new column, no schema change, and therefore no - * CBM_INDEX_FORMAT_VERSION bump. Indexes written by older builds simply lack - * the key; every consumer must tolerate its absence. - * - * Model (weighted degree, Aider-repomap shaped): - * - * importance = sqrt(num_refs) * priv * generic * distinct * test_penalty - * - * num_refs incoming CALLS + USAGE edges. 0 -> sqrt(0) = 0. - * priv 0.1 name is private (leading underscore) - * generic 0.1 the name is defined in >= 5 DISTINCT files - * distinct 10 name is snake_case or camelCase AND len >= 8 - * test_penalty - * 0.1 the symbol is test scaffolding — either the target of an - * incoming TESTS edge (a production helper exercised by a - * test), or it lives in a test file per the graph's own - * canonical cbm_is_test_path() classifier (the same one - * pass_tests uses). - * - * Transitive importance (PageRank) is deliberately NOT built here: weighted - * degree is what this pass claims, and a transitive refinement has to earn - * itself against a fixed judgment set before it ships. - * - * ── Cost ───────────────────────────────────────────────────────────── - * The generic-name multiplier needs |{ file a name is defined in }|. The - * obvious shape — for each node, walk its whole same-name group and - * pairwise-compare file paths — is O(k^2) per node and therefore O(k^3) per - * same-name group of size k. That is not a theoretical concern: on a large - * Java corpus a single name group (`toString`, k≈4.6k) costs ~2 minutes, and - * a handful of such names alone multiplies total index time several-fold. - * - * This implementation is linear in the graph instead: - * - the distinct-file count is computed ONCE PER DISTINCT NAME and memoized - * (g_imp_name_counts), so a group of size k is scanned once, not k times; - * - within a group, file paths are deduplicated through a hash set rather - * than pairwise strcmp. - * Total distinct-file work is therefore Σ over distinct names of k = O(N). - * Both gbuf lookups it rests on (find_by_name, find_edges_by_target_type) are - * hash-indexed, so the per-node loop is O(1) amortised per node. - * - * g_importance_name_visits counts same-name-group member visits and is GATED - * in the importance suite: it must stay linear (ratio ~2) under a corpus - * doubling. A regression to the per-node scan lands at ~4 and fails the gate. - * - * ── Ordering (load-bearing) ────────────────────────────────────────── - * Registered LAST among the predump passes, so every edge type the score - * reads (CALLS, USAGE from extraction; TESTS from pass_tests) already exists. - * A pass ordered earlier would silently score everything at num_refs = 0 with - * no test penalty. In the incremental post-pass sequence it runs right after - * complexity and before the derived-state digest, which hashes - * properties_json: scoring after the digest would make every incremental run - * look changed and defeat the similarity/semantic reuse gates. - * - * Ported from upstream e5f3aab0. Upstream's closure-delta store rescore - * (396ac348) does not apply here: the fork's incremental route always scores - * a whole-project buffer exported from the committed Zova generation, with - * inbound edges relinked before the post-passes. - */ -#include "foundation/constants.h" -#include "pipeline/pipeline.h" -#include "pipeline/pipeline_internal.h" -#include "graph_buffer/graph_buffer.h" -#include "foundation/hash_table.h" -#include "foundation/log.h" -#include "foundation/compat.h" -#include "cbm.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -/* Work counters — deltas are read by the importance suite's linearity gate. - * Never reset here: callers snapshot before/after so nested runs compose. */ -_Atomic uint64_t g_importance_nodes = 0; -_Atomic uint64_t g_importance_name_visits = 0; - -enum { - CBM_IMPORTANCE_GENERIC_MIN_FILES = 5, /* name defined in >= N files -> generic */ - CBM_IMPORTANCE_DISTINCT_MIN_LEN = 8, /* distinctive-identifier length floor */ -}; -static const double CBM_IMPORTANCE_PRIV_MUL = 0.1; -static const double CBM_IMPORTANCE_GENERIC_MUL = 0.1; -static const double CBM_IMPORTANCE_DISTINCT_MUL = 10.0; -static const double CBM_IMPORTANCE_TEST_MUL = 0.1; - -/* The node labels that carry an importance score. File/Module and other - * non-symbol nodes are deliberately excluded. */ -static const char *const CBM_IMPORTANCE_LABELS[] = {"Function", "Method", "Class"}; -enum { CBM_IMPORTANCE_LABEL_COUNT = 3 }; - -/* Int → string for structured logging (thread-safe ring buffer). */ -static const char *itoa_imp(uint64_t val) { - enum { RING = 2, MASK = 1 }; - static CBM_TLS char bufs[RING][CBM_SZ_32]; - static CBM_TLS int idx = 0; - int i = idx; - idx = (idx + 1) & MASK; - snprintf(bufs[i], sizeof(bufs[i]), "%llu", (unsigned long long)val); - return bufs[i]; -} - -static bool name_is_private(const char *name) { - return name != NULL && name[0] == '_'; -} - -/* Distinctive = (snake_case OR camelCase) AND length >= 8. snake_case is an - * embedded '_'; camelCase is a lower->upper "hump". A plain len>=8 lowercase - * word (no '_', no hump) is NOT distinctive. */ -static bool name_is_distinctive(const char *name) { - if (!name) { - return false; - } - size_t len = strlen(name); - if (len < (size_t)CBM_IMPORTANCE_DISTINCT_MIN_LEN) { - return false; - } - if (strchr(name, '_') != NULL) { - return true; /* snake_case */ - } - for (size_t i = 1; i < len; i++) { - if (islower((unsigned char)name[i - 1]) && isupper((unsigned char)name[i])) { - return true; /* camelCase hump */ - } - } - return false; -} - -/* ── JSON property write-back ───────────────────────────────────────── - * - * properties_json is a flat object here (same assumption the sibling passes' - * json_get_int/json_get_bool helpers make), but the value scanner below is - * written to survive nested values so a future richer blob cannot be - * truncated by an in-place overwrite. */ - -/* Return a pointer just past the JSON value beginning at p (first non-space - * char of the value). NULL if the value is unterminated/malformed. */ -static const char *imp_value_end(const char *p) { - if (*p == '"') { - p++; - while (*p) { - if (*p == '\\' && p[1]) { - p += 2; - continue; - } - if (*p == '"') { - return p + 1; - } - p++; - } - return NULL; - } - if (*p == '{' || *p == '[') { - int depth = 0; - bool in_str = false; - while (*p) { - if (in_str) { - if (*p == '\\' && p[1]) { - p += 2; - continue; - } - if (*p == '"') { - in_str = false; - } - } else if (*p == '"') { - in_str = true; - } else if (*p == '{' || *p == '[') { - depth++; - } else if (*p == '}' || *p == ']') { - depth--; - if (depth == 0) { - return p + 1; - } - } - p++; - } - return NULL; - } - while (*p && *p != ',' && *p != '}' && *p != ']') { - p++; - } - return p; -} - -/* Find `"key"` where it is genuinely in KEY position: preceded (modulo - * whitespace) by '{' or ',' and followed (modulo whitespace) by ':'. A bare - * strstr would also match the text inside a string VALUE. Returns a pointer - * to the opening quote, or NULL. */ -static const char *imp_find_key(const char *json, const char *key) { - char pat[CBM_SZ_64]; - int n = snprintf(pat, sizeof(pat), "\"%s\"", key); - if (n < 0 || (size_t)n >= sizeof(pat)) { - return NULL; - } - const char *p = json; - while ((p = strstr(p, pat)) != NULL) { - const char *before = p; - while (before > json && isspace((unsigned char)before[-1])) { - before--; - } - char prev = (before > json) ? before[-1] : '\0'; - const char *after = p + n; - while (isspace((unsigned char)*after)) { - after++; - } - if ((prev == '{' || prev == ',') && *after == ':') { - return p; - } - p += n; - } - return NULL; -} - -/* Write the numeric "importance" key into a node's properties JSON object. - * - * IDEMPOTENT BY CONTRACT. The incremental path exports nodes from the - * committed Zova generation, and those nodes ALREADY carry an - * "importance" key from the previous run. A pure append would emit - * {"importance":1.0,...,"importance":2.0} — silent property corruption that - * no build or schema check would catch. So an existing key is removed and - * the new one appended, which also keeps the key last: the incremental - * complexity pass re-appends its transitive keys after the stored - * importance, and an in-place overwrite would leave importance mid-object - * and make the stored properties differ from a fresh full index (the same - * remove-then-add that pass_complexity uses). - * - * A properties_json that is not a JSON object is left untouched. */ -void cbm_pipeline_importance_append_prop(cbm_gbuf_node_t *node, double score) { - if (!node) { - return; - } - const char *old = node->properties_json ? node->properties_json : "{}"; - size_t olen = strlen(old); - if (olen < 2 || old[olen - 1] != '}') { - return; /* not a JSON object — leave untouched */ - } - - char val[CBM_SZ_32]; - int vn = snprintf(val, sizeof(val), "%.6f", score); - if (vn < 0 || (size_t)vn >= sizeof(val)) { - return; - } - - char *stripped = NULL; - const char *k = imp_find_key(old, "importance"); - if (k) { - const char *colon = strchr(k, ':'); - if (!colon) { - return; /* imp_find_key guarantees one, but never trust a raw scan */ - } - const char *v = colon + 1; - while (isspace((unsigned char)*v)) { - v++; - } - const char *vend = imp_value_end(v); - if (!vend) { - return; - } - /* Drop the member with one adjacent comma: the preceding one, or the - * following one when the member is first. */ - const char *start = k; - while (start > old && isspace((unsigned char)start[-1])) { - start--; - } - const char *end = vend; - if (start > old && start[-1] == ',') { - start--; - } else { - start = k; - while (isspace((unsigned char)*end)) { - end++; - } - if (*end == ',') { - end++; - } - } - size_t head = (size_t)(start - old); - size_t tail = strlen(end); - stripped = malloc(head + tail + 1); - if (!stripped) { - return; - } - memcpy(stripped, old, head); - memcpy(stripped + head, end, tail + 1); - old = stripped; - olen = strlen(old); - if (olen < 2 || old[olen - 1] != '}') { - free(stripped); - return; - } - } - - /* An empty object, possibly with whitespace, takes no leading comma. */ - const char *last = old + olen - 1; - while (last > old + 1 && isspace((unsigned char)last[-1])) { - last--; - } - bool empty = last[-1] == '{'; - char frag[CBM_SZ_64]; - int fn = snprintf(frag, sizeof(frag), "%s\"importance\":%s}", empty ? "" : ",", val); - if (fn < 0 || (size_t)fn >= sizeof(frag)) { - free(stripped); - return; - } - char *neu = malloc(olen - 1 + (size_t)fn + 1); - if (!neu) { - free(stripped); - return; - } - memcpy(neu, old, olen - 1); /* copy without the trailing '}' */ - memcpy(neu + olen - 1, frag, (size_t)fn + 1); - free(stripped); - free(node->properties_json); - node->properties_json = neu; -} - -/* ── Distinct-file counting (the linear core) ─────────────────────────── */ - -typedef struct { - CBMHashTable *counts; /* name -> (void *)(intptr_t)(distinct_files + 1) */ - CBMHashTable *paths; /* scratch file-path set, cleared between names */ -} imp_name_index_t; - -/* Distinct files a name is DEFINED in. Counts distinct file paths, not - * distinct nodes — two defs of one name in the same file count once. - * Computed once per distinct name; every later node in the group is a hash - * hit. Name keys are borrowed from the gbuf nodes, which outlive the pass. */ -static int imp_distinct_file_count(const cbm_gbuf_t *gb, const char *name, imp_name_index_t *ix) { - if (!name || !*name || !ix || !ix->counts) { - return 0; - } - void *cached = cbm_ht_get(ix->counts, name); - if (cached) { - return (int)((intptr_t)cached - 1); - } - - const cbm_gbuf_node_t **nodes = NULL; - int count = 0; - int distinct = 0; - if (cbm_gbuf_find_by_name(gb, name, &nodes, &count) == 0 && count > 0) { - atomic_fetch_add_explicit(&g_importance_name_visits, (uint64_t)count, memory_order_relaxed); - if (ix->paths) { - cbm_ht_clear(ix->paths); - for (int i = 0; i < count; i++) { - const char *fp = nodes[i]->file_path; - if (!fp || cbm_ht_has(ix->paths, fp)) { - continue; - } - cbm_ht_set(ix->paths, fp, (void *)(intptr_t)1); - distinct++; - } - } else { - /* Scratch set unavailable (allocation failure): the generic-name - * multiplier degrades to "never generic" rather than falling back - * to a pairwise scan — a silent quadratic is worse than a missing - * 0.1 multiplier. */ - distinct = 0; - } - } - cbm_ht_set(ix->counts, name, (void *)(intptr_t)(distinct + 1)); - return distinct; -} - -static int incoming_edge_count(const cbm_gbuf_t *gb, int64_t id, const char *type) { - const cbm_gbuf_edge_t **edges = NULL; - int ne = 0; - if (cbm_gbuf_find_edges_by_target_type(gb, id, type, &edges, &ne) != 0) { - return 0; - } - return ne; -} - -void cbm_pipeline_pass_importance(cbm_pipeline_ctx_t *ctx) { - if (!ctx || !ctx->gbuf) { - return; - } - cbm_gbuf_t *gb = ctx->gbuf; - - /* Modest reservations, deliberately not sized from node_count: the memo - * holds one entry per DISTINCT name, far fewer than nodes, and reserving - * buckets for every node would cost hundreds of MB of transient peak on a - * multi-million-node graph to save a handful of amortised rehashes. */ - imp_name_index_t ix = { - .counts = cbm_ht_create(CBM_SZ_4K), - .paths = cbm_ht_create(CBM_SZ_256), - }; - if (!ix.counts) { - cbm_ht_free(ix.paths); - cbm_log_error("pass.importance", "msg", "name_index_alloc_failed"); - return; - } - - uint64_t updated = 0; - for (int li = 0; li < CBM_IMPORTANCE_LABEL_COUNT; li++) { - const cbm_gbuf_node_t **nodes = NULL; - int count = 0; - if (cbm_gbuf_find_by_label(gb, CBM_IMPORTANCE_LABELS[li], &nodes, &count) != 0) { - continue; - } - for (int i = 0; i < count; i++) { - cbm_gbuf_node_t *n = (cbm_gbuf_node_t *)nodes[i]; - - int num_refs = - incoming_edge_count(gb, n->id, "CALLS") + incoming_edge_count(gb, n->id, "USAGE"); - double score = sqrt((double)num_refs); - - if (name_is_private(n->name)) { - score *= CBM_IMPORTANCE_PRIV_MUL; - } - if (imp_distinct_file_count(gb, n->name, &ix) >= CBM_IMPORTANCE_GENERIC_MIN_FILES) { - score *= CBM_IMPORTANCE_GENERIC_MUL; - } - if (name_is_distinctive(n->name)) { - score *= CBM_IMPORTANCE_DISTINCT_MUL; - } - if (cbm_is_test_path(n->file_path) || incoming_edge_count(gb, n->id, "TESTS") > 0) { - score *= CBM_IMPORTANCE_TEST_MUL; - } - - cbm_pipeline_importance_append_prop(n, score); - updated++; - } - } - - cbm_ht_free(ix.paths); - cbm_ht_free(ix.counts); - atomic_fetch_add_explicit(&g_importance_nodes, updated, memory_order_relaxed); - cbm_log_info("pass.importance", "symbols", itoa_imp(updated)); -} diff --git a/src/pipeline/pass_importance.zig b/src/pipeline/pass_importance.zig new file mode 100644 index 00000000..295ebc16 --- /dev/null +++ b/src/pipeline/pass_importance.zig @@ -0,0 +1,376 @@ +//! pass_importance.zig — Index-time per-symbol importance score (weighted degree). +//! +//! Predump pass. Computes an importance score for every Function/Method/Class +//! node and stores it as a numeric "importance" key inside the node's EXISTING +//! properties_json blob — no new column, no schema change, and therefore no +//! CBM_INDEX_FORMAT_VERSION bump. Indexes written by older builds simply lack +//! the key; every consumer must tolerate its absence. +//! +//! Model (weighted degree, Aider-repomap shaped): +//! +//! importance = sqrt(num_refs) * priv * generic * distinct * test_penalty +//! +//! num_refs incoming CALLS + USAGE edges. 0 -> sqrt(0) = 0. +//! priv 0.1 name is private (leading underscore) +//! generic 0.1 the name is defined in >= 5 DISTINCT files +//! distinct 10 name is snake_case or camelCase AND len >= 8 +//! test_penalty +//! 0.1 the symbol is test scaffolding — either the target of an +//! incoming TESTS edge (a production helper exercised by a +//! test), or it lives in a test file per the graph's own +//! canonical cbm_is_test_path() classifier (the same one +//! pass_tests uses). +//! +//! Transitive importance (PageRank) is deliberately NOT built here: weighted +//! degree is what this pass claims, and a transitive refinement has to earn +//! itself against a fixed judgment set before it ships. +//! +//! ── Cost ───────────────────────────────────────────────────────────── +//! The generic-name multiplier needs |{ file a name is defined in }|. The +//! obvious shape — for each node, walk its whole same-name group and +//! pairwise-compare file paths — is O(k^2) per node and therefore O(k^3) per +//! same-name group of size k. That is not a theoretical concern: on a large +//! Java corpus a single name group (`toString`, k≈4.6k) costs ~2 minutes, and +//! a handful of such names alone multiplies total index time several-fold. +//! +//! This implementation is linear in the graph instead: +//! - the distinct-file count is computed ONCE PER DISTINCT NAME and memoized, +//! so a group of size k is scanned once, not k times; +//! - within a group, file paths are deduplicated through a hash set rather +//! than pairwise strcmp. +//! Total distinct-file work is therefore Σ over distinct names of k = O(N). +//! Both gbuf lookups it rests on (find_by_name, find_edges_by_target_type) are +//! hash-indexed, so the per-node loop is O(1) amortised per node. +//! +//! g_importance_name_visits counts same-name-group member visits and is GATED +//! in the importance suite: it must stay linear (ratio ~2) under a corpus +//! doubling. A regression to the per-node scan lands at ~4 and fails the gate. +//! +//! ── Ordering (load-bearing) ────────────────────────────────────────── +//! Registered LAST among the predump passes, so every edge type the score +//! reads (CALLS, USAGE from extraction; TESTS from pass_tests) already exists. +//! A pass ordered earlier would silently score everything at num_refs = 0 with +//! no test penalty. In the incremental post-pass sequence it runs right after +//! complexity and before the derived-state digest, which hashes +//! properties_json: scoring after the digest would make every incremental run +//! look changed and defeat the similarity/semantic reuse gates. +//! +//! Ported from upstream e5f3aab0. Upstream's closure-delta store rescore +//! (396ac348) does not apply here: the fork's incremental route always scores +//! a whole-project buffer exported from the committed Zova generation, with +//! inbound edges relinked before the post-passes. + +const std = @import("std"); +const c = @import("c"); +const interop = @import("../foundation/interop.zig"); +const util = @import("util.zig"); + +const CStr = util.CStr; +const MutStr = util.MutStr; +const Node = c.cbm_gbuf_node_t; +const Edge = c.cbm_gbuf_edge_t; +const HashTable = c.CBMHashTable; +const isSpace = std.ascii.isWhitespace; + +/// Work counters — deltas are read by the importance suite's linearity gate. +/// Never reset here: callers snapshot before/after so nested runs compose. +/// Declared `extern _Atomic uint64_t` in pipeline_internal.h. +export var g_importance_nodes: u64 = 0; +export var g_importance_name_visits: u64 = 0; + +const GENERIC_MIN_FILES = 5; // name defined in >= N files -> generic +const DISTINCT_MIN_LEN = 8; // distinctive-identifier length floor +const PRIV_MUL: f64 = 0.1; +const GENERIC_MUL: f64 = 0.1; +const DISTINCT_MUL: f64 = 10.0; +const TEST_MUL: f64 = 0.1; + +/// The node labels that carry an importance score. File/Module and other +/// non-symbol nodes are deliberately excluded. +const IMPORTANCE_LABELS = [_][*:0]const u8{ "Function", "Method", "Class" }; + +/// Int → string for structured logging (thread-local ring buffer). +threadlocal var itoa_bufs: [2][c.CBM_SZ_32]u8 = undefined; +threadlocal var itoa_idx: usize = 0; + +fn itoaImp(val: u64) CStr { + const i = itoa_idx; + itoa_idx = (itoa_idx + 1) & 1; + _ = c.snprintf(&itoa_bufs[i], itoa_bufs[i].len, "%llu", @as(c_ulonglong, val)); + return &itoa_bufs[i]; +} + +fn nameIsPrivate(name: CStr) bool { + return name != null and name[0] == '_'; +} + +/// Distinctive = (snake_case OR camelCase) AND length >= 8. snake_case is an +/// embedded '_'; camelCase is a lower->upper "hump". A plain len>=8 lowercase +/// word (no '_', no hump) is NOT distinctive. +fn nameIsDistinctive(name: CStr) bool { + if (name == null) return false; + const len = c.strlen(name); + if (len < DISTINCT_MIN_LEN) return false; + if (c.strchr(name, '_') != null) return true; // snake_case + for (1..len) |i| { + if (std.ascii.isLower(name[i - 1]) and std.ascii.isUpper(name[i])) { + return true; // camelCase hump + } + } + return false; +} + +// ── JSON property write-back ──────────────────────────────────────── +// +// properties_json is a flat object here (same assumption the sibling passes' +// jsonGetInt/jsonGetBool helpers make), but the value scanner below is +// written to survive nested values so a future richer blob cannot be +// truncated by an in-place overwrite. + +/// Return a pointer just past the JSON value beginning at p (first non-space +/// char of the value). NULL if the value is unterminated/malformed. +fn impValueEnd(p_in: [*]const u8) ?[*]const u8 { + var p = p_in; + if (p[0] == '"') { + p += 1; + while (p[0] != 0) { + if (p[0] == '\\' and p[1] != 0) { + p += 2; + continue; + } + if (p[0] == '"') return p + 1; + p += 1; + } + return null; + } + if (p[0] == '{' or p[0] == '[') { + var depth: c_int = 0; + var in_str = false; + while (p[0] != 0) { + if (in_str) { + if (p[0] == '\\' and p[1] != 0) { + p += 2; + continue; + } + if (p[0] == '"') in_str = false; + } else if (p[0] == '"') { + in_str = true; + } else if (p[0] == '{' or p[0] == '[') { + depth += 1; + } else if (p[0] == '}' or p[0] == ']') { + depth -= 1; + if (depth == 0) return p + 1; + } + p += 1; + } + return null; + } + while (p[0] != 0 and p[0] != ',' and p[0] != '}' and p[0] != ']') p += 1; + return p; +} + +/// Find `"key"` where it is genuinely in KEY position: preceded (modulo +/// whitespace) by '{' or ',' and followed (modulo whitespace) by ':'. A bare +/// strstr would also match the text inside a string VALUE. Returns a pointer +/// to the opening quote, or NULL. +fn impFindKey(json: [*:0]const u8, key: CStr) ?[*]const u8 { + var pat: [c.CBM_SZ_64]u8 = undefined; + const n_c = c.snprintf(&pat, pat.len, "\"%s\"", key); + if (n_c < 0 or @as(usize, @intCast(n_c)) >= pat.len) return null; + const n: usize = @intCast(n_c); + var p: CStr = c.strstr(json, &pat); + while (p != null) : (p = c.strstr(p + n, &pat)) { + var before: [*]const u8 = p; + while (@intFromPtr(before) > @intFromPtr(json) and isSpace((before - 1)[0])) before -= 1; + const prev: u8 = if (@intFromPtr(before) > @intFromPtr(json)) (before - 1)[0] else 0; + var after: [*]const u8 = p + n; + while (isSpace(after[0])) after += 1; + if ((prev == '{' or prev == ',') and after[0] == ':') return p; + } + return null; +} + +/// Write the numeric "importance" key into a node's properties JSON object. +/// +/// IDEMPOTENT BY CONTRACT. The incremental path exports nodes from the +/// committed Zova generation, and those nodes ALREADY carry an +/// "importance" key from the previous run. A pure append would emit +/// {"importance":1.0,...,"importance":2.0} — silent property corruption that +/// no build or schema check would catch. So an existing key is removed and +/// the new one appended, which also keeps the key last: the incremental +/// complexity pass re-appends its transitive keys after the stored +/// importance, and an in-place overwrite would leave importance mid-object +/// and make the stored properties differ from a fresh full index (the same +/// remove-then-add that pass_complexity uses). +/// +/// A properties_json that is not a JSON object is left untouched. +pub export fn cbm_pipeline_importance_append_prop(node_c: ?*Node, score: f64) void { + const node = node_c orelse return; + var old: [*:0]const u8 = if (node.properties_json != null) node.properties_json else "{}"; + var olen = c.strlen(old); + if (olen < 2 or old[olen - 1] != '}') return; // not a JSON object — leave untouched + + var val: [c.CBM_SZ_32]u8 = undefined; + const vn = c.snprintf(&val, val.len, "%.6f", score); + if (vn < 0 or @as(usize, @intCast(vn)) >= val.len) return; + + var stripped: MutStr = null; + if (impFindKey(old, "importance")) |k| { + const colon: CStr = c.strchr(k, ':'); + if (colon == null) return; // impFindKey guarantees one, but never trust a raw scan + var v: [*]const u8 = colon + 1; + while (isSpace(v[0])) v += 1; + const vend = impValueEnd(v) orelse return; + // Drop the member with one adjacent comma: the preceding one, or the + // following one when the member is first. + var start = k; + while (@intFromPtr(start) > @intFromPtr(old) and isSpace((start - 1)[0])) start -= 1; + var end = vend; + if (@intFromPtr(start) > @intFromPtr(old) and (start - 1)[0] == ',') { + start -= 1; + } else { + start = k; + while (isSpace(end[0])) end += 1; + if (end[0] == ',') end += 1; + } + const head = util.ptrDiff(start, old); + const tail = c.strlen(end); + stripped = @ptrCast(c.malloc(head + tail + 1)); + if (stripped == null) return; + @memcpy(stripped[0..head], old[0..head]); + @memcpy(stripped[head..][0 .. tail + 1], end[0 .. tail + 1]); + old = stripped; + olen = c.strlen(old); + if (olen < 2 or old[olen - 1] != '}') { + c.free(stripped); + return; + } + } + + // An empty object, possibly with whitespace, takes no leading comma. + var last: [*]const u8 = old + olen - 1; + while (@intFromPtr(last) > @intFromPtr(old + 1) and isSpace((last - 1)[0])) last -= 1; + const empty = (last - 1)[0] == '{'; + var frag: [c.CBM_SZ_64]u8 = undefined; + const fn_c = c.snprintf( + &frag, + frag.len, + "%s\"importance\":%s}", + @as(CStr, if (empty) "" else ","), + @as(CStr, &val), + ); + if (fn_c < 0 or @as(usize, @intCast(fn_c)) >= frag.len) { + c.free(stripped); + return; + } + const frag_len: usize = @intCast(fn_c); + const neu: MutStr = @ptrCast(c.malloc(olen - 1 + frag_len + 1)); + if (neu == null) { + c.free(stripped); + return; + } + @memcpy(neu[0 .. olen - 1], old[0 .. olen - 1]); // copy without the trailing '}' + @memcpy(neu[olen - 1 ..][0 .. frag_len + 1], frag[0 .. frag_len + 1]); + c.free(stripped); + c.free(node.properties_json); + node.properties_json = neu; +} + +// ── Distinct-file counting (the linear core) ──────────────────────── + +const ImpNameIndex = struct { + counts: ?*HashTable, // name -> (void *)(intptr_t)(distinct_files + 1) + paths: ?*HashTable, // scratch file-path set, cleared between names +}; + +/// Distinct files a name is DEFINED in. Counts distinct file paths, not +/// distinct nodes — two defs of one name in the same file count once. +/// Computed once per distinct name; every later node in the group is a hash +/// hit. Name keys are borrowed from the gbuf nodes, which outlive the pass. +fn impDistinctFileCount(gb: ?*const c.cbm_gbuf_t, name: CStr, index: *ImpNameIndex) c_int { + if (name == null or name[0] == 0 or index.counts == null) return 0; + const cached = c.cbm_ht_get(index.counts, name); + if (cached != null) { + return @intCast(@as(isize, @bitCast(@intFromPtr(cached))) - 1); + } + + var nodes: [*c][*c]const Node = null; + var count: c_int = 0; + var distinct: c_int = 0; + if (c.cbm_gbuf_find_by_name(gb, name, &nodes, &count) == 0 and count > 0) { + _ = @atomicRmw(u64, &g_importance_name_visits, .Add, @intCast(count), .monotonic); + if (index.paths) |paths| { + c.cbm_ht_clear(paths); + for (0..util.ix(count)) |i| { + const file_path: CStr = nodes[i].*.file_path; + if (file_path == null or c.cbm_ht_has(paths, file_path)) continue; + _ = c.cbm_ht_set(paths, file_path, @ptrFromInt(1)); + distinct += 1; + } + } else { + // Scratch set unavailable (allocation failure): the generic-name + // multiplier degrades to "never generic" rather than falling back + // to a pairwise scan — a silent quadratic is worse than a missing + // 0.1 multiplier. + distinct = 0; + } + } + _ = c.cbm_ht_set(index.counts, name, @ptrFromInt(@as(usize, @intCast(distinct + 1)))); + return distinct; +} + +fn incomingEdgeCount(gb: ?*const c.cbm_gbuf_t, id: i64, edge_type: CStr) c_int { + var edges: [*c][*c]const Edge = null; + var ne: c_int = 0; + if (c.cbm_gbuf_find_edges_by_target_type(gb, id, edge_type, &edges, &ne) != 0) return 0; + return ne; +} + +pub export fn cbm_pipeline_pass_importance(ctx_c: ?*c.cbm_pipeline_ctx_t) void { + const ctx = ctx_c orelse return; + const gb = ctx.gbuf orelse return; + + // Modest reservations, deliberately not sized from node_count: the memo + // holds one entry per DISTINCT name, far fewer than nodes, and reserving + // buckets for every node would cost hundreds of MB of transient peak on a + // multi-million-node graph to save a handful of amortised rehashes. + var index: ImpNameIndex = .{ + .counts = c.cbm_ht_create(c.CBM_SZ_4K), + .paths = c.cbm_ht_create(c.CBM_SZ_256), + }; + if (index.counts == null) { + c.cbm_ht_free(index.paths); + interop.log(c.CBM_LOG_ERROR, "pass.importance", .{ + @as(CStr, "msg"), @as(CStr, "name_index_alloc_failed"), + }); + return; + } + + var updated: u64 = 0; + for (IMPORTANCE_LABELS) |label| { + var nodes: [*c][*c]const Node = null; + var count: c_int = 0; + if (c.cbm_gbuf_find_by_label(gb, label, &nodes, &count) != 0) continue; + for (0..util.ix(count)) |i| { + const n: *Node = @constCast(nodes[i]); + + const num_refs = incomingEdgeCount(gb, n.id, "CALLS") + incomingEdgeCount(gb, n.id, "USAGE"); + var score: f64 = @sqrt(@as(f64, @floatFromInt(num_refs))); + + if (nameIsPrivate(n.name)) score *= PRIV_MUL; + if (impDistinctFileCount(gb, n.name, &index) >= GENERIC_MIN_FILES) score *= GENERIC_MUL; + if (nameIsDistinctive(n.name)) score *= DISTINCT_MUL; + if (c.cbm_is_test_path(n.file_path) or incomingEdgeCount(gb, n.id, "TESTS") > 0) { + score *= TEST_MUL; + } + + cbm_pipeline_importance_append_prop(n, score); + updated += 1; + } + } + + c.cbm_ht_free(index.paths); + c.cbm_ht_free(index.counts); + _ = @atomicRmw(u64, &g_importance_nodes, .Add, updated, .monotonic); + interop.logInfo("pass.importance", .{ @as(CStr, "symbols"), itoaImp(updated) }); +} diff --git a/src/pipeline/pass_infrascan.c b/src/pipeline/pass_infrascan.c deleted file mode 100644 index a9b2418f..00000000 --- a/src/pipeline/pass_infrascan.c +++ /dev/null @@ -1,1329 +0,0 @@ -/* - * pass_infrascan.c — Infrastructure file detection and parsing helpers. - * - * Pure helper functions for detecting infrastructure file types, - * secret filtering, and parsing Dockerfiles, .env files, shell scripts, - * and Terraform files. - */ -#include "foundation/constants.h" - -enum { - IS_ENV_EXT = 4, /* strlen(".env") */ - IS_CONTENT_LIMIT = 4096, - IS_MAX_STAGES = 16, - IS_MAX_PORTS = 16, - IS_MAX_SOURCES = 16, - IS_TOKEN_MIN_1 = 16, /* min alnum for AWS key */ - IS_TOKEN_MIN_2 = 20, /* min alnum for GH classic token */ - IS_FROM_SKIP = 4, /* strlen("FROM") */ - IS_EXPOSE_SKIP = 6, /* strlen("EXPOSE") */ - IS_RUN_SKIP = 3, /* strlen("RUN") / strlen("ENV") / strlen("ARG") */ - IS_CMD_SKIP = 4, /* strlen("CMD ") */ - IS_USER_SKIP = 5, /* strlen("USER ") */ - IS_EXPORT_SKIP = 7, /* strlen("export ") */ - IS_WORKDIR_SKIP = 8, /* strlen("WORKDIR ") */ - IS_ENTRYPOINT_OFF = 10, /* strlen("ENTRYPOINT") */ - IS_AS_SKIP = 3, /* strlen("AS ") */ - IS_SOURCE_DOT = 2, /* strlen(". ") */ - IS_QUOTE_TRIM = 2, /* strip surrounding quotes */ -}; -#define IS_NOT_FOUND (-1) - -#define SLEN(s) (sizeof(s) - 1) -#include "pipeline/pipeline.h" -#include "pipeline/pipeline_internal.h" - -#include -#include -#include -#include - -/* ── Constants ───────────────────────────────────────────────────── */ - -/* String length constants for keyword matching */ -#define LEN_DOCKERFILE 11 /* strlen("dockerfile.") == strlen(".dockerfile") */ -#define LEN_DOCKER_COMPOSE 14 /* strlen("docker-compose") */ -#define LEN_HEALTHCHECK 11 /* strlen("HEALTHCHECK") */ -#define LEN_DOCKER_COMPOSE_SKIP 15 /* strlen("docker-compose") + space */ - -/* Minimum alnum chars after prefix for secret detection */ -#define GITHUB_PAT_MIN_ALNUM 36 /* ghp_ + 36 alnum chars */ - -/* ── Internal helpers ────────────────────────────────────────────── */ - -static void to_lower(const char *src, char *dst, size_t dst_sz) { - size_t i; - size_t len = strlen(src); - if (len >= dst_sz) { - len = dst_sz - SKIP_ONE; - } - for (i = 0; i < len; i++) { - dst[i] = (char)tolower((unsigned char)src[i]); - } - dst[i] = '\0'; -} - -/* Case-insensitive substring search */ -static const char *ci_strstr(const char *haystack, const char *needle) { - if (!*needle) { - return haystack; - } - size_t nlen = strlen(needle); - for (; *haystack; haystack++) { - if (strncasecmp(haystack, needle, nlen) == 0) { - return haystack; - } - } - return NULL; -} - -/* Count consecutive alphanumeric characters */ -static int count_alnum(const char *s) { - int n = 0; - while (isalnum((unsigned char)s[n])) { - n++; - } - return n; -} - -/* Count consecutive alphanumeric or dash characters */ -static int count_alnum_dash(const char *s) { - int n = 0; - while (isalnum((unsigned char)s[n]) || s[n] == '-') { - n++; - } - return n; -} - -/* Skip whitespace, return pointer past it */ -static const char *skip_ws(const char *p) { - while (*p == ' ' || *p == '\t') { - p++; - } - return p; -} - -/* Extract a word ([a-zA-Z0-9_]+) into buf. Returns length. */ -static int extract_word(const char *p, char *buf, size_t buf_sz) { - int n = 0; - while ((isalnum((unsigned char)p[n]) || p[n] == '_') && (size_t)n < buf_sz - SKIP_ONE) { - buf[n] = p[n]; - n++; - } - buf[n] = '\0'; - return n; -} - -/* Extract a non-space token into buf. Returns length. */ -static int extract_token(const char *p, char *buf, size_t buf_sz) { - int n = 0; - while (p[n] && p[n] != ' ' && p[n] != '\t' && (size_t)n < buf_sz - SKIP_ONE) { - buf[n] = p[n]; - n++; - } - buf[n] = '\0'; - return n; -} - -/* Trim trailing whitespace in-place */ -static void rtrim(char *s) { - size_t len = strlen(s); - while (len > 0 && (s[len - SKIP_ONE] == ' ' || s[len - SKIP_ONE] == '\t' || - s[len - SKIP_ONE] == '\r' || s[len - SKIP_ONE] == '\n')) { - s[--len] = '\0'; - } -} - -/* ── File identification ────────────────────────────────────────── */ - -bool cbm_is_dockerfile(const char *name) { - if (!name) { - return false; - } - char lower[CBM_SZ_256]; - to_lower(name, lower, sizeof(lower)); - - if (strcmp(lower, "dockerfile") == 0) { - return true; - } - if (strncmp(lower, "dockerfile.", LEN_DOCKERFILE) == 0) { - return true; - } - size_t len = strlen(lower); - if (len > LEN_DOCKERFILE && strcmp(lower + len - LEN_DOCKERFILE, ".dockerfile") == 0) { - return true; - } - return false; -} - -bool cbm_is_compose_file(const char *name) { - if (!name) { - return false; - } - char lower[CBM_SZ_256]; - to_lower(name, lower, sizeof(lower)); - - bool prefix_match = (strncmp(lower, "docker-compose", LEN_DOCKER_COMPOSE) == 0) || - (strcmp(lower, "compose.yml") == 0) || (strcmp(lower, "compose.yaml") == 0); - if (!prefix_match) { - return false; - } - - const char *ext = strrchr(lower, '.'); - if (!ext) { - return false; - } - return (strcmp(ext, ".yml") == 0 || strcmp(ext, ".yaml") == 0); -} - -bool cbm_is_cloudbuild_file(const char *name) { - if (!name) { - return false; - } - char lower[CBM_SZ_256]; - to_lower(name, lower, sizeof(lower)); - - if (strncmp(lower, "cloudbuild", SLEN("cloudbuild")) != 0) { - return false; - } - const char *ext = strrchr(lower, '.'); - if (!ext) { - return false; - } - return (strcmp(ext, ".yml") == 0 || strcmp(ext, ".yaml") == 0); -} - -bool cbm_is_env_file(const char *name) { - if (!name) { - return false; - } - char lower[CBM_SZ_256]; - to_lower(name, lower, sizeof(lower)); - - if (strcmp(lower, ".env") == 0) { - return true; - } - if (strncmp(lower, ".env.", SLEN(".env.")) == 0) { - return true; - } - size_t len = strlen(lower); - if (len > IS_ENV_EXT && strcmp(lower + len - IS_ENV_EXT, ".env") == 0) { - return true; - } - return false; -} - -bool cbm_is_kustomize_file(const char *name) { - if (!name) { - return false; - } - char lower[CBM_SZ_256]; - to_lower(name, lower, sizeof(lower)); - if (strcmp(lower, "kustomization.yaml") == 0) { - return true; - } - return strcmp(lower, "kustomization.yml") == 0; -} - -bool cbm_is_k8s_manifest(const char *name, const char *content) { - if (!name || !content || cbm_is_kustomize_file(name)) { - return false; - } - enum { K8S_PEEK_SZ = 4097 }; - char buf[K8S_PEEK_SZ]; - size_t n = strnlen(content, IS_CONTENT_LIMIT); - memcpy(buf, content, n); - buf[n] = '\0'; - return ci_strstr(buf, "apiVersion:") != NULL; -} - -bool cbm_is_shell_script(const char *name, const char *ext) { - (void)name; - if (!ext) { - return false; - } - return (strcmp(ext, ".sh") == 0 || strcmp(ext, ".bash") == 0 || strcmp(ext, ".zsh") == 0); -} - -/* ── Secret detection ───────────────────────────────────────────── */ - -static bool key_is_secret(const char *key) { - static const char *patterns[] = { - "secret", "password", "passwd", "token", "api_key", - "apikey", "private_key", "credential", "auth_token", "access_key", - "client_secret", "signing_key", "encryption_key", "ssh_key", "deploy_key", - "service_account", "bearer", "jwt_secret", NULL}; - for (int i = 0; patterns[i]; i++) { - if (ci_strstr(key, patterns[i])) { - return true; - } - } - return false; -} - -bool cbm_is_secret_value(const char *value) { - if (!value || !*value) { - return false; - } - - /* -----BEGIN (PEM key) */ - if (ci_strstr(value, "-----BEGIN")) { - return true; - } - - /* Substring searches — Go uses regex MatchString which finds anywhere */ - const char *p; - - /* AKIA + 16 alnum (AWS key) */ - p = ci_strstr(value, "AKIA"); - if (p && count_alnum(p + IS_CMD_SKIP) >= IS_TOKEN_MIN_1) { - return true; - } - - /* sk- + 20 alnum (API key) */ - p = ci_strstr(value, "sk-"); - if (p && count_alnum(p + IS_RUN_SKIP) >= IS_TOKEN_MIN_2) { - return true; - } - - /* ghp_ + 36 alnum (GitHub PAT) */ - p = ci_strstr(value, "ghp_"); - if (p && count_alnum(p + IS_CMD_SKIP) >= GITHUB_PAT_MIN_ALNUM) { - return true; - } - - /* glpat- + 20 alnum/dash (GitLab PAT) */ - p = ci_strstr(value, "glpat-"); - if (p && count_alnum_dash(p + IS_EXPOSE_SKIP) >= IS_TOKEN_MIN_2) { - return true; - } - - /* xox[bps]- (Slack token) */ - p = ci_strstr(value, "xox"); - if (p && p[IS_RUN_SKIP] != '\0' && - (tolower((unsigned char)p[IS_RUN_SKIP]) == 'b' || - tolower((unsigned char)p[IS_RUN_SKIP]) == 'p' || - tolower((unsigned char)p[IS_RUN_SKIP]) == 's') && - p[IS_CMD_SKIP] == '-' && count_alnum_dash(p + IS_USER_SKIP) >= SKIP_ONE) { - return true; - } - - return false; -} - -bool cbm_is_secret_binding(const char *key, const char *value) { - if (key && key_is_secret(key)) { - return true; - } - if (value && cbm_is_secret_value(value)) { - return true; - } - return false; -} - -/* ── Clean JSON brackets ────────────────────────────────────────── */ - -/* Strip JSON array brackets, quotes, and normalize whitespace. */ -static void clean_json_array_inner(const char *s, size_t len, char *out, size_t out_sz) { - size_t pos = 0; - bool in_space = false; - for (size_t i = SKIP_ONE; i < len - SKIP_ONE && pos < out_sz - SKIP_ONE; i++) { - char c = s[i]; - if (c == '"') { - continue; - } - if (c == ',') { - c = ' '; - } - if (c == ' ' || c == '\t') { - if (!in_space && pos > 0) { - out[pos++] = ' '; - in_space = true; - } - } else { - out[pos++] = c; - in_space = false; - } - } - while (pos > 0 && out[pos - SKIP_ONE] == ' ') { - pos--; - } - out[pos] = '\0'; -} - -void cbm_clean_json_brackets(const char *s, char *out, size_t out_sz) { - if (!s || !out || out_sz == 0) { - return; - } - - size_t len = strlen(s); - if (len >= PAIR_LEN && s[0] == '[' && s[len - SKIP_ONE] == ']') { - clean_json_array_inner(s, len, out, out_sz); - } else { - snprintf(out, out_sz, "%s", s); - } -} - -/* ── Dockerfile parser ──────────────────────────────────────────── */ - -static void df_parse_from(const char *line, cbm_dockerfile_result_t *r) { - if (strncasecmp(line, "FROM", SLEN("FROM")) != 0) { - return; - } - const char *p = line + IS_FROM_SKIP; - if (*p != ' ' && *p != '\t') { - return; - } - p = skip_ws(p); - - /* Extract image */ - if (r->stage_count >= IS_MAX_STAGES) { - return; - } - int idx = r->stage_count; - extract_token(p, r->stage_images[idx], sizeof(r->stage_images[idx])); - - /* Advance past image */ - while (*p && *p != ' ' && *p != '\t') { - p++; - } - p = skip_ws(p); - - /* Check for AS */ - r->stage_names[idx][0] = '\0'; - if (strncasecmp(p, "AS", SLEN("AS")) == 0 && (p[PAIR_LEN] == ' ' || p[PAIR_LEN] == '\t')) { - p = skip_ws(p + IS_AS_SKIP); - extract_word(p, r->stage_names[idx], sizeof(r->stage_names[idx])); - } - r->stage_count++; -} - -static void df_parse_expose(const char *line, cbm_dockerfile_result_t *r) { - if (strncasecmp(line, "EXPOSE", SLEN("EXPOSE")) != 0) { - return; - } - const char *p = line + IS_EXPOSE_SKIP; - if (*p != ' ' && *p != '\t') { - return; - } - p = skip_ws(p); - - /* Parse space-separated ports */ - while (*p && r->port_count < IS_MAX_PORTS) { - char port[CBM_SZ_32]; - int n = extract_token(p, port, sizeof(port)); - if (n == 0) { - break; - } - - /* Strip protocol suffix (e.g. 8080/tcp) */ - char *slash = strchr(port, '/'); - if (slash) { - *slash = '\0'; - } - - snprintf(r->exposed_ports[r->port_count], sizeof(r->exposed_ports[0]), "%s", port); - r->port_count++; - - p += n; - p = skip_ws(p); - } -} - -static void df_parse_env(const char *line, cbm_dockerfile_result_t *r) { - if (strncasecmp(line, "ENV", SLEN("ENV")) != 0) { - return; - } - const char *p = line + IS_RUN_SKIP; - if (*p != ' ' && *p != '\t') { - return; - } - p = skip_ws(p); - - if (r->env_count >= CBM_SZ_64) { - return; - } - - /* Extract key */ - char key[CBM_SZ_128]; - int klen = extract_word(p, key, sizeof(key)); - if (klen == 0) { - return; - } - p += klen; - - /* Separator: = or space */ - if (*p != '=' && *p != ' ' && *p != '\t') { - return; - } - p++; - /* Skip additional whitespace after separator */ - while (*p == ' ' || *p == '\t') { - p++; - } - - /* Value is rest of line (trimmed) */ - char value[CBM_SZ_512]; - snprintf(value, sizeof(value), "%s", p); - rtrim(value); - - /* Filter secrets */ - if (cbm_is_secret_binding(key, value)) { - return; - } - - snprintf(r->env_vars[r->env_count].key, sizeof(r->env_vars[0].key), "%s", key); - snprintf(r->env_vars[r->env_count].value, sizeof(r->env_vars[0].value), "%s", value); - r->env_count++; -} - -static void df_parse_arg(const char *line, cbm_dockerfile_result_t *r) { - if (strncasecmp(line, "ARG", SLEN("ARG")) != 0) { - return; - } - const char *p = line + IS_RUN_SKIP; - if (*p != ' ' && *p != '\t') { - return; - } - p = skip_ws(p); - - if (r->build_arg_count >= CBM_SZ_32) { - return; - } - extract_word(p, r->build_args[r->build_arg_count], sizeof(r->build_args[0])); - if (r->build_args[r->build_arg_count][0]) { - r->build_arg_count++; - } -} - -/* Parse a Dockerfile CMD/ENTRYPOINT value: strip brackets, trim. */ -static void df_parse_json_cmd(const char *line, int prefix_len, char *out, size_t out_sz) { - const char *p = skip_ws(line + prefix_len); - char raw[CBM_SZ_512]; - snprintf(raw, sizeof(raw), "%s", p); - rtrim(raw); - cbm_clean_json_brackets(raw, out, out_sz); -} - -static void df_parse_directives(const char *line, cbm_dockerfile_result_t *r) { - if (strncasecmp(line, "WORKDIR", SLEN("WORKDIR")) == 0 && - (line[IS_EXPORT_SKIP] == ' ' || line[IS_EXPORT_SKIP] == '\t')) { - const char *p = skip_ws(line + IS_WORKDIR_SKIP); - snprintf(r->workdir, sizeof(r->workdir), "%s", p); - rtrim(r->workdir); - return; - } - if (strncasecmp(line, "CMD", SLEN("CMD")) == 0 && - (line[IS_RUN_SKIP] == ' ' || line[IS_RUN_SKIP] == '\t')) { - df_parse_json_cmd(line, IS_CMD_SKIP, r->cmd, sizeof(r->cmd)); - return; - } - if (strncasecmp(line, "ENTRYPOINT", SLEN("ENTRYPOINT")) == 0 && - (line[IS_ENTRYPOINT_OFF] == ' ' || line[IS_ENTRYPOINT_OFF] == '\t')) { - df_parse_json_cmd(line, LEN_HEALTHCHECK, r->entrypoint, sizeof(r->entrypoint)); - return; - } - if (strncasecmp(line, "USER", SLEN("USER")) == 0 && - (line[IS_CMD_SKIP] == ' ' || line[IS_CMD_SKIP] == '\t')) { - const char *p = skip_ws(line + IS_USER_SKIP); - extract_word(p, r->user, sizeof(r->user)); - return; - } - if (strncasecmp(line, "HEALTHCHECK", LEN_HEALTHCHECK) == 0 && - (line[LEN_HEALTHCHECK] == ' ' || line[LEN_HEALTHCHECK] == '\t')) { - const char *cmd_pos = ci_strstr(line + LEN_HEALTHCHECK, "CMD"); - if (cmd_pos) { - cmd_pos = skip_ws(cmd_pos + IS_RUN_SKIP); - snprintf(r->healthcheck, sizeof(r->healthcheck), "%s", cmd_pos); - rtrim(r->healthcheck); - } - } -} - -int cbm_parse_dockerfile_source(const char *source, cbm_dockerfile_result_t *out) { - if (!source || !out) { - return CBM_NOT_FOUND; - } - memset(out, 0, sizeof(*out)); - - /* Process line by line */ - const char *p = source; - char line[CBM_SZ_4K]; - - while (*p) { - /* Extract one line */ - const char *eol = strchr(p, '\n'); - size_t line_len = eol ? (size_t)(eol - p) : strlen(p); - if (line_len >= sizeof(line)) { - line_len = sizeof(line) - SKIP_ONE; - } - memcpy(line, p, line_len); - line[line_len] = '\0'; - p = eol ? eol + SKIP_ONE : p + line_len; - - /* Trim */ - char *trimmed = line; - while (*trimmed == ' ' || *trimmed == '\t') { - trimmed++; - } - rtrim(trimmed); - - /* Skip empty lines and comments */ - if (*trimmed == '\0' || *trimmed == '#') { - continue; - } - - df_parse_from(trimmed, out); - df_parse_expose(trimmed, out); - df_parse_env(trimmed, out); - df_parse_arg(trimmed, out); - df_parse_directives(trimmed, out); - } - - /* No stages = empty/invalid Dockerfile */ - if (out->stage_count == 0) { - return CBM_NOT_FOUND; - } - - /* base_image = last stage's image (use intermediate copy to avoid restrict overlap) */ - { - char tmp[sizeof(out->base_image)]; - snprintf(tmp, sizeof(tmp), "%s", out->stage_images[out->stage_count - SKIP_ONE]); - memcpy(out->base_image, tmp, sizeof(out->base_image)); - } - - return 0; -} - -/* ── Dotenv parser ──────────────────────────────────────────────── */ - -/* Strip matching surrounding quotes from a value string in-place. */ -static void strip_surrounding_quotes(char *value) { - size_t vlen = strlen(value); - if (vlen >= CBM_QUOTE_PAIR && ((value[0] == '"' && value[vlen - SKIP_ONE] == '"') || - (value[0] == '\'' && value[vlen - SKIP_ONE] == '\''))) { - memmove(value, value + CBM_QUOTE_OFFSET, vlen - CBM_QUOTE_PAIR); - value[vlen - IS_QUOTE_TRIM] = '\0'; - } -} - -/* Parse a single dotenv line: KEY=VALUE. Returns 1 if env var added. */ -static int parse_dotenv_line(const char *trimmed, cbm_dotenv_result_t *out) { - char key[CBM_SZ_128]; - int klen = 0; - if (isalpha((unsigned char)trimmed[0]) || trimmed[0] == '_') { - klen = extract_word(trimmed, key, sizeof(key)); - } - if (klen == 0 || trimmed[klen] != '=') { - return 0; - } - - char value[CBM_SZ_512]; - snprintf(value, sizeof(value), "%s", trimmed + klen + SKIP_ONE); - rtrim(value); - strip_surrounding_quotes(value); - - if (cbm_is_secret_binding(key, value) || out->env_count >= CBM_SZ_64) { - return 0; - } - snprintf(out->env_vars[out->env_count].key, sizeof(out->env_vars[0].key), "%s", key); - snprintf(out->env_vars[out->env_count].value, sizeof(out->env_vars[0].value), "%s", value); - out->env_count++; - return SKIP_ONE; -} - -int cbm_parse_dotenv_source(const char *source, cbm_dotenv_result_t *out) { - if (!source || !out) { - return CBM_NOT_FOUND; - } - memset(out, 0, sizeof(*out)); - - const char *p = source; - char line[CBM_SZ_4K]; - - while (*p) { - const char *eol = strchr(p, '\n'); - size_t line_len = eol ? (size_t)(eol - p) : strlen(p); - if (line_len >= sizeof(line)) { - line_len = sizeof(line) - SKIP_ONE; - } - memcpy(line, p, line_len); - line[line_len] = '\0'; - p = eol ? eol + SKIP_ONE : p + line_len; - - char *trimmed = line; - while (*trimmed == ' ' || *trimmed == '\t') { - trimmed++; - } - rtrim(trimmed); - - if (*trimmed == '\0' || *trimmed == '#') { - continue; - } - - parse_dotenv_line(trimmed, out); - } - - return (out->env_count > 0) ? 0 : IS_NOT_FOUND; -} - -/* ── Shell script parser ────────────────────────────────────────── */ - -static void shell_parse_export(const char *line, cbm_shell_result_t *r) { - /* export VAR=VALUE */ - if (strncmp(line, "export", SLEN("export")) != 0 || - (line[IS_EXPOSE_SKIP] != ' ' && line[IS_EXPOSE_SKIP] != '\t')) { - return; - } - const char *p = skip_ws(line + IS_EXPORT_SKIP); - - char key[CBM_SZ_128]; - int klen = extract_word(p, key, sizeof(key)); - if (klen == 0 || p[klen] != '=') { - return; - } - - const char *val_start = p + klen + SKIP_ONE; - char value[CBM_SZ_512]; - snprintf(value, sizeof(value), "%s", val_start); - rtrim(value); - - /* Strip surrounding quotes */ - size_t vlen = strlen(value); - if (vlen >= CBM_QUOTE_PAIR && ((value[0] == '"' && value[vlen - SKIP_ONE] == '"') || - (value[0] == '\'' && value[vlen - SKIP_ONE] == '\''))) { - memmove(value, value + CBM_QUOTE_OFFSET, vlen - CBM_QUOTE_PAIR); - value[vlen - IS_QUOTE_TRIM] = '\0'; - } - - if (cbm_is_secret_binding(key, value)) { - return; - } - if (r->env_count >= CBM_SZ_64) { - return; - } - snprintf(r->env_vars[r->env_count].key, sizeof(r->env_vars[0].key), "%s", key); - snprintf(r->env_vars[r->env_count].value, sizeof(r->env_vars[0].value), "%s", value); - r->env_count++; -} - -static void shell_parse_plain_var(const char *line, cbm_shell_result_t *r) { - /* VAR=VALUE (only if no spaces in line — avoids matching commands) */ - if (strchr(line, ' ') || strchr(line, '\t')) { - return; - } - - char key[CBM_SZ_128]; - int klen = extract_word(line, key, sizeof(key)); - if (klen == 0 || line[klen] != '=') { - return; - } - - const char *val_start = line + klen + SKIP_ONE; - char value[CBM_SZ_512]; - snprintf(value, sizeof(value), "%s", val_start); - rtrim(value); - - /* Strip surrounding quotes */ - size_t vlen = strlen(value); - if (vlen >= CBM_QUOTE_PAIR && ((value[0] == '"' && value[vlen - SKIP_ONE] == '"') || - (value[0] == '\'' && value[vlen - SKIP_ONE] == '\''))) { - memmove(value, value + CBM_QUOTE_OFFSET, vlen - CBM_QUOTE_PAIR); - value[vlen - IS_QUOTE_TRIM] = '\0'; - } - - if (cbm_is_secret_binding(key, value)) { - return; - } - if (r->env_count >= CBM_SZ_64) { - return; - } - snprintf(r->env_vars[r->env_count].key, sizeof(r->env_vars[0].key), "%s", key); - snprintf(r->env_vars[r->env_count].value, sizeof(r->env_vars[0].value), "%s", value); - r->env_count++; -} - -static void shell_parse_source(const char *line, cbm_shell_result_t *r) { - /* source or . */ - const char *p = NULL; - if (strncmp(line, "source", SLEN("source")) == 0 && - (line[IS_EXPOSE_SKIP] == ' ' || line[IS_EXPOSE_SKIP] == '\t')) { - p = skip_ws(line + IS_EXPORT_SKIP); - } else if (line[0] == '.' && (line[SKIP_ONE] == ' ' || line[SKIP_ONE] == '\t')) { - p = skip_ws(line + IS_SOURCE_DOT); - } - if (!p) { - return; - } - - if (r->source_count >= IS_MAX_SOURCES) { - return; - } - - /* Strip surrounding quotes */ - char path[CBM_SZ_256]; - snprintf(path, sizeof(path), "%s", p); - rtrim(path); - size_t plen = strlen(path); - if (plen >= CBM_QUOTE_PAIR && ((path[0] == '"' && path[plen - SKIP_ONE] == '"') || - (path[0] == '\'' && path[plen - SKIP_ONE] == '\''))) { - memmove(path, path + CBM_QUOTE_OFFSET, plen - CBM_QUOTE_PAIR); - path[plen - IS_QUOTE_TRIM] = '\0'; - } - - /* Extract just the path token (no trailing args) */ - char *space = strchr(path, ' '); - if (space) { - *space = '\0'; - } - - snprintf(r->sources[r->source_count], sizeof(r->sources[0]), "%s", path); - r->source_count++; -} - -static void shell_parse_docker(const char *line, cbm_shell_result_t *r) { - /* docker or docker-compose */ - const char *p = NULL; - const char *tool = NULL; - - if (strncmp(line, "docker-compose", LEN_DOCKER_COMPOSE) == 0 && - (line[LEN_DOCKER_COMPOSE] == ' ' || line[LEN_DOCKER_COMPOSE] == '\t')) { - tool = "docker-compose"; - p = skip_ws(line + LEN_DOCKER_COMPOSE_SKIP); - } else if (strncmp(line, "docker", SLEN("docker")) == 0 && - (line[IS_EXPOSE_SKIP] == ' ' || line[IS_EXPOSE_SKIP] == '\t')) { - tool = "docker"; - p = skip_ws(line + IS_EXPORT_SKIP); - } - if (!tool || !p || !*p) { - return; - } - - if (r->docker_cmd_count >= IS_MAX_STAGES) { - return; - } - - char subcmd[CBM_SZ_64]; - extract_word(p, subcmd, sizeof(subcmd)); - if (subcmd[0]) { - snprintf(r->docker_cmds[r->docker_cmd_count], sizeof(r->docker_cmds[0]), "%s %s", tool, - subcmd); - r->docker_cmd_count++; - } -} - -/* Check for shebang on first non-empty line. Returns true if line was consumed. */ -static bool shell_try_shebang(const char *trimmed, cbm_shell_result_t *out, bool *shebang_checked) { - if (*shebang_checked) { - return false; - } - if (trimmed[0] == '#' && trimmed[SKIP_ONE] == '!') { - const char *sb = trimmed + PAIR_LEN; - while (*sb == ' ') { - sb++; - } - snprintf(out->shebang, sizeof(out->shebang), "%s", sb); - rtrim(out->shebang); - *shebang_checked = true; - return true; - } - *shebang_checked = true; - return false; -} - -/* Dispatch a single trimmed shell line to the appropriate parser. */ -static void shell_dispatch_line(const char *trimmed, cbm_shell_result_t *out) { - if (strncmp(trimmed, "export", SLEN("export")) == 0) { - shell_parse_export(trimmed, out); - return; - } - if (strncmp(trimmed, "source", SLEN("source")) == 0 || - (trimmed[0] == '.' && trimmed[SKIP_ONE] == ' ')) { - shell_parse_source(trimmed, out); - return; - } - if (strncmp(trimmed, "docker", SLEN("docker")) == 0) { - shell_parse_docker(trimmed, out); - } - shell_parse_plain_var(trimmed, out); -} - -int cbm_parse_shell_source(const char *source, cbm_shell_result_t *out) { - if (!source || !out) { - return CBM_NOT_FOUND; - } - memset(out, 0, sizeof(*out)); - - const char *p = source; - char line[CBM_SZ_4K]; - bool shebang_checked = false; - - while (*p) { - const char *eol = strchr(p, '\n'); - size_t line_len = eol ? (size_t)(eol - p) : strlen(p); - if (line_len >= sizeof(line)) { - line_len = sizeof(line) - SKIP_ONE; - } - memcpy(line, p, line_len); - line[line_len] = '\0'; - p = eol ? eol + SKIP_ONE : p + line_len; - - char *trimmed = line; - while (*trimmed == ' ' || *trimmed == '\t') { - trimmed++; - } - rtrim(trimmed); - - if (*trimmed == '\0') { - continue; - } - - if (shell_try_shebang(trimmed, out, &shebang_checked)) { - continue; - } - - if (*trimmed == '#') { - continue; - } - - shell_dispatch_line(trimmed, out); - } - - bool has_content = out->shebang[0] != '\0' || out->env_count > 0 || out->source_count > 0 || - out->docker_cmd_count > 0; - return has_content ? 0 : IS_NOT_FOUND; -} - -/* ── Terraform parser ───────────────────────────────────────────── */ - -typedef enum { - TF_BLOCK_NONE = 0, - TF_BLOCK_VARIABLE, - TF_BLOCK_MODULE, - TF_BLOCK_TERRAFORM -} tf_block_kind_t; - -/* Extract a double-quoted string value after = sign. - * Handles: key = "value" and key = value */ -static void tf_extract_quoted(const char *line, const char *prefix, char *out, size_t out_sz) { - const char *p = skip_ws(line); - size_t plen = strlen(prefix); - if (strncmp(p, prefix, plen) != 0) { - return; - } - p = skip_ws(p + plen); - if (*p != '=') { - return; - } - p = skip_ws(p + SKIP_ONE); - - /* Strip optional quotes */ - if (*p == '"') { - p++; - const char *end = strchr(p, '"'); - size_t vlen = end ? (size_t)(end - p) : strlen(p); - if (vlen >= out_sz) { - vlen = out_sz - SKIP_ONE; - } - memcpy(out, p, vlen); - out[vlen] = '\0'; - } else { - extract_token(p, out, out_sz); - } -} - -/* Count occurrences of a character in a string */ -static int count_char(const char *s, char c) { - int n = 0; - for (; *s; s++) { - if (*s == c) { - n++; - } - } - return n; -} - -/* Extract double-quoted identifier after keyword. - * Matches: keyword "identifier" → sets out to "identifier" - * Returns number of chars consumed from p, or 0 if no match. */ -static int tf_extract_ident(const char *p, const char *keyword, char *out, size_t out_sz) { - size_t klen = strlen(keyword); - if (strncmp(p, keyword, klen) != 0) { - return 0; - } - const char *after = skip_ws(p + klen); - if (*after != '"') { - return 0; - } - after++; - const char *end = strchr(after, '"'); - if (!end) { - return 0; - } - size_t ilen = (size_t)(end - after); - if (ilen >= out_sz) { - ilen = out_sz - SKIP_ONE; - } - memcpy(out, after, ilen); - out[ilen] = '\0'; - return (int)(end - p + SKIP_ONE); -} - -/* Parse a second quoted string after the first ident (for resource/data blocks). */ -static void tf_extract_second_quoted(const char *trimmed, int consumed, char *out, size_t outsz) { - const char *rest = skip_ws(trimmed + consumed); - out[0] = '\0'; - if (*rest == '"') { - rest++; - const char *end = strchr(rest, '"'); - if (end) { - size_t len = (size_t)(end - rest); - if (len >= outsz) { - len = outsz - SKIP_ONE; - } - memcpy(out, rest, len); - out[len] = '\0'; - } - } -} - -/* Try to parse a simple "keyword ident" block (variable, output, provider). */ -static bool tf_try_simple_block(const char *trimmed, const char *keyword, char *names, int name_sz, - int *count, int max_count) { - char ident[CBM_SZ_128]; - int consumed = tf_extract_ident(trimmed, keyword, ident, sizeof(ident)); - if (consumed <= 0 || *count >= max_count) { - return false; - } - snprintf(names + ((size_t)(*count) * (size_t)name_sz), name_sz, "%s", ident); - (*count)++; - return true; -} - -/* Try to match terraform/locals keyword blocks. */ -static bool tf_try_keyword_block(const char *trimmed, tf_block_kind_t *cur_block, - cbm_terraform_result_t *out) { - if (strncmp(trimmed, "terraform", SLEN("terraform")) == 0 && strstr(trimmed, "{")) { - *cur_block = TF_BLOCK_TERRAFORM; - return true; - } - if (strncmp(trimmed, "locals", SLEN("locals")) == 0 && strstr(trimmed, "{")) { - out->has_locals = true; - return true; - } - return false; -} - -/* Detect a top-level block header. Returns true if line was consumed as a header. */ -static bool tf_detect_block_header(const char *trimmed, cbm_terraform_result_t *out, - tf_block_kind_t *cur_block, int *cur_var_idx, int *cur_mod_idx) { - char ident1[CBM_SZ_128]; - - /* resource "type" "name" */ - { - int rc = tf_extract_ident(trimmed, "resource", ident1, sizeof(ident1)); - if (rc > 0 && out->resource_count < CBM_SZ_32) { - char ident2[CBM_SZ_128]; - tf_extract_second_quoted(trimmed, rc, ident2, sizeof(ident2)); - snprintf(out->resources[out->resource_count].type, sizeof(out->resources[0].type), "%s", - ident1); - snprintf(out->resources[out->resource_count].name, sizeof(out->resources[0].name), "%s", - ident2); - out->resource_count++; - return true; - } - } - - /* variable "name" */ - int consumed = tf_extract_ident(trimmed, "variable", ident1, sizeof(ident1)); - if (consumed > 0 && out->variable_count < CBM_SZ_32) { - *cur_block = TF_BLOCK_VARIABLE; - *cur_var_idx = out->variable_count; - snprintf(out->variables[*cur_var_idx].name, sizeof(out->variables[0].name), "%s", ident1); - out->variable_count++; - return true; - } - - /* output "name" */ - if (tf_try_simple_block(trimmed, "output", out->outputs[0], (int)sizeof(out->outputs[0]), - &out->output_count, CBM_SZ_32)) { - return true; - } - - /* provider "name" */ - if (tf_try_simple_block(trimmed, "provider", out->providers[0], (int)sizeof(out->providers[0]), - &out->provider_count, IS_MAX_STAGES)) { - return true; - } - - /* module "name" */ - consumed = tf_extract_ident(trimmed, "module", ident1, sizeof(ident1)); - if (consumed > 0 && out->module_count < IS_MAX_STAGES) { - *cur_block = TF_BLOCK_MODULE; - *cur_mod_idx = out->module_count; - snprintf(out->modules[*cur_mod_idx].tf_name, sizeof(out->modules[0].tf_name), "%s", ident1); - out->module_count++; - return true; - } - - /* data "type" "name" */ - { - int dc = tf_extract_ident(trimmed, "data", ident1, sizeof(ident1)); - if (dc > 0 && out->data_source_count < IS_MAX_STAGES) { - char ident2[CBM_SZ_128]; - tf_extract_second_quoted(trimmed, dc, ident2, sizeof(ident2)); - snprintf(out->data_sources[out->data_source_count].type, - sizeof(out->data_sources[0].type), "%s", ident1); - snprintf(out->data_sources[out->data_source_count].name, - sizeof(out->data_sources[0].name), "%s", ident2); - out->data_source_count++; - return true; - } - } - - return tf_try_keyword_block(trimmed, cur_block, out); -} - -/* Extract attributes from a line inside a variable block. */ -static void tf_parse_variable_attrs(const char *trimmed, cbm_tf_variable_t *v) { - char def_val[CBM_SZ_256] = {0}; - tf_extract_quoted(trimmed, "default", def_val, sizeof(def_val)); - if (def_val[0] && !cbm_is_secret_binding(v->name, def_val)) { - snprintf(v->default_val, sizeof(v->default_val), "%s", def_val); - } - char type_val[CBM_SZ_64] = {0}; - tf_extract_quoted(trimmed, "type", type_val, sizeof(type_val)); - if (type_val[0]) { - snprintf(v->type, sizeof(v->type), "%s", type_val); - } - char desc_val[CBM_SZ_256] = {0}; - tf_extract_quoted(trimmed, "description", desc_val, sizeof(desc_val)); - if (desc_val[0]) { - snprintf(v->description, sizeof(v->description), "%s", desc_val); - } -} - -/* Extract attributes inside a block (variable/module/terraform). */ -static void tf_parse_block_attrs(const char *trimmed, cbm_terraform_result_t *out, - tf_block_kind_t cur_block, int cur_var_idx, int cur_mod_idx) { - switch (cur_block) { - case TF_BLOCK_VARIABLE: - if (cur_var_idx >= 0) { - tf_parse_variable_attrs(trimmed, &out->variables[cur_var_idx]); - } - break; - case TF_BLOCK_MODULE: - if (cur_mod_idx >= 0) { - char src_val[CBM_SZ_256] = {0}; - tf_extract_quoted(trimmed, "source", src_val, sizeof(src_val)); - if (src_val[0]) { - snprintf(out->modules[cur_mod_idx].source, sizeof(out->modules[0].source), "%s", - src_val); - } - } - break; - case TF_BLOCK_TERRAFORM: { - const char *bp = skip_ws(trimmed); - char backend_name[CBM_SZ_128] = {0}; - tf_extract_ident(bp, "backend", backend_name, sizeof(backend_name)); - if (backend_name[0]) { - snprintf(out->backend, sizeof(out->backend), "%s", backend_name); - } - break; - } - default: - break; - } -} - -/* Check if any terraform content was parsed. */ -static bool tf_result_has_content(const cbm_terraform_result_t *out) { - return out->resource_count > 0 || out->variable_count > 0 || out->output_count > 0 || - out->provider_count > 0 || out->module_count > 0 || out->data_source_count > 0 || - out->backend[0] != '\0' || out->has_locals; -} - -/* Read next non-empty, non-comment line. Advances *pp. Returns trimmed or NULL. */ -static char *tf_next_line(const char **pp, char *line, size_t line_sz) { - const char *p = *pp; - const char *eol = strchr(p, '\n'); - size_t len = eol ? (size_t)(eol - p) : strlen(p); - if (len >= line_sz) { - len = line_sz - SKIP_ONE; - } - memcpy(line, p, len); - line[len] = '\0'; - *pp = eol ? eol + SKIP_ONE : p + len; - - char *trimmed = line; - while (*trimmed == ' ' || *trimmed == '\t') { - trimmed++; - } - rtrim(trimmed); - if (*trimmed == '\0' || *trimmed == '#' || (trimmed[0] == '/' && trimmed[SKIP_ONE] == '/')) { - return NULL; - } - return trimmed; -} - -int cbm_parse_terraform_source(const char *source, cbm_terraform_result_t *out) { - if (!source || !out) { - return CBM_NOT_FOUND; - } - memset(out, 0, sizeof(*out)); - - const char *p = source; - char line[CBM_SZ_4K]; - int brace_depth = 0; - tf_block_kind_t cur_block = TF_BLOCK_NONE; - int cur_var_idx = IS_NOT_FOUND; - int cur_mod_idx = IS_NOT_FOUND; - - while (*p) { - char *trimmed = tf_next_line(&p, line, sizeof(line)); - if (!trimmed) { - continue; - } - - brace_depth += count_char(trimmed, '{') - count_char(trimmed, '}'); - - if (brace_depth <= SKIP_ONE && cur_block == TF_BLOCK_NONE) { - if (tf_detect_block_header(trimmed, out, &cur_block, &cur_var_idx, &cur_mod_idx)) { - continue; - } - } - - if (cur_block != TF_BLOCK_NONE && brace_depth >= SKIP_ONE && brace_depth <= PAIR_LEN) { - tf_parse_block_attrs(trimmed, out, cur_block, cur_var_idx, cur_mod_idx); - } - - if (brace_depth == 0 && cur_block != TF_BLOCK_NONE) { - cur_block = TF_BLOCK_NONE; - cur_var_idx = IS_NOT_FOUND; - cur_mod_idx = IS_NOT_FOUND; - } - } - - return tf_result_has_content(out) ? 0 : IS_NOT_FOUND; -} - -/* ── Infra QN helper ────────────────────────────────────────────── */ - -char *cbm_infra_qn(const char *project_name, const char *rel_path, const char *infra_type, - const char *service_name) { - char *base = cbm_pipeline_fqn_compute(project_name, rel_path, ""); - if (!base) { - return NULL; - } - - char result[CBM_SZ_1K]; - if (service_name && service_name[0] && infra_type && - strcmp(infra_type, "compose-service") == 0) { - snprintf(result, sizeof(result), "%s::%s", base, service_name); - } else { - snprintf(result, sizeof(result), "%s.__infra__", base); - } - - free(base); - return strdup(result); -} - -/* ── Helm Chart.yaml parser (#338) ───────────────────────────────── */ - -/* Copy the scalar value following a "key:" up to end-of-line into out, - * trimming surrounding whitespace, optional quotes, and trailing comments. */ -static void helm_copy_scalar(const char *s, const char *eol, char *out, size_t cap) { - while (s < eol && (*s == ' ' || *s == '\t')) { - s++; - } - char quote = 0; - if (s < eol && (*s == '"' || *s == '\'')) { - quote = *s; - s++; - } - const char *e = s; - while (e < eol) { - if (quote) { - if (*e == quote) { - break; - } - } else if (*e == ' ' || *e == '\t' || *e == '#' || *e == '\r') { - break; - } - e++; - } - size_t n = (size_t)(e - s); - if (n >= cap) { - n = cap - 1; - } - memcpy(out, s, n); - out[n] = '\0'; -} - -/* Match "key:" at the start of a content pointer; return value start or NULL. */ -static const char *helm_match_key(const char *content, const char *eol, const char *key) { - size_t kl = strlen(key); - if ((size_t)(eol - content) < kl + 1) { - return NULL; - } - if (strncmp(content, key, kl) != 0 || content[kl] != ':') { - return NULL; - } - return content + kl + 1; -} - -int cbm_parse_helm_chart(const char *source, cbm_helm_chart_t *out) { - if (!source || !out) { - return -1; - } - out->chart_name[0] = '\0'; - out->dep_count = 0; - - bool in_deps = false; - const char *p = source; - while (*p) { - const char *eol = strchr(p, '\n'); - if (!eol) { - eol = p + strlen(p); - } - int indent = 0; - const char *content = p; - while (content < eol && *content == ' ') { - content++; - indent++; - } - if (content < eol && *content != '#') { - if (indent == 0) { - /* New top-level key ends any dependencies block. */ - in_deps = (helm_match_key(content, eol, "dependencies") != NULL); - const char *nv = helm_match_key(content, eol, "name"); - if (!in_deps && nv && out->chart_name[0] == '\0') { - helm_copy_scalar(nv, eol, out->chart_name, CBM_HELM_NAME_MAX); - } - } else if (in_deps) { - /* List item: optional leading "- ", then look for name:. */ - const char *q = content; - if (*q == '-') { - q++; - while (q < eol && *q == ' ') { - q++; - } - } - const char *nv = helm_match_key(q, eol, "name"); - if (nv && out->dep_count < CBM_HELM_MAX_DEPS) { - char buf[CBM_HELM_NAME_MAX]; - helm_copy_scalar(nv, eol, buf, CBM_HELM_NAME_MAX); - if (buf[0]) { - memcpy(out->deps[out->dep_count], buf, CBM_HELM_NAME_MAX); - out->dep_count++; - } - } - } - } - if (*eol == '\0') { - break; - } - p = eol + 1; - } - return (out->chart_name[0] || out->dep_count > 0) ? 0 : -1; -} diff --git a/src/pipeline/pass_infrascan.zig b/src/pipeline/pass_infrascan.zig new file mode 100644 index 00000000..5c1ce822 --- /dev/null +++ b/src/pipeline/pass_infrascan.zig @@ -0,0 +1,1049 @@ +//! pass_infrascan.zig — Infrastructure file detection and parsing helpers. +//! +//! Pure helper functions for detecting infrastructure file types, +//! secret filtering, and parsing Dockerfiles, .env files, shell scripts, +//! and Terraform files. + +const std = @import("std"); +const c = @import("c"); +const util = @import("util.zig"); + +const CStr = util.CStr; +const MutStr = util.MutStr; +const ix = util.ix; +const streq = util.streq; +const Str = [*:0]const u8; +const DockerfileResult = c.cbm_dockerfile_result_t; +const DotenvResult = c.cbm_dotenv_result_t; +const ShellResult = c.cbm_shell_result_t; +const TerraformResult = c.cbm_terraform_result_t; + +const IS_ENV_EXT = 4; // strlen(".env") +const IS_CONTENT_LIMIT = 4096; +const IS_MAX_STAGES = 16; +const IS_MAX_PORTS = 16; +const IS_MAX_SOURCES = 16; +const IS_TOKEN_MIN_1 = 16; // min alnum for AWS key +const IS_TOKEN_MIN_2 = 20; // min alnum for GH classic token +const IS_NOT_FOUND = -1; + +/// String length constants for keyword matching +const LEN_DOCKERFILE = 11; // strlen("dockerfile.") == strlen(".dockerfile") +const LEN_DOCKER_COMPOSE = 14; // strlen("docker-compose") +const LEN_HEALTHCHECK = 11; // strlen("HEALTHCHECK") + +/// Minimum alnum chars after prefix for secret detection +const GITHUB_PAT_MIN_ALNUM = 36; // ghp_ + 36 alnum chars + +// ── Internal helpers ──────────────────────────────────────────────── + +fn toLower(src: CStr, dst: []u8) void { + var len = c.strlen(src); + if (len >= dst.len) len = dst.len - 1; + for (0..len) |i| dst[i] = std.ascii.toLower(src[i]); + dst[len] = 0; +} + +/// strncmp(s, prefix, strlen(prefix)) == 0. +fn hasPrefix(s: [*]const u8, comptime prefix: [:0]const u8) bool { + return c.strncmp(s, prefix, prefix.len) == 0; +} + +/// strncasecmp(s, prefix, strlen(prefix)) == 0. +fn hasPrefixCi(s: [*]const u8, prefix: []const u8) bool { + for (prefix, 0..) |ch, i| { + if (std.ascii.toLower(s[i]) != std.ascii.toLower(ch)) return false; + } + return true; +} + +fn isBlank(ch: u8) bool { + return ch == ' ' or ch == '\t'; +} + +/// Case-insensitive substring search +fn ciStrstr(haystack_in: [*:0]const u8, needle: []const u8) ?[*:0]const u8 { + if (needle.len == 0) return haystack_in; + var haystack = haystack_in; + while (haystack[0] != 0) : (haystack += 1) { + if (hasPrefixCi(haystack, needle)) return haystack; + } + return null; +} + +/// Count consecutive alphanumeric characters +fn countAlnum(s: [*]const u8) c_int { + var n: usize = 0; + while (std.ascii.isAlphanumeric(s[n])) n += 1; + return @intCast(n); +} + +/// Count consecutive alphanumeric or dash characters +fn countAlnumDash(s: [*]const u8) c_int { + var n: usize = 0; + while (std.ascii.isAlphanumeric(s[n]) or s[n] == '-') n += 1; + return @intCast(n); +} + +/// Skip whitespace, return pointer past it +fn skipWs(p_in: [*:0]const u8) [*:0]const u8 { + var p = p_in; + while (isBlank(p[0])) p += 1; + return p; +} + +/// Extract a word ([a-zA-Z0-9_]+) into buf. Returns length. +fn extractWord(p: [*]const u8, buf: []u8) usize { + var n: usize = 0; + while ((std.ascii.isAlphanumeric(p[n]) or p[n] == '_') and n < buf.len - 1) { + buf[n] = p[n]; + n += 1; + } + buf[n] = 0; + return n; +} + +/// Extract a non-space token into buf. Returns length. +fn extractToken(p: [*]const u8, buf: []u8) usize { + var n: usize = 0; + while (p[n] != 0 and !isBlank(p[n]) and n < buf.len - 1) { + buf[n] = p[n]; + n += 1; + } + buf[n] = 0; + return n; +} + +/// Trim trailing whitespace in-place +fn rtrim(s: [*:0]u8) void { + var len = c.strlen(s); + while (len > 0 and (s[len - 1] == ' ' or s[len - 1] == '\t' or + s[len - 1] == '\r' or s[len - 1] == '\n')) + { + len -= 1; + s[len] = 0; + } +} + +/// snprintf(dst, sizeof(dst), "%s", src) for a fixed char array. +fn copyStr(dst: []u8, src: CStr) void { + _ = c.snprintf(dst.ptr, dst.len, "%s", src); +} + +// ── File identification ───────────────────────────────────────────── + +pub export fn cbm_is_dockerfile(name: CStr) bool { + if (name == null) return false; + var lower: [c.CBM_SZ_256:0]u8 = undefined; + toLower(name, &lower); + + if (streq(&lower, "dockerfile")) return true; + if (c.strncmp(&lower, "dockerfile.", LEN_DOCKERFILE) == 0) return true; + const len = c.strlen(&lower); + if (len > LEN_DOCKERFILE and streq(@as(CStr, &lower) + len - LEN_DOCKERFILE, ".dockerfile")) { + return true; + } + return false; +} + +/// True when the lowercased name ends in .yml or .yaml. +fn hasYamlExt(lower: CStr) bool { + const ext: CStr = c.strrchr(lower, '.'); + if (ext == null) return false; + return streq(ext, ".yml") or streq(ext, ".yaml"); +} + +pub export fn cbm_is_compose_file(name: CStr) bool { + if (name == null) return false; + var lower: [c.CBM_SZ_256:0]u8 = undefined; + toLower(name, &lower); + + const prefix_match = c.strncmp(&lower, "docker-compose", LEN_DOCKER_COMPOSE) == 0 or + streq(&lower, "compose.yml") or streq(&lower, "compose.yaml"); + if (!prefix_match) return false; + + return hasYamlExt(&lower); +} + +pub export fn cbm_is_cloudbuild_file(name: CStr) bool { + if (name == null) return false; + var lower: [c.CBM_SZ_256:0]u8 = undefined; + toLower(name, &lower); + + if (!hasPrefix(&lower, "cloudbuild")) return false; + return hasYamlExt(&lower); +} + +pub export fn cbm_is_env_file(name: CStr) bool { + if (name == null) return false; + var lower: [c.CBM_SZ_256:0]u8 = undefined; + toLower(name, &lower); + + if (streq(&lower, ".env")) return true; + if (hasPrefix(&lower, ".env.")) return true; + const len = c.strlen(&lower); + if (len > IS_ENV_EXT and streq(@as(CStr, &lower) + len - IS_ENV_EXT, ".env")) return true; + return false; +} + +pub export fn cbm_is_kustomize_file(name: CStr) bool { + if (name == null) return false; + var lower: [c.CBM_SZ_256:0]u8 = undefined; + toLower(name, &lower); + if (streq(&lower, "kustomization.yaml")) return true; + return streq(&lower, "kustomization.yml"); +} + +pub export fn cbm_is_k8s_manifest(name: CStr, content: CStr) bool { + if (name == null or content == null or cbm_is_kustomize_file(name)) return false; + var buf: [IS_CONTENT_LIMIT:0]u8 = undefined; + // strnlen(content, IS_CONTENT_LIMIT) + var n: usize = 0; + while (n < IS_CONTENT_LIMIT and content[n] != 0) n += 1; + @memcpy(buf[0..n], content[0..n]); + buf[n] = 0; + return ciStrstr(&buf, "apiVersion:") != null; +} + +pub export fn cbm_is_shell_script(name: CStr, ext: CStr) bool { + _ = name; + if (ext == null) return false; + return streq(ext, ".sh") or streq(ext, ".bash") or streq(ext, ".zsh"); +} + +// ── Secret detection ──────────────────────────────────────────────── + +fn keyIsSecret(key: Str) bool { + const patterns = [_][]const u8{ + "secret", "password", "passwd", "token", "api_key", + "apikey", "private_key", "credential", "auth_token", "access_key", + "client_secret", "signing_key", "encryption_key", "ssh_key", "deploy_key", + "service_account", "bearer", "jwt_secret", + }; + for (patterns) |pattern| { + if (ciStrstr(key, pattern) != null) return true; + } + return false; +} + +pub export fn cbm_is_secret_value(value_c: ?Str) bool { + const value = value_c orelse return false; + if (value[0] == 0) return false; + + // -----BEGIN (PEM key) + if (ciStrstr(value, "-----BEGIN") != null) return true; + + // Substring searches — Go uses regex MatchString which finds anywhere + + // AKIA + 16 alnum (AWS key) + if (ciStrstr(value, "AKIA")) |p| { + if (countAlnum(p + 4) >= IS_TOKEN_MIN_1) return true; + } + + // sk- + 20 alnum (API key) + if (ciStrstr(value, "sk-")) |p| { + if (countAlnum(p + 3) >= IS_TOKEN_MIN_2) return true; + } + + // ghp_ + 36 alnum (GitHub PAT) + if (ciStrstr(value, "ghp_")) |p| { + if (countAlnum(p + 4) >= GITHUB_PAT_MIN_ALNUM) return true; + } + + // glpat- + 20 alnum/dash (GitLab PAT) + if (ciStrstr(value, "glpat-")) |p| { + if (countAlnumDash(p + 6) >= IS_TOKEN_MIN_2) return true; + } + + // xox[bps]- (Slack token) + if (ciStrstr(value, "xox")) |p| { + if (p[3] != 0) { + const kind = std.ascii.toLower(p[3]); + if ((kind == 'b' or kind == 'p' or kind == 's') and p[4] == '-' and + countAlnumDash(p + 5) >= 1) + { + return true; + } + } + } + + return false; +} + +pub export fn cbm_is_secret_binding(key: ?Str, value: ?Str) bool { + if (key) |k| { + if (keyIsSecret(k)) return true; + } + if (value != null and cbm_is_secret_value(value)) return true; + return false; +} + +// ── Clean JSON brackets ───────────────────────────────────────────── + +/// Strip JSON array brackets, quotes, and normalize whitespace. +fn cleanJsonArrayInner(s: [*]const u8, len: usize, out: [*]u8, out_sz: usize) void { + var pos: usize = 0; + var in_space = false; + var i: usize = 1; + while (i < len - 1 and pos < out_sz - 1) : (i += 1) { + var ch = s[i]; + if (ch == '"') continue; + if (ch == ',') ch = ' '; + if (ch == ' ' or ch == '\t') { + if (!in_space and pos > 0) { + out[pos] = ' '; + pos += 1; + in_space = true; + } + } else { + out[pos] = ch; + pos += 1; + in_space = false; + } + } + while (pos > 0 and out[pos - 1] == ' ') pos -= 1; + out[pos] = 0; +} + +pub export fn cbm_clean_json_brackets(s: CStr, out: MutStr, out_sz: usize) void { + if (s == null or out == null or out_sz == 0) return; + + const len = c.strlen(s); + if (len >= 2 and s[0] == '[' and s[len - 1] == ']') { + cleanJsonArrayInner(s, len, out, out_sz); + } else { + _ = c.snprintf(out, out_sz, "%s", s); + } +} + +// ── Dockerfile parser ─────────────────────────────────────────────── + +fn dfParseFrom(line: Str, r: *DockerfileResult) void { + if (!hasPrefixCi(line, "FROM")) return; + var p: Str = line + 4; + if (!isBlank(p[0])) return; + p = skipWs(p); + + // Extract image + if (r.stage_count >= IS_MAX_STAGES) return; + const idx = ix(r.stage_count); + _ = extractToken(p, &r.stage_images[idx]); + + // Advance past image + while (p[0] != 0 and !isBlank(p[0])) p += 1; + p = skipWs(p); + + // Check for AS + r.stage_names[idx][0] = 0; + if (hasPrefixCi(p, "AS") and isBlank(p[2])) { + p = skipWs(p + 3); + _ = extractWord(p, &r.stage_names[idx]); + } + r.stage_count += 1; +} + +fn dfParseExpose(line: Str, r: *DockerfileResult) void { + if (!hasPrefixCi(line, "EXPOSE")) return; + var p: Str = line + 6; + if (!isBlank(p[0])) return; + p = skipWs(p); + + // Parse space-separated ports + while (p[0] != 0 and r.port_count < IS_MAX_PORTS) { + var port: [c.CBM_SZ_32]u8 = undefined; + const n = extractToken(p, &port); + if (n == 0) break; + + // Strip protocol suffix (e.g. 8080/tcp) + const slash: MutStr = c.strchr(&port, '/'); + if (slash != null) slash[0] = 0; + + copyStr(&r.exposed_ports[ix(r.port_count)], &port); + r.port_count += 1; + + p += n; + p = skipWs(p); + } +} + +fn dfParseEnv(line: Str, r: *DockerfileResult) void { + if (!hasPrefixCi(line, "ENV")) return; + var p: Str = line + 3; + if (!isBlank(p[0])) return; + p = skipWs(p); + + if (r.env_count >= c.CBM_SZ_64) return; + + // Extract key + var key: [c.CBM_SZ_128:0]u8 = undefined; + const klen = extractWord(p, &key); + if (klen == 0) return; + p += klen; + + // Separator: = or space + if (p[0] != '=' and !isBlank(p[0])) return; + p += 1; + // Skip additional whitespace after separator + while (isBlank(p[0])) p += 1; + + // Value is rest of line (trimmed) + var value: [c.CBM_SZ_512:0]u8 = undefined; + copyStr(&value, p); + rtrim(&value); + + // Filter secrets + if (cbm_is_secret_binding(&key, &value)) return; + + const env = &r.env_vars[ix(r.env_count)]; + copyStr(&env.key, &key); + copyStr(&env.value, &value); + r.env_count += 1; +} + +fn dfParseArg(line: Str, r: *DockerfileResult) void { + if (!hasPrefixCi(line, "ARG")) return; + var p: Str = line + 3; + if (!isBlank(p[0])) return; + p = skipWs(p); + + if (r.build_arg_count >= c.CBM_SZ_32) return; + const arg = &r.build_args[ix(r.build_arg_count)]; + _ = extractWord(p, arg); + if (arg[0] != 0) r.build_arg_count += 1; +} + +/// Parse a Dockerfile CMD/ENTRYPOINT value: strip brackets, trim. +fn dfParseJsonCmd(line: Str, prefix_len: usize, out: []u8) void { + const p = skipWs(line + prefix_len); + var raw: [c.CBM_SZ_512:0]u8 = undefined; + copyStr(&raw, p); + rtrim(&raw); + cbm_clean_json_brackets(&raw, out.ptr, out.len); +} + +fn dfParseDirectives(line: Str, r: *DockerfileResult) void { + if (hasPrefixCi(line, "WORKDIR") and isBlank(line[7])) { + const p = skipWs(line + 8); + copyStr(&r.workdir, p); + rtrim(@ptrCast(&r.workdir)); + return; + } + if (hasPrefixCi(line, "CMD") and isBlank(line[3])) { + dfParseJsonCmd(line, 4, &r.cmd); + return; + } + if (hasPrefixCi(line, "ENTRYPOINT") and isBlank(line[10])) { + dfParseJsonCmd(line, 11, &r.entrypoint); + return; + } + if (hasPrefixCi(line, "USER") and isBlank(line[4])) { + const p = skipWs(line + 5); + _ = extractWord(p, &r.user); + return; + } + if (hasPrefixCi(line, "HEALTHCHECK") and isBlank(line[LEN_HEALTHCHECK])) { + if (ciStrstr(line + LEN_HEALTHCHECK, "CMD")) |cmd_pos| { + copyStr(&r.healthcheck, skipWs(cmd_pos + 3)); + rtrim(@ptrCast(&r.healthcheck)); + } + } +} + +/// Iterates `source` line by line, copying each line into a caller buffer +/// (truncated to fit) and returning it without leading blanks or trailing +/// whitespace. +const LineReader = struct { + p: [*:0]const u8, + line: [c.CBM_SZ_4K:0]u8 = undefined, + + fn next(self: *LineReader) ?[*:0]u8 { + if (self.p[0] == 0) return null; + const eol: CStr = c.strchr(self.p, '\n'); + var line_len: usize = if (eol != null) util.ptrDiff(eol, self.p) else c.strlen(self.p); + if (line_len >= c.CBM_SZ_4K) line_len = c.CBM_SZ_4K - 1; + @memcpy(self.line[0..line_len], self.p[0..line_len]); + self.line[line_len] = 0; + // Without a newline the C parsers advanced by the truncated length, + // so the tail of an over-long last line is read as a further line. + self.p = if (eol != null) @ptrCast(eol + 1) else self.p + line_len; + + var trimmed: [*:0]u8 = &self.line; + while (isBlank(trimmed[0])) trimmed += 1; + rtrim(trimmed); + return trimmed; + } +}; + +pub export fn cbm_parse_dockerfile_source(source: ?Str, out_c: ?*DockerfileResult) c_int { + const out = out_c orelse return c.CBM_NOT_FOUND; + var lines: LineReader = .{ .p = source orelse return c.CBM_NOT_FOUND }; + out.* = std.mem.zeroes(DockerfileResult); + + // Process line by line + while (lines.next()) |trimmed| { + // Skip empty lines and comments + if (trimmed[0] == 0 or trimmed[0] == '#') continue; + + dfParseFrom(trimmed, out); + dfParseExpose(trimmed, out); + dfParseEnv(trimmed, out); + dfParseArg(trimmed, out); + dfParseDirectives(trimmed, out); + } + + // No stages = empty/invalid Dockerfile + if (out.stage_count == 0) return c.CBM_NOT_FOUND; + + // base_image = last stage's image (use intermediate copy to avoid restrict overlap) + { + var tmp: [c.CBM_SZ_256]u8 = undefined; + _ = c.snprintf(&tmp, tmp.len, "%s", @as(CStr, &out.stage_images[ix(out.stage_count) - 1])); + out.base_image = tmp; + } + + return 0; +} + +// ── Dotenv parser ─────────────────────────────────────────────────── + +/// Strip matching surrounding quotes from a value string in-place. +fn stripSurroundingQuotes(value: [*:0]u8) void { + const vlen = c.strlen(value); + if (vlen >= 2 and ((value[0] == '"' and value[vlen - 1] == '"') or + (value[0] == '\'' and value[vlen - 1] == '\''))) + { + _ = c.memmove(value, value + 1, vlen - 2); + value[vlen - 2] = 0; + } +} + +/// Parse a single dotenv line: KEY=VALUE. Returns true if an env var was added. +fn parseDotenvLine(trimmed: Str, out: *DotenvResult) bool { + var key: [c.CBM_SZ_128:0]u8 = undefined; + var klen: usize = 0; + if (std.ascii.isAlphabetic(trimmed[0]) or trimmed[0] == '_') { + klen = extractWord(trimmed, &key); + } + if (klen == 0 or trimmed[klen] != '=') return false; + + var value: [c.CBM_SZ_512:0]u8 = undefined; + copyStr(&value, trimmed + klen + 1); + rtrim(&value); + stripSurroundingQuotes(&value); + + if (cbm_is_secret_binding(&key, &value) or out.env_count >= c.CBM_SZ_64) return false; + const env = &out.env_vars[ix(out.env_count)]; + copyStr(&env.key, &key); + copyStr(&env.value, &value); + out.env_count += 1; + return true; +} + +pub export fn cbm_parse_dotenv_source(source: ?Str, out_c: ?*DotenvResult) c_int { + const out = out_c orelse return c.CBM_NOT_FOUND; + var lines: LineReader = .{ .p = source orelse return c.CBM_NOT_FOUND }; + out.* = std.mem.zeroes(DotenvResult); + + while (lines.next()) |trimmed| { + if (trimmed[0] == 0 or trimmed[0] == '#') continue; + _ = parseDotenvLine(trimmed, out); + } + + return if (out.env_count > 0) 0 else IS_NOT_FOUND; +} + +// ── Shell script parser ───────────────────────────────────────────── + +/// Store VAR=VALUE from `val_start` unless it is a secret or the table is full. +fn shellAddVar(r: *ShellResult, key: *[c.CBM_SZ_128:0]u8, val_start: Str) void { + var value: [c.CBM_SZ_512:0]u8 = undefined; + copyStr(&value, val_start); + rtrim(&value); + + // Strip surrounding quotes + stripSurroundingQuotes(&value); + + if (cbm_is_secret_binding(key, &value)) return; + if (r.env_count >= c.CBM_SZ_64) return; + const env = &r.env_vars[ix(r.env_count)]; + copyStr(&env.key, key); + copyStr(&env.value, &value); + r.env_count += 1; +} + +fn shellParseExport(line: Str, r: *ShellResult) void { + // export VAR=VALUE + if (!hasPrefix(line, "export") or !isBlank(line[6])) return; + const p = skipWs(line + 7); + + var key: [c.CBM_SZ_128:0]u8 = undefined; + const klen = extractWord(p, &key); + if (klen == 0 or p[klen] != '=') return; + + shellAddVar(r, &key, p + klen + 1); +} + +fn shellParsePlainVar(line: Str, r: *ShellResult) void { + // VAR=VALUE (only if no spaces in line — avoids matching commands) + if (c.strchr(line, ' ') != null or c.strchr(line, '\t') != null) return; + + var key: [c.CBM_SZ_128:0]u8 = undefined; + const klen = extractWord(line, &key); + if (klen == 0 or line[klen] != '=') return; + + shellAddVar(r, &key, line + klen + 1); +} + +fn shellParseSource(line: Str, r: *ShellResult) void { + // source or . + var p: ?Str = null; + if (hasPrefix(line, "source") and isBlank(line[6])) { + p = skipWs(line + 7); + } else if (line[0] == '.' and isBlank(line[1])) { + p = skipWs(line + 2); + } + const file = p orelse return; + + if (r.source_count >= IS_MAX_SOURCES) return; + + // Strip surrounding quotes + var path: [c.CBM_SZ_256:0]u8 = undefined; + copyStr(&path, file); + rtrim(&path); + stripSurroundingQuotes(&path); + + // Extract just the path token (no trailing args) + const space: MutStr = c.strchr(&path, ' '); + if (space != null) space[0] = 0; + + copyStr(&r.sources[ix(r.source_count)], &path); + r.source_count += 1; +} + +fn shellParseDocker(line: Str, r: *ShellResult) void { + // docker or docker-compose + var p: ?Str = null; + var tool: CStr = null; + + if (c.strncmp(line, "docker-compose", LEN_DOCKER_COMPOSE) == 0 and isBlank(line[LEN_DOCKER_COMPOSE])) { + tool = "docker-compose"; + p = skipWs(line + LEN_DOCKER_COMPOSE + 1); + } else if (hasPrefix(line, "docker") and isBlank(line[6])) { + tool = "docker"; + p = skipWs(line + 7); + } + const rest = p orelse return; + if (tool == null or rest[0] == 0) return; + + if (r.docker_cmd_count >= IS_MAX_STAGES) return; + + var subcmd: [c.CBM_SZ_64]u8 = undefined; + _ = extractWord(rest, &subcmd); + if (subcmd[0] != 0) { + const slot = &r.docker_cmds[ix(r.docker_cmd_count)]; + _ = c.snprintf(slot, slot.len, "%s %s", tool, @as(CStr, &subcmd)); + r.docker_cmd_count += 1; + } +} + +/// Check for shebang on first non-empty line. Returns true if line was consumed. +fn shellTryShebang(trimmed: Str, out: *ShellResult, shebang_checked: *bool) bool { + if (shebang_checked.*) return false; + if (trimmed[0] == '#' and trimmed[1] == '!') { + var sb: Str = trimmed + 2; + while (sb[0] == ' ') sb += 1; + copyStr(&out.shebang, sb); + rtrim(@ptrCast(&out.shebang)); + shebang_checked.* = true; + return true; + } + shebang_checked.* = true; + return false; +} + +/// Dispatch a single trimmed shell line to the appropriate parser. +fn shellDispatchLine(trimmed: Str, out: *ShellResult) void { + if (hasPrefix(trimmed, "export")) { + shellParseExport(trimmed, out); + return; + } + if (hasPrefix(trimmed, "source") or (trimmed[0] == '.' and trimmed[1] == ' ')) { + shellParseSource(trimmed, out); + return; + } + if (hasPrefix(trimmed, "docker")) { + shellParseDocker(trimmed, out); + } + shellParsePlainVar(trimmed, out); +} + +pub export fn cbm_parse_shell_source(source: ?Str, out_c: ?*ShellResult) c_int { + const out = out_c orelse return c.CBM_NOT_FOUND; + var lines: LineReader = .{ .p = source orelse return c.CBM_NOT_FOUND }; + out.* = std.mem.zeroes(ShellResult); + + var shebang_checked = false; + + while (lines.next()) |trimmed| { + if (trimmed[0] == 0) continue; + if (shellTryShebang(trimmed, out, &shebang_checked)) continue; + if (trimmed[0] == '#') continue; + shellDispatchLine(trimmed, out); + } + + const has_content = out.shebang[0] != 0 or out.env_count > 0 or out.source_count > 0 or + out.docker_cmd_count > 0; + return if (has_content) 0 else IS_NOT_FOUND; +} + +// ── Terraform parser ──────────────────────────────────────────────── + +const TfBlockKind = enum { none, variable, module, terraform }; + +/// Extract a double-quoted string value after = sign. +/// Handles: key = "value" and key = value +fn tfExtractQuoted(line: Str, prefix: []const u8, out: []u8) void { + var p = skipWs(line); + if (c.strncmp(p, prefix.ptr, prefix.len) != 0) return; + p = skipWs(p + prefix.len); + if (p[0] != '=') return; + p = skipWs(p + 1); + + // Strip optional quotes + if (p[0] == '"') { + p += 1; + const end: CStr = c.strchr(p, '"'); + var vlen: usize = if (end != null) util.ptrDiff(end, p) else c.strlen(p); + if (vlen >= out.len) vlen = out.len - 1; + @memcpy(out[0..vlen], p[0..vlen]); + out[vlen] = 0; + } else { + _ = extractToken(p, out); + } +} + +/// Count occurrences of a character in a string +fn countChar(s: Str, ch: u8) c_int { + var n: c_int = 0; + var i: usize = 0; + while (s[i] != 0) : (i += 1) { + if (s[i] == ch) n += 1; + } + return n; +} + +/// Extract double-quoted identifier after keyword. +/// Matches: keyword "identifier" → sets out to "identifier" +/// Returns number of chars consumed from p, or 0 if no match. +fn tfExtractIdent(p: Str, keyword: []const u8, out: []u8) usize { + if (c.strncmp(p, keyword.ptr, keyword.len) != 0) return 0; + var after = skipWs(p + keyword.len); + if (after[0] != '"') return 0; + after += 1; + const end: CStr = c.strchr(after, '"'); + if (end == null) return 0; + var ilen = util.ptrDiff(end, after); + if (ilen >= out.len) ilen = out.len - 1; + @memcpy(out[0..ilen], after[0..ilen]); + out[ilen] = 0; + return util.ptrDiff(end, p) + 1; +} + +/// Parse a second quoted string after the first ident (for resource/data blocks). +fn tfExtractSecondQuoted(trimmed: Str, consumed: usize, out: []u8) void { + var rest = skipWs(trimmed + consumed); + out[0] = 0; + if (rest[0] == '"') { + rest += 1; + const end: CStr = c.strchr(rest, '"'); + if (end != null) { + var len = util.ptrDiff(end, rest); + if (len >= out.len) len = out.len - 1; + @memcpy(out[0..len], rest[0..len]); + out[len] = 0; + } + } +} + +/// Try to parse a simple "keyword ident" block (variable, output, provider). +/// `names` is the table of fixed-size name slots the identifier is added to. +fn tfTrySimpleBlock(trimmed: Str, keyword: []const u8, names: anytype, count: *c_int) bool { + var ident: [c.CBM_SZ_128]u8 = undefined; + const consumed = tfExtractIdent(trimmed, keyword, &ident); + if (consumed == 0 or count.* >= names.len) return false; + copyStr(&names[ix(count.*)], &ident); + count.* += 1; + return true; +} + +/// Try to match terraform/locals keyword blocks. +fn tfTryKeywordBlock(trimmed: Str, cur_block: *TfBlockKind, out: *TerraformResult) bool { + if (hasPrefix(trimmed, "terraform") and c.strstr(trimmed, "{") != null) { + cur_block.* = .terraform; + return true; + } + if (hasPrefix(trimmed, "locals") and c.strstr(trimmed, "{") != null) { + out.has_locals = true; + return true; + } + return false; +} + +/// Detect a top-level block header. Returns true if line was consumed as a header. +fn tfDetectBlockHeader( + trimmed: Str, + out: *TerraformResult, + cur_block: *TfBlockKind, + cur_var_idx: *c_int, + cur_mod_idx: *c_int, +) bool { + var ident1: [c.CBM_SZ_128]u8 = undefined; + + // resource "type" "name" + { + const rc = tfExtractIdent(trimmed, "resource", &ident1); + if (rc > 0 and out.resource_count < c.CBM_SZ_32) { + var ident2: [c.CBM_SZ_128]u8 = undefined; + tfExtractSecondQuoted(trimmed, rc, &ident2); + const resource = &out.resources[ix(out.resource_count)]; + copyStr(&resource.type, &ident1); + copyStr(&resource.name, &ident2); + out.resource_count += 1; + return true; + } + } + + // variable "name" + var consumed = tfExtractIdent(trimmed, "variable", &ident1); + if (consumed > 0 and out.variable_count < c.CBM_SZ_32) { + cur_block.* = .variable; + cur_var_idx.* = out.variable_count; + copyStr(&out.variables[ix(cur_var_idx.*)].name, &ident1); + out.variable_count += 1; + return true; + } + + // output "name" + if (tfTrySimpleBlock(trimmed, "output", &out.outputs, &out.output_count)) return true; + + // provider "name" + if (tfTrySimpleBlock(trimmed, "provider", &out.providers, &out.provider_count)) return true; + + // module "name" + consumed = tfExtractIdent(trimmed, "module", &ident1); + if (consumed > 0 and out.module_count < IS_MAX_STAGES) { + cur_block.* = .module; + cur_mod_idx.* = out.module_count; + copyStr(&out.modules[ix(cur_mod_idx.*)].tf_name, &ident1); + out.module_count += 1; + return true; + } + + // data "type" "name" + { + const dc = tfExtractIdent(trimmed, "data", &ident1); + if (dc > 0 and out.data_source_count < IS_MAX_STAGES) { + var ident2: [c.CBM_SZ_128]u8 = undefined; + tfExtractSecondQuoted(trimmed, dc, &ident2); + const data_source = &out.data_sources[ix(out.data_source_count)]; + copyStr(&data_source.type, &ident1); + copyStr(&data_source.name, &ident2); + out.data_source_count += 1; + return true; + } + } + + return tfTryKeywordBlock(trimmed, cur_block, out); +} + +/// Extract attributes from a line inside a variable block. +fn tfParseVariableAttrs(trimmed: Str, v: *c.cbm_tf_variable_t) void { + var def_val: [c.CBM_SZ_256:0]u8 = @splat(0); + tfExtractQuoted(trimmed, "default", &def_val); + if (def_val[0] != 0 and !cbm_is_secret_binding(@ptrCast(&v.name), &def_val)) { + copyStr(&v.default_val, &def_val); + } + var type_val: [c.CBM_SZ_64]u8 = @splat(0); + tfExtractQuoted(trimmed, "type", &type_val); + if (type_val[0] != 0) copyStr(&v.type, &type_val); + var desc_val: [c.CBM_SZ_256]u8 = @splat(0); + tfExtractQuoted(trimmed, "description", &desc_val); + if (desc_val[0] != 0) copyStr(&v.description, &desc_val); +} + +/// Extract attributes inside a block (variable/module/terraform). +fn tfParseBlockAttrs( + trimmed: Str, + out: *TerraformResult, + cur_block: TfBlockKind, + cur_var_idx: c_int, + cur_mod_idx: c_int, +) void { + switch (cur_block) { + .variable => { + if (cur_var_idx >= 0) tfParseVariableAttrs(trimmed, &out.variables[ix(cur_var_idx)]); + }, + .module => { + if (cur_mod_idx >= 0) { + var src_val: [c.CBM_SZ_256]u8 = @splat(0); + tfExtractQuoted(trimmed, "source", &src_val); + if (src_val[0] != 0) copyStr(&out.modules[ix(cur_mod_idx)].source, &src_val); + } + }, + .terraform => { + const bp = skipWs(trimmed); + var backend_name: [c.CBM_SZ_128]u8 = @splat(0); + _ = tfExtractIdent(bp, "backend", &backend_name); + if (backend_name[0] != 0) copyStr(&out.backend, &backend_name); + }, + .none => {}, + } +} + +/// Check if any terraform content was parsed. +fn tfResultHasContent(out: *const TerraformResult) bool { + return out.resource_count > 0 or out.variable_count > 0 or out.output_count > 0 or + out.provider_count > 0 or out.module_count > 0 or out.data_source_count > 0 or + out.backend[0] != 0 or out.has_locals; +} + +pub export fn cbm_parse_terraform_source(source: ?Str, out_c: ?*TerraformResult) c_int { + const out = out_c orelse return c.CBM_NOT_FOUND; + var lines: LineReader = .{ .p = source orelse return c.CBM_NOT_FOUND }; + out.* = std.mem.zeroes(TerraformResult); + + var brace_depth: c_int = 0; + var cur_block: TfBlockKind = .none; + var cur_var_idx: c_int = IS_NOT_FOUND; + var cur_mod_idx: c_int = IS_NOT_FOUND; + + while (lines.next()) |trimmed| { + // Skip empty lines and # or // comments. + if (trimmed[0] == 0 or trimmed[0] == '#' or (trimmed[0] == '/' and trimmed[1] == '/')) { + continue; + } + + brace_depth += countChar(trimmed, '{') - countChar(trimmed, '}'); + + if (brace_depth <= 1 and cur_block == .none) { + if (tfDetectBlockHeader(trimmed, out, &cur_block, &cur_var_idx, &cur_mod_idx)) continue; + } + + if (cur_block != .none and brace_depth >= 1 and brace_depth <= 2) { + tfParseBlockAttrs(trimmed, out, cur_block, cur_var_idx, cur_mod_idx); + } + + if (brace_depth == 0 and cur_block != .none) { + cur_block = .none; + cur_var_idx = IS_NOT_FOUND; + cur_mod_idx = IS_NOT_FOUND; + } + } + + return if (tfResultHasContent(out)) 0 else IS_NOT_FOUND; +} + +// ── Infra QN helper ───────────────────────────────────────────────── + +pub export fn cbm_infra_qn(project_name: CStr, rel_path: CStr, infra_type: CStr, service_name: CStr) MutStr { + const base: MutStr = c.cbm_pipeline_fqn_compute(project_name, rel_path, ""); + if (base == null) return null; + + var result: [c.CBM_SZ_1K]u8 = undefined; + if (service_name != null and service_name[0] != 0 and infra_type != null and + streq(infra_type, "compose-service")) + { + _ = c.snprintf(&result, result.len, "%s::%s", base, service_name); + } else { + _ = c.snprintf(&result, result.len, "%s.__infra__", base); + } + + c.free(base); + return c.strdup(&result); +} + +// ── Helm Chart.yaml parser (#338) ─────────────────────────────────── + +/// Copy the scalar value following a "key:" up to end-of-line into out, +/// trimming surrounding whitespace, optional quotes, and trailing comments. +fn helmCopyScalar(s_in: [*]const u8, eol: [*]const u8, out: []u8) void { + var s = s_in; + while (@intFromPtr(s) < @intFromPtr(eol) and isBlank(s[0])) s += 1; + var quote: u8 = 0; + if (@intFromPtr(s) < @intFromPtr(eol) and (s[0] == '"' or s[0] == '\'')) { + quote = s[0]; + s += 1; + } + var e = s; + while (@intFromPtr(e) < @intFromPtr(eol)) : (e += 1) { + if (quote != 0) { + if (e[0] == quote) break; + } else if (e[0] == ' ' or e[0] == '\t' or e[0] == '#' or e[0] == '\r') { + break; + } + } + var n = util.ptrDiff(e, s); + if (n >= out.len) n = out.len - 1; + @memcpy(out[0..n], s[0..n]); + out[n] = 0; +} + +/// Match "key:" at the start of a content pointer; return value start or NULL. +fn helmMatchKey(content: [*]const u8, eol: [*]const u8, key: []const u8) ?[*]const u8 { + if (util.ptrDiff(eol, content) < key.len + 1) return null; + if (c.strncmp(content, key.ptr, key.len) != 0 or content[key.len] != ':') return null; + return content + key.len + 1; +} + +pub export fn cbm_parse_helm_chart(source: ?Str, out_c: ?*c.cbm_helm_chart_t) c_int { + var p: [*]const u8 = source orelse return -1; + const out = out_c orelse return -1; + out.chart_name[0] = 0; + out.dep_count = 0; + + var in_deps = false; + while (p[0] != 0) { + var eol: [*]const u8 = undefined; + const newline: CStr = c.strchr(p, '\n'); + if (newline != null) { + eol = newline; + } else { + eol = p + c.strlen(p); + } + var indent: c_int = 0; + var content = p; + while (@intFromPtr(content) < @intFromPtr(eol) and content[0] == ' ') { + content += 1; + indent += 1; + } + if (@intFromPtr(content) < @intFromPtr(eol) and content[0] != '#') { + if (indent == 0) { + // New top-level key ends any dependencies block. + in_deps = helmMatchKey(content, eol, "dependencies") != null; + const nv = helmMatchKey(content, eol, "name"); + if (!in_deps and nv != null and out.chart_name[0] == 0) { + helmCopyScalar(nv.?, eol, &out.chart_name); + } + } else if (in_deps) { + // List item: optional leading "- ", then look for name:. + var q = content; + if (q[0] == '-') { + q += 1; + while (@intFromPtr(q) < @intFromPtr(eol) and q[0] == ' ') q += 1; + } + const nv = helmMatchKey(q, eol, "name"); + if (nv != null and out.dep_count < c.CBM_HELM_MAX_DEPS) { + var buf: [c.CBM_HELM_NAME_MAX]u8 = undefined; + helmCopyScalar(nv.?, eol, &buf); + if (buf[0] != 0) { + out.deps[ix(out.dep_count)] = buf; + out.dep_count += 1; + } + } + } + } + if (eol[0] == 0) break; + p = eol + 1; + } + return if (out.chart_name[0] != 0 or out.dep_count > 0) 0 else -1; +} diff --git a/src/pipeline/pass_k8s.c b/src/pipeline/pass_k8s.c deleted file mode 100644 index 39e0afd1..00000000 --- a/src/pipeline/pass_k8s.c +++ /dev/null @@ -1,666 +0,0 @@ -/* - * pass_k8s.c — Pipeline pass for Kubernetes manifest and Kustomize overlay processing. - * - * For each discovered YAML file: - * 1. Check if it is a kustomize overlay (kustomization.yaml / kustomization.yml) - * → emit a Module node and IMPORTS edges for each resources/bases/patches entry - * 2. Else if it is a generic k8s manifest (apiVersion: detected) - * → emit one Resource node per file (first document only — multi-document YAML is not yet - * supported) - * - * Depends on: pass_infrascan.c (cbm_is_kustomize_file, cbm_is_k8s_manifest, cbm_infra_qn), - * extraction layer (cbm.h), graph_buffer, pipeline internals. - */ -#include "foundation/constants.h" -#include "pipeline/pipeline.h" -#include -#include "pipeline/pipeline_internal.h" -#include "graph_buffer/graph_buffer.h" -#include "discover/discover.h" -#include "foundation/log.h" -#include "foundation/compat.h" -#include "foundation/compat_fs.h" -#include "foundation/limits.h" -#include "foundation/str_util.h" -#include "cbm.h" - -#include -#include -#include - -/* ── Internal helpers ────────────────────────────────────────────── */ - -/* Read entire file into heap-allocated buffer. Returns NULL on error. - * Caller must free(). Sets *out_len to byte count. */ -static char *k8s_read_file(const char *path, int *out_len) { - FILE *f = cbm_fopen(path, "rb"); - if (!f) { - return NULL; - } - - (void)fseek(f, 0, SEEK_END); - long size = ftell(f); - (void)fseek(f, 0, SEEK_SET); - - if (size <= 0 || size > cbm_max_file_bytes()) { /* generous, env-configurable cap (B4) */ - (void)fclose(f); - return NULL; - } - - /* +pad: tree-sitter lexer lookahead reads past EOF; keep it in-bounds */ - enum { CBM_TS_LOOKAHEAD_PAD = 16 }; - char *buf = malloc((size_t)size + CBM_TS_LOOKAHEAD_PAD); - if (!buf) { - (void)fclose(f); - return NULL; - } - - size_t nread = fread(buf, SKIP_ONE, size, f); - (void)fclose(f); - if (nread > (size_t)size) { - nread = (size_t)size; - } - memset(buf + nread, 0, CBM_TS_LOOKAHEAD_PAD); - *out_len = (int)nread; - return buf; -} - -/* ── Kustomize handler ───────────────────────────────────────────── */ - -static void handle_kustomize(cbm_pipeline_ctx_t *ctx, const char *path, const char *rel_path) { - /* Emit Module node for this kustomize overlay file */ - char *mod_qn = cbm_infra_qn(ctx->project_name, rel_path, "kustomize", NULL); - if (!mod_qn) { - return; - } - - int64_t mod_id = cbm_gbuf_upsert_node(ctx->gbuf, "Module", cbm_path_base(rel_path), mod_qn, - rel_path, SKIP_ONE, 0, "{\"source\":\"kustomize\"}"); - free(mod_qn); - - if (mod_id <= 0) { - return; - } - - /* Emit IMPORTS edges for resources/bases/patches/components entries */ - int import_count = 0; - CBMFileResult *res = NULL; - int src_len = 0; - char *source = k8s_read_file(path, &src_len); - if (source) { - res = cbm_extract_file(source, src_len, CBM_LANG_KUSTOMIZE, ctx->project_name, rel_path, - CBM_EXTRACT_BUDGET, NULL, NULL); - free(source); - } - - if (res) { - for (int j = 0; j < res->imports.count; j++) { - CBMImport *imp = &res->imports.items[j]; - if (!imp->module_path) { - continue; - } - - /* Compute target file QN */ - char *target_qn = - cbm_pipeline_fqn_compute(ctx->project_name, imp->module_path, "__file__"); - if (!target_qn) { - continue; - } - - const cbm_gbuf_node_t *target = cbm_gbuf_find_by_qn(ctx->gbuf, target_qn); - free(target_qn); - - if (target) { - cbm_gbuf_insert_edge(ctx->gbuf, mod_id, target->id, "IMPORTS", - "{\"via\":\"kustomize\"}"); - import_count++; - } - } - cbm_free_result(res); - } - - cbm_log_info("pass.k8s.kustomize", "file", rel_path, "imports", cbm_log_itoa(import_count)); -} - -/* ── K8s cross-manifest label-selector matching ────────────────────── - * A Service routes traffic to the workload Pods whose labels match the - * Service's spec.selector. We record, per manifest, the Resource node id plus - * its selector label-values (Services) and pod label-values (workloads), then - * after all manifests are processed connect each Service to the workload(s) it - * targets via an INFRA_MAPS edge. This models the runtime traffic path the same - * way k8s itself resolves Service → Endpoints by label. - * - * Matching is by label *value*: a Service selector `app: frontend` matches a - * workload that either carries the pod label value "frontend" OR is named - * "frontend" (the ubiquitous app==name convention, used when the pod template - * omits explicit labels). */ -enum { K8S_MAX_LABELS = 16, K8S_MAX_RECORDS = 512, K8S_LABEL_LEN = 128 }; - -typedef struct { - int64_t node_id; - char name[K8S_LABEL_LEN]; - bool is_service; - bool is_workload; - char selector_vals[K8S_MAX_LABELS][K8S_LABEL_LEN]; /* Service spec.selector values */ - int n_selector; - char label_vals[K8S_MAX_LABELS][K8S_LABEL_LEN]; /* workload pod-template label values */ - int n_label; -} k8s_record_t; - -typedef struct { - k8s_record_t *items; - int count; - int cap; -} k8s_record_array_t; - -/* Count leading-space indentation of a line (tabs are invalid YAML indent). */ -static int k8s_indent(const char *line) { - int n = 0; - while (line[n] == ' ') { - n++; - } - return n; -} - -/* Split `key: value` (already de-indented). Returns 1 if a key was found; fills - * key/val (val empty when the key opens a nested block). */ -static int k8s_split_kv(const char *t, char *key, size_t key_sz, char *val, size_t val_sz) { - key[0] = '\0'; - val[0] = '\0'; - if (t[0] == '#' || t[0] == '-' || t[0] == '\0') { - return 0; - } - const char *colon = strchr(t, ':'); - if (!colon) { - return 0; - } - size_t klen = (size_t)(colon - t); - if (klen == 0 || klen >= key_sz) { - return 0; - } - memcpy(key, t, klen); - key[klen] = '\0'; - const char *v = colon + 1; - while (*v == ' ' || *v == '\t') { - v++; - } - size_t vn = 0; - while (v[vn] && v[vn] != '\r' && v[vn] != '\n' && v[vn] != '#' && vn + 1 < val_sz) { - val[vn] = v[vn]; - vn++; - } - /* trim trailing space */ - while (vn > 0 && (val[vn - 1] == ' ' || val[vn - 1] == '\t')) { - vn--; - } - val[vn] = '\0'; - /* strip surrounding quotes */ - if (vn >= 2 && (val[0] == '"' || val[0] == '\'') && val[vn - 1] == val[0]) { - memmove(val, val + 1, vn - 2); - val[vn - 2] = '\0'; - } - return 1; -} - -static void k8s_add_val(char dst[][K8S_LABEL_LEN], int *n, const char *v) { - if (*n >= K8S_MAX_LABELS || !v || !v[0]) { - return; - } - snprintf(dst[*n], K8S_LABEL_LEN, "%s", v); - (*n)++; -} - -/* Scan a single-document k8s manifest's text for the resource name, selector - * label-values (Service) and pod-template label-values (workload). A small - * indentation path-stack distinguishes spec.selector from - * spec.template.metadata.labels and from top-level metadata.name. */ -static void k8s_scan_labels(const char *source, k8s_record_t *rec) { - enum { K8S_PATH_DEPTH = 12 }; - struct { - int indent; - char key[64]; - } stack[K8S_PATH_DEPTH]; - int depth = 0; - bool got_name = false; - - const char *p = source; - while (p && *p) { - const char *eol = strchr(p, '\n'); - size_t len = eol ? (size_t)(eol - p) : strlen(p); - char line[CBM_SZ_512]; - size_t cp = len < sizeof(line) - 1 ? len : sizeof(line) - 1; - memcpy(line, p, cp); - line[cp] = '\0'; - - /* End of first YAML document — stop (one Resource per file). */ - const char *trimmed = line; - while (*trimmed == ' ') { - trimmed++; - } - if (strncmp(trimmed, "---", 3) == 0 && line == trimmed) { - break; - } - - if (trimmed[0] && trimmed[0] != '#') { - int ind = k8s_indent(line); - while (depth > 0 && stack[depth - 1].indent >= ind) { - depth--; - } - char key[64]; - char val[K8S_LABEL_LEN]; - if (k8s_split_kv(trimmed, key, sizeof(key), val, sizeof(val))) { - /* Build the current dotted path for context decisions. */ - bool under_selector = (depth >= 1 && strcmp(stack[depth - 1].key, "selector") == 0); - bool under_labels = (depth >= 1 && strcmp(stack[depth - 1].key, "labels") == 0); - bool under_metadata = (depth >= 1 && strcmp(stack[depth - 1].key, "metadata") == 0); - - if (val[0] == '\0') { - /* Block-opening key: push onto the path stack. */ - if (depth < K8S_PATH_DEPTH) { - stack[depth].indent = ind; - snprintf(stack[depth].key, sizeof(stack[depth].key), "%s", key); - depth++; - } - } else { - /* Leaf key: value. */ - if (ind == 0 && strcmp(key, "kind") == 0) { - rec->is_service = (strcmp(val, "Service") == 0); - rec->is_workload = - (strcmp(val, "Deployment") == 0 || strcmp(val, "StatefulSet") == 0 || - strcmp(val, "DaemonSet") == 0 || strcmp(val, "ReplicaSet") == 0 || - strcmp(val, "ReplicationController") == 0 || strcmp(val, "Pod") == 0 || - strcmp(val, "Job") == 0 || strcmp(val, "CronJob") == 0); - } else if (!got_name && under_metadata && strcmp(key, "name") == 0) { - snprintf(rec->name, sizeof(rec->name), "%s", val); - got_name = true; - } else if (under_selector) { - /* Service spec.selector matchLabels values. */ - k8s_add_val(rec->selector_vals, &rec->n_selector, val); - } else if (under_labels) { - /* Pod-template / metadata labels values. */ - k8s_add_val(rec->label_vals, &rec->n_label, val); - } - } - } - } - - if (!eol) { - break; - } - p = eol + 1; - } -} - -/* True if any of the service's selector values matches the workload. */ -static bool k8s_selector_matches(const k8s_record_t *svc, const k8s_record_t *wl) { - for (int s = 0; s < svc->n_selector; s++) { - const char *sv = svc->selector_vals[s]; - if (!sv[0]) { - continue; - } - if (wl->name[0] && strcmp(sv, wl->name) == 0) { - return true; - } - for (int l = 0; l < wl->n_label; l++) { - if (strcmp(sv, wl->label_vals[l]) == 0) { - return true; - } - } - } - return false; -} - -/* After all manifests are recorded, connect each Service to the workload(s) its - * selector targets via an INFRA_MAPS edge (Service Resource → workload Resource). */ -static void k8s_link_selectors(cbm_pipeline_ctx_t *ctx, const k8s_record_array_t *recs) { - int edges = 0; - for (int i = 0; i < recs->count; i++) { - const k8s_record_t *svc = &recs->items[i]; - if (!svc->is_service || svc->n_selector == 0 || svc->node_id <= 0) { - continue; - } - for (int j = 0; j < recs->count; j++) { - const k8s_record_t *wl = &recs->items[j]; - if (i == j || !wl->is_workload || wl->node_id <= 0) { - continue; - } - if (k8s_selector_matches(svc, wl)) { - char props[CBM_SZ_256]; - snprintf(props, sizeof(props), - "{\"kind\":\"selector\",\"service\":\"%s\",\"workload\":\"%s\"}", - svc->name, wl->name); - cbm_gbuf_insert_edge(ctx->gbuf, svc->node_id, wl->node_id, "INFRA_MAPS", props); - edges++; - } - } - } - if (edges > 0) { - cbm_log_info("pass.k8s.selectors", "linked", cbm_log_itoa(edges)); - } -} - -/* ── K8s manifest handler ────────────────────────────────────────── */ - -/* source/src_len are the already-read file bytes (caller retains ownership and - * must free after this call returns). When `rec` is non-NULL it is populated - * with the first Resource's node id, name and label/selector values for later - * cross-manifest selector matching. */ -static void handle_k8s_manifest(cbm_pipeline_ctx_t *ctx, const char *path, const char *rel_path, - const char *source, int src_len, k8s_record_t *rec) { - (void)path; /* retained for symmetry; source is always provided now */ - int resource_count = 0; - - CBMFileResult *res = cbm_extract_file(source, src_len, CBM_LANG_K8S, ctx->project_name, - rel_path, CBM_EXTRACT_BUDGET, NULL, NULL); - if (!res) { - return; - } - - /* Compute file node QN for DEFINES edges */ - char *file_qn = cbm_pipeline_fqn_compute(ctx->project_name, rel_path, "__file__"); - const cbm_gbuf_node_t *file_node = file_qn ? cbm_gbuf_find_by_qn(ctx->gbuf, file_qn) : NULL; - free(file_qn); - - for (int d = 0; d < res->defs.count; d++) { - CBMDefinition *def = &res->defs.items[d]; - if (!def->label || strcmp(def->label, "Resource") != 0) { - continue; - } - if (!def->name || !def->qualified_name) { - continue; - } - - int64_t node_id = - cbm_gbuf_upsert_node(ctx->gbuf, "Resource", def->name, def->qualified_name, rel_path, - (int)def->start_line, (int)def->end_line, "{\"source\":\"k8s\"}"); - - /* DEFINES edge: File → Resource */ - if (file_node && node_id > 0) { - cbm_gbuf_insert_edge(ctx->gbuf, file_node->id, node_id, "DEFINES", "{}"); - } - - /* Capture the first Resource for cross-manifest selector matching. */ - if (rec && rec->node_id <= 0 && node_id > 0) { - rec->node_id = node_id; - } - - resource_count++; - } - - cbm_free_result(res); - - /* Record selector / pod-label values for later Service → workload linking. */ - if (rec && rec->node_id > 0) { - k8s_scan_labels(source, rec); - } - - cbm_log_info("pass.k8s.manifest", "file", rel_path, "resources", cbm_log_itoa(resource_count)); -} - -/* ── Helm chart handler ──────────────────────────────────────────── */ - -static bool is_helm_chart_file(const char *base) { - return strcmp(base, "Chart.yaml") == 0 || strcmp(base, "Chart.yml") == 0; -} - -/* Emit a Chart node for a Chart.yaml and a DEPENDS_ON edge to a (shared, - * deduplicated) Chart node per declared dependency (#338). */ -static void handle_helm_chart(cbm_pipeline_ctx_t *ctx, const char *rel_path, const char *source) { - cbm_helm_chart_t hc; - if (cbm_parse_helm_chart(source, &hc) != 0) { - return; - } - - const char *cname = hc.chart_name[0] ? hc.chart_name : cbm_path_base(rel_path); - char *chart_qn = cbm_infra_qn(ctx->project_name, rel_path, "helm-chart", NULL); - if (!chart_qn) { - return; - } - int64_t chart_id = cbm_gbuf_upsert_node(ctx->gbuf, "Chart", cname, chart_qn, rel_path, SKIP_ONE, - 0, "{\"source\":\"helm\"}"); - free(chart_qn); - - char *file_qn = cbm_pipeline_fqn_compute(ctx->project_name, rel_path, "__file__"); - const cbm_gbuf_node_t *file_node = file_qn ? cbm_gbuf_find_by_qn(ctx->gbuf, file_qn) : NULL; - if (file_node && chart_id > 0) { - cbm_gbuf_insert_edge(ctx->gbuf, file_node->id, chart_id, "DEFINES", "{}"); - } - free(file_qn); - - int dep_edges = 0; - for (int i = 0; i < hc.dep_count && chart_id > 0; i++) { - /* Stable per-project QN so multiple charts depending on the same chart - * link to one shared dependency node. */ - char dep_qn[CBM_SZ_512]; - snprintf(dep_qn, sizeof(dep_qn), "%s.__helm_dep__.%s", ctx->project_name, hc.deps[i]); - int64_t dep_id = - cbm_gbuf_upsert_node(ctx->gbuf, "Chart", hc.deps[i], dep_qn, rel_path, SKIP_ONE, 0, - "{\"source\":\"helm\",\"external\":true}"); - if (dep_id > 0) { - cbm_gbuf_insert_edge(ctx->gbuf, chart_id, dep_id, "DEPENDS_ON", "{}"); - dep_edges++; - } - } - cbm_log_info("pass.k8s.helm", "file", rel_path, "deps", cbm_log_itoa(dep_edges)); -} - -/* ── Dependency-manifest handler (go.mod / requirements.txt) ──────── */ - -static bool is_gomod_file(const char *base) { - return strcmp(base, "go.mod") == 0; -} - -static bool is_requirements_file(const char *base) { - return strcmp(base, "requirements.txt") == 0; -} - -/* Emit a DEPENDS_ON edge from the manifest file node to a (shared, per-project) - * external Package node. Mirrors the Helm Chart.yaml DEPENDS_ON shape. */ -static int emit_dep_edge(cbm_pipeline_ctx_t *ctx, const cbm_gbuf_node_t *src, const char *rel_path, - const char *ecosystem, const char *name) { - if (!name || !name[0]) { - return 0; - } - char dep_qn[CBM_SZ_512]; - snprintf(dep_qn, sizeof(dep_qn), "%s.__%s_dep__.%s", ctx->project_name, ecosystem, name); - char dep_props[CBM_SZ_256]; - snprintf(dep_props, sizeof(dep_props), "{\"source\":\"%s\",\"external\":true}", ecosystem); - int64_t dep_id = - cbm_gbuf_upsert_node(ctx->gbuf, "Package", name, dep_qn, rel_path, SKIP_ONE, 0, dep_props); - if (dep_id > 0 && dep_id != src->id) { - cbm_gbuf_insert_edge(ctx->gbuf, src->id, dep_id, "DEPENDS_ON", "{}"); - return 1; - } - return 0; -} - -/* Copy the first whitespace-delimited token of `line` into `out`. */ -static void first_token(const char *line, char *out, size_t out_sz) { - out[0] = '\0'; - while (*line == ' ' || *line == '\t') { - line++; - } - size_t n = 0; - while (line[n] && line[n] != ' ' && line[n] != '\t' && line[n] != '\r' && line[n] != '\n' && - n + 1 < out_sz) { - out[n] = line[n]; - n++; - } - out[n] = '\0'; -} - -/* Parse go.mod `require` directives (single-line and block forms) and emit a - * DEPENDS_ON edge per dependency. go.mod requires are not surfaced as imports - * by the extraction layer, so we parse the manifest text directly here. */ -static int parse_gomod_deps(cbm_pipeline_ctx_t *ctx, const cbm_gbuf_node_t *src, - const char *rel_path, const char *source) { - int edges = 0; - bool in_block = false; - const char *p = source; - while (p && *p) { - const char *eol = strchr(p, '\n'); - size_t len = eol ? (size_t)(eol - p) : strlen(p); - char line[CBM_SZ_512]; - size_t cp = len < sizeof(line) - 1 ? len : sizeof(line) - 1; - memcpy(line, p, cp); - line[cp] = '\0'; - const char *t = line; - while (*t == ' ' || *t == '\t') { - t++; - } - if (in_block) { - if (t[0] == ')') { - in_block = false; - } else if (t[0] && t[0] != '/') { - char name[CBM_SZ_256]; - first_token(t, name, sizeof(name)); - edges += emit_dep_edge(ctx, src, rel_path, "gomod", name); - } - } else if (strncmp(t, "require", 7) == 0 && (t[7] == ' ' || t[7] == '\t' || t[7] == '(')) { - const char *rest = t + 7; - while (*rest == ' ' || *rest == '\t') { - rest++; - } - if (*rest == '(') { - in_block = true; - } else if (*rest) { - char name[CBM_SZ_256]; - first_token(rest, name, sizeof(name)); - edges += emit_dep_edge(ctx, src, rel_path, "gomod", name); - } - } - if (!eol) { - break; - } - p = eol + 1; - } - return edges; -} - -/* Parse requirements.txt entries (one package spec per line) and emit a - * DEPENDS_ON edge per dependency. The package name is the leading token up to - * the first version/extras/comment delimiter. */ -static int parse_requirements_deps(cbm_pipeline_ctx_t *ctx, const cbm_gbuf_node_t *src, - const char *rel_path, const char *source) { - int edges = 0; - const char *p = source; - while (p && *p) { - const char *eol = strchr(p, '\n'); - size_t len = eol ? (size_t)(eol - p) : strlen(p); - char line[CBM_SZ_512]; - size_t cp = len < sizeof(line) - 1 ? len : sizeof(line) - 1; - memcpy(line, p, cp); - line[cp] = '\0'; - const char *t = line; - while (*t == ' ' || *t == '\t') { - t++; - } - /* Skip blanks, comments, options (-r, --hash), and URLs. */ - if (t[0] && t[0] != '#' && t[0] != '-' && strstr(t, "://") == NULL) { - char name[CBM_SZ_256]; - size_t n = 0; - while (t[n] && t[n] != '=' && t[n] != '<' && t[n] != '>' && t[n] != '!' && - t[n] != '~' && t[n] != '[' && t[n] != ';' && t[n] != ' ' && t[n] != '\t' && - t[n] != '\r' && n + 1 < sizeof(name)) { - name[n] = t[n]; - n++; - } - name[n] = '\0'; - edges += emit_dep_edge(ctx, src, rel_path, "pypi", name); - } - if (!eol) { - break; - } - p = eol + 1; - } - return edges; -} - -static void handle_dep_manifest(cbm_pipeline_ctx_t *ctx, const char *rel_path, const char *source, - const char *ecosystem) { - if (!source) { - return; - } - char *file_qn = cbm_pipeline_fqn_compute(ctx->project_name, rel_path, "__file__"); - const cbm_gbuf_node_t *src = file_qn ? cbm_gbuf_find_by_qn(ctx->gbuf, file_qn) : NULL; - free(file_qn); - if (!src) { - return; - } - int dep_edges = strcmp(ecosystem, "gomod") == 0 - ? parse_gomod_deps(ctx, src, rel_path, source) - : parse_requirements_deps(ctx, src, rel_path, source); - cbm_log_info("pass.k8s.depmanifest", "file", rel_path, "deps", cbm_log_itoa(dep_edges)); -} - -/* ── Pass entry point ────────────────────────────────────────────── */ - -int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count) { - cbm_log_info("pass.start", "pass", "k8s", "files", cbm_log_itoa(file_count)); - - cbm_init(); - - int kustomize_count = 0; - int manifest_count = 0; - int helm_count = 0; - - /* Collect per-manifest selector/label records for cross-manifest matching. */ - k8s_record_array_t recs = {0}; - recs.items = calloc(K8S_MAX_RECORDS, sizeof(*recs.items)); - recs.cap = recs.items ? K8S_MAX_RECORDS : 0; - - for (int i = 0; i < file_count; i++) { - if (cbm_pipeline_check_cancel(ctx)) { - free(recs.items); - return CBM_NOT_FOUND; - } - - const char *path = files[i].path; - const char *rel = files[i].rel_path; - CBMLanguage lang = files[i].language; - const char *base = cbm_path_base(rel); - - if (is_gomod_file(base) || lang == CBM_LANG_GOMOD || is_requirements_file(base)) { - int dep_len = 0; - char *dep_src = k8s_read_file(path, &dep_len); - if (dep_src) { - handle_dep_manifest(ctx, rel, dep_src, - is_requirements_file(base) ? "pypi" : "gomod"); - free(dep_src); - } - } else if (cbm_is_kustomize_file(base)) { - handle_kustomize(ctx, path, rel); - kustomize_count++; - } else if (lang == CBM_LANG_YAML || lang == CBM_LANG_K8S) { - /* Read source once to classify (and reuse for uncached extraction). */ - int src_len = 0; - char *source = k8s_read_file(path, &src_len); - if (source) { - if (is_helm_chart_file(base)) { - handle_helm_chart(ctx, rel, source); - helm_count++; - } else if (cbm_is_k8s_manifest(base, source)) { - /* Extract with CBM_LANG_K8S: the YAML extraction has no - * "Resource" definitions. Pass the already-read source - * buffer so handle_k8s_manifest does not re-read. */ - k8s_record_t *rec = (recs.count < recs.cap) ? &recs.items[recs.count] : NULL; - handle_k8s_manifest(ctx, path, rel, source, src_len, rec); - if (rec && rec->node_id > 0) { - recs.count++; - } - manifest_count++; - } - free(source); - } - } - } - - /* Connect Services to the workloads their selectors target (INFRA_MAPS). */ - k8s_link_selectors(ctx, &recs); - free(recs.items); - - cbm_log_info("pass.done", "pass", "k8s", "kustomize", cbm_log_itoa(kustomize_count), - "manifests", cbm_log_itoa(manifest_count)); - (void)helm_count; - return 0; -} diff --git a/src/pipeline/pass_k8s.zig b/src/pipeline/pass_k8s.zig new file mode 100644 index 00000000..6c7e55d9 --- /dev/null +++ b/src/pipeline/pass_k8s.zig @@ -0,0 +1,600 @@ +//! pass_k8s.zig — Pipeline pass for Kubernetes manifest and Kustomize overlay processing. +//! +//! For each discovered YAML file: +//! 1. Check if it is a kustomize overlay (kustomization.yaml / kustomization.yml) +//! → emit a Module node and IMPORTS edges for each resources/bases/patches entry +//! 2. Else if it is a generic k8s manifest (apiVersion: detected) +//! → emit one Resource node per file (first document only — multi-document YAML is not yet +//! supported) +//! +//! Depends on: pass_infrascan (cbm_is_kustomize_file, cbm_is_k8s_manifest, cbm_infra_qn), +//! extraction layer (cbm.h), graph_buffer, pipeline internals. + +const std = @import("std"); +const c = @import("c"); +const interop = @import("../foundation/interop.zig"); +const util = @import("util.zig"); + +const CStr = util.CStr; +const MutStr = util.MutStr; +const ix = util.ix; +const streq = util.streq; +const Ctx = c.cbm_pipeline_ctx_t; +const Node = c.cbm_gbuf_node_t; + +// ── Internal helpers ──────────────────────────────────────────────── + +/// Read entire file into heap-allocated buffer. Returns NULL on error. +/// Caller must free(). Sets out_len to byte count. +fn k8sReadFile(path: CStr, out_len: *c_int) MutStr { + const f = c.cbm_fopen(path, "rb") orelse return null; + + _ = c.fseek(f, 0, c.SEEK_END); + const size = c.ftell(f); + _ = c.fseek(f, 0, c.SEEK_SET); + + if (size <= 0 or size > c.cbm_max_file_bytes()) { // generous, env-configurable cap (B4) + _ = c.fclose(f); + return null; + } + const usize_size: usize = @intCast(size); + + // +pad: tree-sitter lexer lookahead reads past EOF; keep it in-bounds + const CBM_TS_LOOKAHEAD_PAD = 16; + const buf: MutStr = @ptrCast(c.malloc(usize_size + CBM_TS_LOOKAHEAD_PAD)); + if (buf == null) { + _ = c.fclose(f); + return null; + } + + var nread = c.fread(buf, 1, usize_size, f); + _ = c.fclose(f); + if (nread > usize_size) nread = usize_size; + @memset(buf[nread..][0..CBM_TS_LOOKAHEAD_PAD], 0); + out_len.* = @intCast(nread); + return buf; +} + +fn logFileCount(msg: [*:0]const u8, rel_path: CStr, key: [*:0]const u8, count: c_int) void { + interop.logInfo(msg, .{ + @as(CStr, "file"), rel_path, @as(CStr, key), @as(CStr, c.cbm_log_itoa(count)), + }); +} + +// ── Kustomize handler ─────────────────────────────────────────────── + +fn handleKustomize(ctx: *Ctx, path: CStr, rel_path: CStr) void { + // Emit Module node for this kustomize overlay file + const mod_qn = c.cbm_infra_qn(ctx.project_name, rel_path, "kustomize", null); + if (mod_qn == null) return; + + const mod_id = c.cbm_gbuf_upsert_node(ctx.gbuf, "Module", c.cbm_path_base(rel_path), mod_qn, rel_path, 1, 0, "{\"source\":\"kustomize\"}"); + c.free(mod_qn); + + if (mod_id <= 0) return; + + // Emit IMPORTS edges for resources/bases/patches/components entries + var import_count: c_int = 0; + var res: ?*c.CBMFileResult = null; + var src_len: c_int = 0; + const source = k8sReadFile(path, &src_len); + if (source != null) { + res = c.cbm_extract_file(source, src_len, c.CBM_LANG_KUSTOMIZE, ctx.project_name, rel_path, c.CBM_EXTRACT_BUDGET, null, null); + c.free(source); + } + + if (res) |result| { + for (0..ix(result.imports.count)) |j| { + const imp = util.at(result.imports.items, j); + if (imp.module_path == null) continue; + + // Compute target file QN + const target_qn = c.cbm_pipeline_fqn_compute(ctx.project_name, imp.module_path, "__file__"); + if (target_qn == null) continue; + + const target: ?*const Node = c.cbm_gbuf_find_by_qn(ctx.gbuf, target_qn); + c.free(target_qn); + + if (target) |t| { + _ = c.cbm_gbuf_insert_edge(ctx.gbuf, mod_id, t.id, "IMPORTS", "{\"via\":\"kustomize\"}"); + import_count += 1; + } + } + c.cbm_free_result(result); + } + + logFileCount("pass.k8s.kustomize", rel_path, "imports", import_count); +} + +// ── K8s cross-manifest label-selector matching ────────────────────── +// A Service routes traffic to the workload Pods whose labels match the +// Service's spec.selector. We record, per manifest, the Resource node id plus +// its selector label-values (Services) and pod label-values (workloads), then +// after all manifests are processed connect each Service to the workload(s) it +// targets via an INFRA_MAPS edge. This models the runtime traffic path the same +// way k8s itself resolves Service → Endpoints by label. +// +// Matching is by label *value*: a Service selector `app: frontend` matches a +// workload that either carries the pod label value "frontend" OR is named +// "frontend" (the ubiquitous app==name convention, used when the pod template +// omits explicit labels). +const K8S_MAX_LABELS = 16; +const K8S_MAX_RECORDS = 512; +const K8S_LABEL_LEN = 128; + +const LabelVals = [K8S_MAX_LABELS][K8S_LABEL_LEN]u8; + +const K8sRecord = extern struct { + node_id: i64, + name: [K8S_LABEL_LEN]u8, + is_service: bool, + is_workload: bool, + selector_vals: LabelVals, // Service spec.selector values + n_selector: c_int, + label_vals: LabelVals, // workload pod-template label values + n_label: c_int, +}; + +const K8sRecordArray = struct { + items: [*c]K8sRecord = null, + count: c_int = 0, + cap: c_int = 0, +}; + +/// Count leading-space indentation of a line (tabs are invalid YAML indent). +fn k8sIndent(line: [*]const u8) c_int { + var n: usize = 0; + while (line[n] == ' ') n += 1; + return @intCast(n); +} + +/// Split `key: value` (already de-indented). Returns true if a key was found; +/// fills key/val (val empty when the key opens a nested block). +fn k8sSplitKv(t: [*:0]const u8, key: []u8, val: []u8) bool { + key[0] = 0; + val[0] = 0; + if (t[0] == '#' or t[0] == '-' or t[0] == 0) return false; + const colon: CStr = c.strchr(t, ':'); + if (colon == null) return false; + const klen = util.ptrDiff(colon, t); + if (klen == 0 or klen >= key.len) return false; + @memcpy(key[0..klen], t[0..klen]); + key[klen] = 0; + var v: [*]const u8 = colon + 1; + while (v[0] == ' ' or v[0] == '\t') v += 1; + var vn: usize = 0; + while (v[vn] != 0 and v[vn] != '\r' and v[vn] != '\n' and v[vn] != '#' and vn + 1 < val.len) { + val[vn] = v[vn]; + vn += 1; + } + // trim trailing space + while (vn > 0 and (val[vn - 1] == ' ' or val[vn - 1] == '\t')) vn -= 1; + val[vn] = 0; + // strip surrounding quotes + if (vn >= 2 and (val[0] == '"' or val[0] == '\'') and val[vn - 1] == val[0]) { + _ = c.memmove(val.ptr, val.ptr + 1, vn - 2); + val[vn - 2] = 0; + } + return true; +} + +fn k8sAddVal(dst: *LabelVals, n: *c_int, v: CStr) void { + if (n.* >= K8S_MAX_LABELS or v == null or v[0] == 0) return; + _ = c.snprintf(&dst[ix(n.*)], K8S_LABEL_LEN, "%s", v); + n.* += 1; +} + +/// Scan a single-document k8s manifest's text for the resource name, selector +/// label-values (Service) and pod-template label-values (workload). A small +/// indentation path-stack distinguishes spec.selector from +/// spec.template.metadata.labels and from top-level metadata.name. +fn k8sScanLabels(source: CStr, rec: *K8sRecord) void { + const K8S_PATH_DEPTH = 12; + const Frame = struct { + indent: c_int, + key: [64]u8, + }; + var stack: [K8S_PATH_DEPTH]Frame = undefined; + var depth: usize = 0; + var got_name = false; + + var p: CStr = source; + while (p != null and p[0] != 0) { + const eol: CStr = c.strchr(p, '\n'); + const len: usize = if (eol != null) util.ptrDiff(eol, p) else c.strlen(p); + var line: [c.CBM_SZ_512:0]u8 = undefined; + const cp = @min(len, c.CBM_SZ_512 - 1); + @memcpy(line[0..cp], p[0..cp]); + line[cp] = 0; + + // End of first YAML document — stop (one Resource per file). + var trimmed: [*:0]const u8 = &line; + while (trimmed[0] == ' ') trimmed += 1; + if (c.strncmp(trimmed, "---", 3) == 0 and @intFromPtr(trimmed) == @intFromPtr(&line)) break; + + if (trimmed[0] != 0 and trimmed[0] != '#') { + const ind = k8sIndent(&line); + while (depth > 0 and stack[depth - 1].indent >= ind) depth -= 1; + var key: [64]u8 = undefined; + var val: [K8S_LABEL_LEN]u8 = undefined; + if (k8sSplitKv(trimmed, &key, &val)) { + // Build the current dotted path for context decisions. + const under_selector = depth >= 1 and streq(&stack[depth - 1].key, "selector"); + const under_labels = depth >= 1 and streq(&stack[depth - 1].key, "labels"); + const under_metadata = depth >= 1 and streq(&stack[depth - 1].key, "metadata"); + + if (val[0] == 0) { + // Block-opening key: push onto the path stack. + if (depth < K8S_PATH_DEPTH) { + stack[depth].indent = ind; + _ = c.snprintf(&stack[depth].key, stack[depth].key.len, "%s", @as(CStr, &key)); + depth += 1; + } + } else { + // Leaf key: value. + if (ind == 0 and streq(&key, "kind")) { + rec.is_service = streq(&val, "Service"); + rec.is_workload = + streq(&val, "Deployment") or streq(&val, "StatefulSet") or + streq(&val, "DaemonSet") or streq(&val, "ReplicaSet") or + streq(&val, "ReplicationController") or streq(&val, "Pod") or + streq(&val, "Job") or streq(&val, "CronJob"); + } else if (!got_name and under_metadata and streq(&key, "name")) { + _ = c.snprintf(&rec.name, rec.name.len, "%s", @as(CStr, &val)); + got_name = true; + } else if (under_selector) { + // Service spec.selector matchLabels values. + k8sAddVal(&rec.selector_vals, &rec.n_selector, &val); + } else if (under_labels) { + // Pod-template / metadata labels values. + k8sAddVal(&rec.label_vals, &rec.n_label, &val); + } + } + } + } + + if (eol == null) break; + p = eol + 1; + } +} + +/// True if any of the service's selector values matches the workload. +fn k8sSelectorMatches(svc: *const K8sRecord, wl: *const K8sRecord) bool { + for (svc.selector_vals[0..ix(svc.n_selector)]) |*sv| { + if (sv[0] == 0) continue; + if (wl.name[0] != 0 and streq(sv, &wl.name)) return true; + for (wl.label_vals[0..ix(wl.n_label)]) |*label| { + if (streq(sv, label)) return true; + } + } + return false; +} + +/// After all manifests are recorded, connect each Service to the workload(s) its +/// selector targets via an INFRA_MAPS edge (Service Resource → workload Resource). +fn k8sLinkSelectors(ctx: *Ctx, recs: *const K8sRecordArray) void { + var edges: c_int = 0; + const count = ix(recs.count); + for (0..count) |i| { + const svc = util.at(recs.items, i); + if (!svc.is_service or svc.n_selector == 0 or svc.node_id <= 0) continue; + for (0..count) |j| { + const wl = util.at(recs.items, j); + if (i == j or !wl.is_workload or wl.node_id <= 0) continue; + if (k8sSelectorMatches(svc, wl)) { + var props: [c.CBM_SZ_256]u8 = undefined; + _ = c.snprintf( + &props, + props.len, + "{\"kind\":\"selector\",\"service\":\"%s\",\"workload\":\"%s\"}", + @as(CStr, &svc.name), + @as(CStr, &wl.name), + ); + _ = c.cbm_gbuf_insert_edge(ctx.gbuf, svc.node_id, wl.node_id, "INFRA_MAPS", &props); + edges += 1; + } + } + } + if (edges > 0) { + interop.logInfo("pass.k8s.selectors", .{ + @as(CStr, "linked"), @as(CStr, c.cbm_log_itoa(edges)), + }); + } +} + +// ── K8s manifest handler ──────────────────────────────────────────── + +/// The File node of rel_path, or null. +fn fileNodeOf(ctx: *Ctx, rel_path: CStr) ?*const Node { + const file_qn = c.cbm_pipeline_fqn_compute(ctx.project_name, rel_path, "__file__"); + const node: ?*const Node = if (file_qn != null) c.cbm_gbuf_find_by_qn(ctx.gbuf, file_qn) else null; + c.free(file_qn); + return node; +} + +/// source/src_len are the already-read file bytes (caller retains ownership and +/// must free after this call returns). When `rec` is non-NULL it is populated +/// with the first Resource's node id, name and label/selector values for later +/// cross-manifest selector matching. +fn handleK8sManifest(ctx: *Ctx, rel_path: CStr, source: CStr, src_len: c_int, rec: ?*K8sRecord) void { + var resource_count: c_int = 0; + + const res: *c.CBMFileResult = @as(?*c.CBMFileResult, c.cbm_extract_file(source, src_len, c.CBM_LANG_K8S, ctx.project_name, rel_path, c.CBM_EXTRACT_BUDGET, null, null)) orelse return; + + // Compute file node QN for DEFINES edges + const file_node = fileNodeOf(ctx, rel_path); + + for (0..ix(res.defs.count)) |d| { + const def = util.at(res.defs.items, d); + if (def.label == null or !streq(def.label, "Resource")) continue; + if (def.name == null or def.qualified_name == null) continue; + + const node_id = c.cbm_gbuf_upsert_node(ctx.gbuf, "Resource", def.name, def.qualified_name, rel_path, @intCast(def.start_line), @intCast(def.end_line), "{\"source\":\"k8s\"}"); + + // DEFINES edge: File → Resource + if (file_node != null and node_id > 0) { + _ = c.cbm_gbuf_insert_edge(ctx.gbuf, file_node.?.id, node_id, "DEFINES", "{}"); + } + + // Capture the first Resource for cross-manifest selector matching. + if (rec) |r| { + if (r.node_id <= 0 and node_id > 0) r.node_id = node_id; + } + + resource_count += 1; + } + + c.cbm_free_result(res); + + // Record selector / pod-label values for later Service → workload linking. + if (rec) |r| { + if (r.node_id > 0) k8sScanLabels(source, r); + } + + logFileCount("pass.k8s.manifest", rel_path, "resources", resource_count); +} + +// ── Helm chart handler ────────────────────────────────────────────── + +fn isHelmChartFile(base: CStr) bool { + return streq(base, "Chart.yaml") or streq(base, "Chart.yml"); +} + +/// Emit a Chart node for a Chart.yaml and a DEPENDS_ON edge to a (shared, +/// deduplicated) Chart node per declared dependency (#338). +fn handleHelmChart(ctx: *Ctx, rel_path: CStr, source: CStr) void { + var hc: c.cbm_helm_chart_t = undefined; + if (c.cbm_parse_helm_chart(source, &hc) != 0) return; + + const cname: CStr = if (hc.chart_name[0] != 0) &hc.chart_name else c.cbm_path_base(rel_path); + const chart_qn = c.cbm_infra_qn(ctx.project_name, rel_path, "helm-chart", null); + if (chart_qn == null) return; + const chart_id = c.cbm_gbuf_upsert_node(ctx.gbuf, "Chart", cname, chart_qn, rel_path, 1, 0, "{\"source\":\"helm\"}"); + c.free(chart_qn); + + if (fileNodeOf(ctx, rel_path)) |file_node| { + if (chart_id > 0) { + _ = c.cbm_gbuf_insert_edge(ctx.gbuf, file_node.id, chart_id, "DEFINES", "{}"); + } + } + + var dep_edges: c_int = 0; + var i: usize = 0; + while (i < hc.dep_count and chart_id > 0) : (i += 1) { + // Stable per-project QN so multiple charts depending on the same chart + // link to one shared dependency node. + const dep_name: CStr = &hc.deps[i]; + var dep_qn: [c.CBM_SZ_512]u8 = undefined; + _ = c.snprintf(&dep_qn, dep_qn.len, "%s.__helm_dep__.%s", ctx.project_name, dep_name); + const dep_id = c.cbm_gbuf_upsert_node(ctx.gbuf, "Chart", dep_name, &dep_qn, rel_path, 1, 0, "{\"source\":\"helm\",\"external\":true}"); + if (dep_id > 0) { + _ = c.cbm_gbuf_insert_edge(ctx.gbuf, chart_id, dep_id, "DEPENDS_ON", "{}"); + dep_edges += 1; + } + } + logFileCount("pass.k8s.helm", rel_path, "deps", dep_edges); +} + +// ── Dependency-manifest handler (go.mod / requirements.txt) ───────── + +fn isGomodFile(base: CStr) bool { + return streq(base, "go.mod"); +} + +fn isRequirementsFile(base: CStr) bool { + return streq(base, "requirements.txt"); +} + +/// Emit a DEPENDS_ON edge from the manifest file node to a (shared, per-project) +/// external Package node. Mirrors the Helm Chart.yaml DEPENDS_ON shape. +fn emitDepEdge(ctx: *Ctx, src: *const Node, rel_path: CStr, ecosystem: CStr, name: [*:0]const u8) c_int { + if (name[0] == 0) return 0; + var dep_qn: [c.CBM_SZ_512]u8 = undefined; + _ = c.snprintf(&dep_qn, dep_qn.len, "%s.__%s_dep__.%s", ctx.project_name, ecosystem, name); + var dep_props: [c.CBM_SZ_256]u8 = undefined; + _ = c.snprintf(&dep_props, dep_props.len, "{\"source\":\"%s\",\"external\":true}", ecosystem); + const dep_id = c.cbm_gbuf_upsert_node(ctx.gbuf, "Package", name, &dep_qn, rel_path, 1, 0, &dep_props); + if (dep_id > 0 and dep_id != src.id) { + _ = c.cbm_gbuf_insert_edge(ctx.gbuf, src.id, dep_id, "DEPENDS_ON", "{}"); + return 1; + } + return 0; +} + +/// Copy the first whitespace-delimited token of `line` into `out`. +fn firstToken(line_in: [*:0]const u8, out: [:0]u8) void { + out[0] = 0; + var line = line_in; + while (line[0] == ' ' or line[0] == '\t') line += 1; + var n: usize = 0; + while (line[n] != 0 and line[n] != ' ' and line[n] != '\t' and line[n] != '\r' and + line[n] != '\n' and n + 1 < out.len + 1) + { + out[n] = line[n]; + n += 1; + } + out[n] = 0; +} + +/// One line of `source` starting at p, copied into `line` (truncated to fit). +/// Returns the pointer to the line's newline, or null on the last line. +fn copyLine(p: CStr, line: *[c.CBM_SZ_512:0]u8) CStr { + const eol: CStr = c.strchr(p, '\n'); + const len: usize = if (eol != null) util.ptrDiff(eol, p) else c.strlen(p); + const cp = @min(len, c.CBM_SZ_512 - 1); + @memcpy(line[0..cp], p[0..cp]); + line[cp] = 0; + return eol; +} + +/// Parse go.mod `require` directives (single-line and block forms) and emit a +/// DEPENDS_ON edge per dependency. go.mod requires are not surfaced as imports +/// by the extraction layer, so we parse the manifest text directly here. +fn parseGomodDeps(ctx: *Ctx, src: *const Node, rel_path: CStr, source: CStr) c_int { + var edges: c_int = 0; + var in_block = false; + var p: CStr = source; + while (p != null and p[0] != 0) { + var line: [c.CBM_SZ_512:0]u8 = undefined; + const eol = copyLine(p, &line); + var t: [*:0]const u8 = &line; + while (t[0] == ' ' or t[0] == '\t') t += 1; + if (in_block) { + if (t[0] == ')') { + in_block = false; + } else if (t[0] != 0 and t[0] != '/') { + var name: [c.CBM_SZ_256 - 1:0]u8 = undefined; + firstToken(t, &name); + edges += emitDepEdge(ctx, src, rel_path, "gomod", &name); + } + } else if (c.strncmp(t, "require", 7) == 0 and (t[7] == ' ' or t[7] == '\t' or t[7] == '(')) { + var rest: [*:0]const u8 = t + 7; + while (rest[0] == ' ' or rest[0] == '\t') rest += 1; + if (rest[0] == '(') { + in_block = true; + } else if (rest[0] != 0) { + var name: [c.CBM_SZ_256 - 1:0]u8 = undefined; + firstToken(rest, &name); + edges += emitDepEdge(ctx, src, rel_path, "gomod", &name); + } + } + if (eol == null) break; + p = eol + 1; + } + return edges; +} + +/// Parse requirements.txt entries (one package spec per line) and emit a +/// DEPENDS_ON edge per dependency. The package name is the leading token up to +/// the first version/extras/comment delimiter. +fn parseRequirementsDeps(ctx: *Ctx, src: *const Node, rel_path: CStr, source: CStr) c_int { + var edges: c_int = 0; + var p: CStr = source; + while (p != null and p[0] != 0) { + var line: [c.CBM_SZ_512:0]u8 = undefined; + const eol = copyLine(p, &line); + var t: [*:0]const u8 = &line; + while (t[0] == ' ' or t[0] == '\t') t += 1; + // Skip blanks, comments, options (-r, --hash), and URLs. + if (t[0] != 0 and t[0] != '#' and t[0] != '-' and c.strstr(t, "://") == null) { + var name: [c.CBM_SZ_256 - 1:0]u8 = undefined; + var n: usize = 0; + while (t[n] != 0 and t[n] != '=' and t[n] != '<' and t[n] != '>' and t[n] != '!' and + t[n] != '~' and t[n] != '[' and t[n] != ';' and t[n] != ' ' and t[n] != '\t' and + t[n] != '\r' and n + 1 < c.CBM_SZ_256) + { + name[n] = t[n]; + n += 1; + } + name[n] = 0; + edges += emitDepEdge(ctx, src, rel_path, "pypi", &name); + } + if (eol == null) break; + p = eol + 1; + } + return edges; +} + +fn handleDepManifest(ctx: *Ctx, rel_path: CStr, source: CStr, ecosystem: CStr) void { + if (source == null) return; + const src = fileNodeOf(ctx, rel_path) orelse return; + const dep_edges = if (streq(ecosystem, "gomod")) + parseGomodDeps(ctx, src, rel_path, source) + else + parseRequirementsDeps(ctx, src, rel_path, source); + logFileCount("pass.k8s.depmanifest", rel_path, "deps", dep_edges); +} + +// ── Pass entry point ──────────────────────────────────────────────── + +pub export fn cbm_pipeline_pass_k8s(ctx: *Ctx, files: [*c]const c.cbm_file_info_t, file_count: c_int) c_int { + interop.logInfo("pass.start", .{ + @as(CStr, "pass"), @as(CStr, "k8s"), @as(CStr, "files"), @as(CStr, c.cbm_log_itoa(file_count)), + }); + + _ = c.cbm_init(); + + var kustomize_count: c_int = 0; + var manifest_count: c_int = 0; + + // Collect per-manifest selector/label records for cross-manifest matching. + var recs: K8sRecordArray = .{}; + recs.items = interop.callocN(K8sRecord, K8S_MAX_RECORDS); + recs.cap = if (recs.items != null) K8S_MAX_RECORDS else 0; + + var i: usize = 0; + while (i < file_count) : (i += 1) { + if (util.checkCancel(ctx) != 0) { + c.free(recs.items); + return c.CBM_NOT_FOUND; + } + + const path: CStr = files[i].path; + const rel: CStr = files[i].rel_path; + const lang = files[i].language; + const base: CStr = c.cbm_path_base(rel); + + if (isGomodFile(base) or lang == c.CBM_LANG_GOMOD or isRequirementsFile(base)) { + var dep_len: c_int = 0; + const dep_src = k8sReadFile(path, &dep_len); + if (dep_src != null) { + handleDepManifest(ctx, rel, dep_src, if (isRequirementsFile(base)) "pypi" else "gomod"); + c.free(dep_src); + } + } else if (c.cbm_is_kustomize_file(base)) { + handleKustomize(ctx, path, rel); + kustomize_count += 1; + } else if (lang == c.CBM_LANG_YAML or lang == c.CBM_LANG_K8S) { + // Read source once to classify (and reuse for uncached extraction). + var src_len: c_int = 0; + const source = k8sReadFile(path, &src_len); + if (source != null) { + if (isHelmChartFile(base)) { + handleHelmChart(ctx, rel, source); + } else if (c.cbm_is_k8s_manifest(base, source)) { + // Extract with CBM_LANG_K8S: the YAML extraction has no + // "Resource" definitions. Pass the already-read source + // buffer so handleK8sManifest does not re-read. + const rec: ?*K8sRecord = if (recs.count < recs.cap) util.at(recs.items, ix(recs.count)) else null; + handleK8sManifest(ctx, rel, source, src_len, rec); + if (rec) |r| { + if (r.node_id > 0) recs.count += 1; + } + manifest_count += 1; + } + c.free(source); + } + } + } + + // Connect Services to the workloads their selectors target (INFRA_MAPS). + k8sLinkSelectors(ctx, &recs); + c.free(recs.items); + + interop.logInfo("pass.done", .{ + @as(CStr, "pass"), @as(CStr, "k8s"), + @as(CStr, "kustomize"), @as(CStr, c.cbm_log_itoa(kustomize_count)), + @as(CStr, "manifests"), @as(CStr, c.cbm_log_itoa(manifest_count)), + }); + return 0; +} diff --git a/src/pipeline/pass_lsp_cross.c b/src/pipeline/pass_lsp_cross.c deleted file mode 100644 index c0ad159b..00000000 --- a/src/pipeline/pass_lsp_cross.c +++ /dev/null @@ -1,1694 +0,0 @@ -/* - * pass_lsp_cross.c — Cross-file LSP type-aware call resolution pass. - * - * See pass_lsp_cross.h for the high-level contract. This file is the - * pipeline glue that converts the existing per-file extraction state - * (CBMDefinition / CBMImport / IMPORTS-edge gbuf state) into the input - * shape each language LSP's cbm_run_X_lsp_cross expects, then merges - * the resulting CBMResolvedCall entries back into per-file results. - * - * The pass is a no-op for any file whose CBMFileResult is missing or - * whose language has no cross-file LSP entry registered (e.g. Rust / - * Java today). Per-LSP emit functions dedup against entries already in - * resolved_calls, so this pass is also idempotent — safe to invoke - * multiple times if the pipeline gains a re-run path later. - */ -#include "pipeline/pass_lsp_cross.h" -#include "pipeline/pipeline.h" // cbm_pipeline_add_file_error -#include "pipeline/pipeline_internal.h" -#include "cbm.h" // CBMLSPStatus -#include "lsp/go_lsp.h" -#include "lsp/c_lsp.h" -#include "lsp/py_lsp.h" -#include "lsp/ts_lsp.h" -#include "lsp/php_lsp.h" -#include "lsp/java_lsp.h" -#include "lsp/kotlin_lsp.h" -#include "lsp/rust_lsp.h" -#include "lsp/rust_cargo.h" -#include "lsp/zig_lsp.h" -#include "graph_buffer/graph_buffer.h" -#include "foundation/constants.h" -#include "foundation/hash_table.h" -#include "foundation/log.h" -#include "foundation/compat_fs.h" - -#include -#include -#include -#include -#include - -/* ── Constants ─────────────────────────────────────────────────── */ - -enum { - PXC_MAX_FILE_BYTES_FACTOR = 100, /* source size cap, in MiB */ -}; - -/* ── Local helpers ─────────────────────────────────────────────── */ - -/* True for languages whose module QN is derived from the CONTAINING DIRECTORY - * (Java package, Go package) rather than the filename stem. MUST match the - * extraction-side cbm_lang_module_is_dir() in internal/cbm/helpers.c so the - * cross-file LSP caller_qn agrees with the def-node QN (the lsp_resolve join - * keys on exact equality). */ -static bool pxc_module_is_dir(CBMLanguage lang) { - return lang == CBM_LANG_JAVA || lang == CBM_LANG_GO; -} - -/* Slurp a file into a malloc'd, NUL-terminated buffer. Mirrors the - * read_file helper in pass_parallel.c (kept local so the - * pipeline doesn't grow a public read-file API just for this pass). */ -static char *pxc_read_file(const char *path, int *out_len) { - FILE *f = cbm_fopen(path, "rb"); - if (!f) - return NULL; - (void)fseek(f, 0, SEEK_END); - long size = ftell(f); - (void)fseek(f, 0, SEEK_SET); - if (size <= 0 || size > (long)PXC_MAX_FILE_BYTES_FACTOR * (long)CBM_SZ_1K * (long)CBM_SZ_1K) { - (void)fclose(f); - return NULL; - } - /* +pad: tree-sitter lexer lookahead reads past EOF; keep it in-bounds */ - enum { CBM_TS_LOOKAHEAD_PAD = 16 }; - char *buf = (char *)malloc((size_t)size + CBM_TS_LOOKAHEAD_PAD); - if (!buf) { - (void)fclose(f); - return NULL; - } - size_t nread = fread(buf, 1, (size_t)size, f); - (void)fclose(f); - if (nread > (size_t)size) - nread = (size_t)size; - memset(buf + nread, 0, CBM_TS_LOOKAHEAD_PAD); - *out_len = (int)nread; - return buf; -} - -/* Map a CBMDefinition.label to a CBMLSPDef.label. Per-language LSP registrars - * only care about type-like containers (Class/Struct/Interface/Trait/Enum/Type) - * plus Protocol/Function/Method — variables, modules, decorators, etc. are - * skipped. Struct passes through so Rust/Go struct type-registration via the - * cross-file LSP path is not dropped. */ -static const char *pxc_map_label(const char *label) { - if (!label) - return NULL; - if (cbm_label_is_type_like(label) || strcmp(label, "Protocol") == 0 || - strcmp(label, "Function") == 0 || strcmp(label, "Method") == 0) { - return label; - } - return NULL; -} - -/* Build the embedded_types "|"-separated string from base_classes[]. - * Returns NULL when there are no bases. Allocated in the supplied arena. */ -static const char *pxc_join_pipe(CBMArena *arena, const char *const *items) { - if (!items || !items[0]) - return NULL; - int count = 0; - size_t total = 0; - for (int i = 0; items[i]; i++) { - count++; - total += strlen(items[i]); - } - if (count == 0) - return NULL; - /* count - 1 separators + NUL. */ - size_t bufsz = total + (size_t)(count - 1) + 1; - char *buf = (char *)cbm_arena_alloc(arena, bufsz); - if (!buf) - return NULL; - char *p = buf; - for (int i = 0; i < count; i++) { - size_t n = strlen(items[i]); - memcpy(p, items[i], n); - p += n; - if (i + 1 < count) - *p++ = '|'; - } - *p = '\0'; - return buf; -} - -/* ── Cross-file base-class QN resolution ────────────────────────── - * - * CBMDefinition.base_classes carries the SOURCE SPELLING of each base - * ("Base", "django.db.Model", "React.Component"): extraction strips - * keywords and generic arguments, but it cannot know WHERE the name is - * declared. The Python and TS cross-file registrars, however, consume - * CBMLSPDef.embedded_types as fully-qualified names — py_lookup_attribute - * and ts_lookup_member feed each entry straight into - * cbm_registry_lookup_type. An unqualified spelling therefore matched - * nothing declared in ANOTHER file: `class Child(Base)` in child.py never - * saw Base in base.py, so a call to an inherited method through a typed - * receiver had no member to find and fell through to the weak textual - * cascade (where the #592/#606 guard correctly kills it). - * - * Resolve each base name ONCE per definition here, from exactly the - * inputs pass_semantic uses to draw its INHERITS edge: the project - * registry, the declaring module, and the file's import map. Two - * properties follow. The LSP's inheritance view is the same relation the - * graph records, so the two cannot diverge. And the binding is - * import- or same-module-backed, not a short-name guess, so the CALLS - * edge it enables is a supported fact that the weak-member guard keeps. - * - * Cost: O(defs) hash lookups + ONE import map per file. No per-call-site - * hierarchy walk, no registry scan, no per-file registry rebuild. - */ -static bool pxc_lang_resolves_base_qns(CBMLanguage lang) { - switch (lang) { - case CBM_LANG_PYTHON: - case CBM_LANG_JAVASCRIPT: - case CBM_LANG_TYPESCRIPT: - case CBM_LANG_TSX: - return true; - default: - /* Go / JVM / C# / C++ / Rust registrars qualify their own embedded - * types already (struct embedding, parent_class chains, impl-trait - * provenance). Re-resolving here would fight those paths, so they - * keep the raw join. */ - return false; - } -} - -/* True for the registry strategies that are pure short-name guesses. - * EXPLICIT drop-list, mirroring cbm_tsjs_suppress_weak_method_match: a - * base class bound by "some project type happens to share this name" is - * exactly the fabricated relation #606 removed, and inheritance - * multiplies it — every inherited member of the wrong base would become - * a callable target. Import-, module- and suffix-aware strategies are - * kept; anything unresolved simply retains its source spelling and the - * behaviour that predates this resolution. */ -static bool pxc_base_strategy_is_weak(const char *strategy) { - if (!strategy || !strategy[0]) { - return true; - } - return strcmp(strategy, "suffix_match") == 0 || strcmp(strategy, "unique_name") == 0 || - strcmp(strategy, "field_type_hint") == 0 || strcmp(strategy, "fuzzy") == 0; -} - -/* Resolve one base-class spelling to a project QN. Mirrors - * pass_semantic.c::resolve_as_class — same registry, same type-like veto — - * then additionally rejects weak short-name strategies (see above). - * Returns NULL when the base is not a confidently-known project type; - * stdlib and third-party bases land here and keep their raw spelling. */ -static const char *pxc_resolve_base_qn(const cbm_registry_t *reg, const char *raw, - const char *module_qn, const char **imp_keys, - const char **imp_vals, int imp_count) { - if (!reg || !raw || !raw[0]) { - return NULL; - } - cbm_resolution_t res = cbm_registry_resolve(reg, raw, module_qn, imp_keys, imp_vals, imp_count); - if (!res.qualified_name || !res.qualified_name[0]) { - return NULL; - } - if (pxc_base_strategy_is_weak(res.strategy)) { - return NULL; - } - if (!cbm_label_is_type_like(cbm_registry_label_of(reg, res.qualified_name))) { - return NULL; - } - return res.qualified_name; -} - -/* pxc_join_pipe over base_classes, substituting each resolved QN for its - * source spelling. Unresolved entries pass through verbatim so a base the - * registry does not know keeps working exactly as before. */ -static const char *pxc_join_base_qns(CBMArena *arena, const char *const *bases, - const cbm_registry_t *reg, const char *module_qn, - const char **imp_keys, const char **imp_vals, int imp_count) { - if (!bases || !bases[0]) { - return NULL; - } - int count = 0; - while (bases[count]) { - count++; - } - const char **resolved = - (const char **)cbm_arena_alloc(arena, (size_t)(count + 1) * sizeof(const char *)); - if (!resolved) { - return pxc_join_pipe(arena, bases); - } - for (int i = 0; i < count; i++) { - const char *qn = - pxc_resolve_base_qn(reg, bases[i], module_qn, imp_keys, imp_vals, imp_count); - resolved[i] = qn ? qn : bases[i]; - } - resolved[count] = NULL; - return pxc_join_pipe(arena, resolved); -} - -static bool pxc_is_jvm_lang(CBMLanguage lang); - -static const char *pxc_last_component(const char *qn) { - if (!qn) { - return NULL; - } - const char *dot = strrchr(qn, '.'); - return dot ? dot + 1 : qn; -} - -static const char *pxc_jvm_type_qn(CBMArena *arena, const char *namespace_name, - const char *type_qn_or_name) { - if (!arena || !namespace_name || !namespace_name[0] || !type_qn_or_name) { - return type_qn_or_name; - } - const char *short_name = pxc_last_component(type_qn_or_name); - if (!short_name || !short_name[0]) { - return type_qn_or_name; - } - return cbm_arena_sprintf(arena, "%s.%s", namespace_name, short_name); -} - -static const char *pxc_jvm_def_qn(CBMArena *arena, const CBMDefinition *src, - const char *namespace_name, const char *label) { - if (!arena || !src || !namespace_name || !namespace_name[0]) { - return src ? src->qualified_name : NULL; - } - if (strcmp(label, "Method") == 0 || strcmp(label, "Function") == 0 || - strcmp(label, "Constructor") == 0) { - if (src->parent_class && src->parent_class[0]) { - return cbm_arena_sprintf(arena, "%s.%s.%s", namespace_name, - pxc_last_component(src->parent_class), src->name); - } - return cbm_arena_sprintf(arena, "%s.%s", namespace_name, src->name); - } - return cbm_arena_sprintf(arena, "%s.%s", namespace_name, src->name); -} - -static const char *pxc_infer_jvm_namespace(CBMArena *arena, const char *rel_path, - CBMLanguage lang) { - if (!arena || !rel_path || !pxc_is_jvm_lang(lang)) { - return NULL; - } - const char *root = NULL; - const char *lang_root = lang == CBM_LANG_KOTLIN ? "kotlin/" : "java/"; - if (strncmp(rel_path, "src/main/", 9) == 0 && - strncmp(rel_path + 9, lang_root, strlen(lang_root)) == 0) { - root = rel_path + 9 + strlen(lang_root); - } else if (strncmp(rel_path, "src/test/", 9) == 0 && - strncmp(rel_path + 9, lang_root, strlen(lang_root)) == 0) { - root = rel_path + 9 + strlen(lang_root); - } else { - const char *needle = lang == CBM_LANG_KOTLIN ? "/kotlin/" : "/java/"; - root = strstr(rel_path, needle); - if (root) { - root += strlen(needle); - } else if (strncmp(rel_path, "src/", 4) == 0) { - root = rel_path + 4; - } else { - root = strstr(rel_path, "/src/"); - if (root) { - root += strlen("/src/"); - } - } - } - if (!root || !root[0]) { - return NULL; - } - if (strncmp(root, "main/", 5) == 0 || strncmp(root, "test/", 5) == 0) { - root += 5; - } - if (strncmp(root, "java/", 5) == 0) { - root += 5; - } else if (strncmp(root, "kotlin/", 7) == 0) { - root += 7; - } - const char *slash = strrchr(root, '/'); - if (!slash || slash <= root) { - return NULL; - } - size_t len = (size_t)(slash - root); - char *ns = (char *)cbm_arena_alloc(arena, len + 1); - if (!ns) { - return NULL; - } - memcpy(ns, root, len); - ns[len] = '\0'; - for (size_t i = 0; i < len; i++) { - if (ns[i] == '/') { - ns[i] = '.'; - } - } - return ns; -} - -static const char *pxc_import_lookup(const char *local, size_t len, const char **imp_keys, - const char **imp_vals, int imp_count) { - for (int i = 0; i < imp_count; i++) { - if (imp_keys[i] && imp_vals[i] && strlen(imp_keys[i]) == len && - strncmp(imp_keys[i], local, len) == 0) { - return imp_vals[i]; - } - } - return NULL; -} - -/* The innermost container enclosing a container-level Zig alias (a QN prefix - * of its parent_class, above the module) that declares `head` in this file, - * as `.`. NULL when none does. */ -static const char *pxc_zig_container_head(CBMArena *arena, const CBMFileResult *file, - const char *scope, const char *module_qn, - const char *head, size_t len) { - size_t mlen = strlen(module_qn); - size_t slen = strlen(scope); - while (slen > mlen) { - for (int i = 0; i < file->defs.count; i++) { - const char *qn = file->defs.items[i].qualified_name; - if (qn && strlen(qn) == slen + 1 + len && strncmp(qn, scope, slen) == 0 && - qn[slen] == '.' && strncmp(qn + slen + 1, head, len) == 0) { - return cbm_arena_strdup(arena, qn); - } - } - while (slen > mlen && scope[slen - 1] != '.') { - slen--; - } - slen = slen > 0 ? slen - 1 : 0; - } - return NULL; -} - -/* Zig: an alias (`const db = @import("db.zig");`, `pub const Database = - * zova.Database;`, alias_value = the value text), at module or container - * level, becomes a "Variable" def whose return_types is the QN it names. The - * head resolves lexically: a declaration in an enclosing container, then the - * declaring file's imports (`const v = @import(..)` is imported as `v`, - * `@import("x.zig").C` as `x`); any other head is a path from the declaring - * module. Returns -1 for anything else. */ -static int pxc_build_zig_alias(CBMArena *arena, const CBMFileResult *file, - const CBMDefinition *src, const char *module_qn, CBMLSPDef *dst, - const char **imp_keys, const char **imp_vals, int imp_count) { - const char *text = src->alias_value; - if (!text || !text[0] || !module_qn || strpbrk(text, " \t\r\n")) { - return -1; - } - const char *target = NULL; - if (strncmp(text, "@import(\"", strlen("@import(\"")) == 0) { - const char *path = text + strlen("@import(\""); - const char *close = strchr(path, '"'); - if (!close || close[1] != ')') { - return -1; - } - const char *rest = close + 2; - if ((size_t)(close - path) == strlen("std") && strncmp(path, "std", 3) == 0) { - target = cbm_arena_sprintf(arena, "std%s", rest); /* the opaque std namespace */ - } else { - const char *local = src->name; - size_t len = strlen(local); - if (rest[0]) { - local = path; - for (const char *p = path; p < close; p++) { - if (*p == '/') { - local = p + 1; - } - } - len = (size_t)(close - local); - if (len > 4 && strncmp(local + len - 4, ".zig", 4) == 0) { - len -= 4; - } - } - const char *head = pxc_import_lookup(local, len, imp_keys, imp_vals, imp_count); - target = head ? cbm_arena_sprintf(arena, "%s%s", head, rest) : NULL; - } - } else { - size_t len = strcspn(text, "."); - const char *head = src->parent_class ? pxc_zig_container_head(arena, file, src->parent_class, - module_qn, text, len) - : NULL; - if (!head) { - head = pxc_import_lookup(text, len, imp_keys, imp_vals, imp_count); - } - target = head ? cbm_arena_sprintf(arena, "%s%s", head, text + len) - : cbm_arena_sprintf(arena, "%s.%s", module_qn, text); - } - if (!target) { - return -1; - } - memset(dst, 0, sizeof(*dst)); - dst->qualified_name = src->qualified_name; - dst->short_name = src->name; - dst->label = "Variable"; - dst->def_module_qn = module_qn; - dst->return_types = target; - dst->lang = CBM_LANG_ZIG; - return 0; -} - -/* Zig: a type function (`fn List(comptime T: type) type`) names the - * containers it returns, whose members extraction scopes under its QN. The - * resolver sees it as a type too: a "Struct" def of the same QN (its fields - * fold in like any struct's) whose type_params lists its parameters' - * kinds (cbm_zig_param_kinds of the signature text). */ -static bool pxc_is_zig_type_fn(const CBMDefinition *src) { - return src->label && (strcmp(src->label, "Function") == 0 || strcmp(src->label, "Method") == 0) && - src->return_type && - strcmp(src->return_type, "type") == 0 && src->qualified_name && src->name; -} - -static int pxc_build_zig_type_fn(CBMArena *arena, const CBMDefinition *src, const char *module_qn, - CBMLSPDef *dst) { - const char *out = cbm_zig_param_kinds(arena, src->signature ? src->signature : ""); - if (!out) { - return -1; - } - memset(dst, 0, sizeof(*dst)); - dst->qualified_name = src->qualified_name; - dst->short_name = src->name; - dst->label = "Struct"; - dst->def_module_qn = module_qn; - dst->type_params = out; - dst->lang = CBM_LANG_ZIG; - return 0; -} - -/* Convert one CBMDefinition into a CBMLSPDef. Returns 0 on success, -1 - * to skip (unsupported label or missing required field). dst gets borrowed - * pointers into src and into `arena` for synthesised composites. */ -static int pxc_build_lsp_def(CBMArena *arena, const CBMDefinition *src, const char *module_qn, - const char *namespace_name, CBMLanguage lang, CBMLSPDef *dst, - const cbm_registry_t *reg, const char **imp_keys, - const char **imp_vals, int imp_count) { - const char *label = pxc_map_label(src->label); - if (!label || !src->qualified_name || !src->name) - return -1; - memset(dst, 0, sizeof(*dst)); - if (pxc_is_jvm_lang(lang) && namespace_name && namespace_name[0]) { - dst->qualified_name = pxc_jvm_def_qn(arena, src, namespace_name, label); - dst->receiver_type = pxc_jvm_type_qn(arena, namespace_name, src->parent_class); - } else { - dst->qualified_name = src->qualified_name; - dst->receiver_type = src->parent_class; - } - dst->short_name = src->name; - dst->label = label; - dst->def_module_qn = module_qn; - dst->namespace_name = namespace_name; - dst->is_interface = (strcmp(label, "Interface") == 0 || strcmp(label, "Protocol") == 0); - /* Single return-type string. The per-language registrars split on '|' - * for multi-return languages (Go); single-return languages just see one - * piece, which is what's already stored. */ - dst->return_types = src->return_type; - /* Languages whose cross registrars read embedded_types as QNs get their - * bases resolved against the project registry; everyone else keeps the - * raw source spelling their own registrar already knows how to handle. */ - dst->embedded_types = (reg && pxc_lang_resolves_base_qns(lang)) - ? pxc_join_base_qns(arena, src->base_classes, reg, module_qn, - imp_keys, imp_vals, imp_count) - : pxc_join_pipe(arena, src->base_classes); - dst->lang = lang; - return 0; -} - -/* Go and Zig: fold per-field "Field" definitions into their owning struct's - * field_defs. extract_defs.c emits one flat CBMDefinition per struct field - * (label "Field", parent_class = owning struct QN, name = field name, - * return_type = raw type text). Those rows are dropped by pxc_build_lsp_def - * (pxc_map_label excludes "Field"), so without this fold every Go struct - * registers with zero fields and field-chain calls (h.svc.Handle) can - * never resolve. Fields are always declared in the same file as their struct, - * so scanning the file's own defs covers every case. Runs inside - * cbm_pxc_collect_all_defs — one site covers both the prebuilt-registry path - * and the per-file fallback, since both consume all_defs. */ -static void pxc_fold_go_struct_fields(CBMArena *arena, const CBMFileResult *result, - CBMLSPDef *defs, int start, int end) { - if (!arena || !result || !defs || start >= end) { - return; - } - for (int si = start; si < end; si++) { - CBMLSPDef *dst = &defs[si]; - if (!dst->label || strcmp(dst->label, "Struct") != 0 || !dst->qualified_name) { - continue; - } - int count = 0; - size_t total = 0; /* "name:type" bytes; separators and NUL added below */ - for (int di = 0; di < result->defs.count; di++) { - const CBMDefinition *fd = &result->defs.items[di]; - if (!fd->label || !fd->parent_class || !fd->name || !fd->name[0] || !fd->return_type || - !fd->return_type[0] || strcmp(fd->label, "Field") != 0 || - strcmp(fd->parent_class, dst->qualified_name) != 0) { - continue; - } - total += strlen(fd->name) + 1 + strlen(fd->return_type); - count++; - } - if (count == 0) { - continue; - } - /* count - 1 separators + NUL. */ - size_t bufsz = total + (size_t)(count - 1) + 1; - char *buf = (char *)cbm_arena_alloc(arena, bufsz); - if (!buf) { - continue; - } - char *p = buf; - int written = 0; - for (int di = 0; di < result->defs.count; di++) { - const CBMDefinition *fd = &result->defs.items[di]; - if (!fd->label || !fd->parent_class || !fd->name || !fd->name[0] || !fd->return_type || - !fd->return_type[0] || strcmp(fd->label, "Field") != 0 || - strcmp(fd->parent_class, dst->qualified_name) != 0) { - continue; - } - size_t n = strlen(fd->name); - memcpy(p, fd->name, n); - p += n; - *p++ = ':'; - n = strlen(fd->return_type); - memcpy(p, fd->return_type, n); - p += n; - if (written + 1 < count) { - *p++ = '|'; - } - written++; - } - *p = '\0'; - dst->field_defs = buf; - } -} - -static int pxc_build_import_map(const cbm_gbuf_t *gbuf, const char *project_name, - const char *rel_path, const char ***out_keys, - const char ***out_vals, int *out_count); -static void pxc_free_import_map(const char **keys, const char **vals, int count); - -/* A file's declared namespace, or for Java/Kotlin the one inferred from its - * source-root path, stored back on the result (in its arena) so later readers - * of result->namespace_name see the same value. NULL when there is none. */ -const char *cbm_pxc_apply_inferred_namespace(CBMFileResult *result, const cbm_file_info_t *file) { - const char *namespace_name = result->namespace_name; - if ((!namespace_name || !namespace_name[0]) && file->rel_path) { - namespace_name = pxc_infer_jvm_namespace(&result->arena, file->rel_path, file->language); - if (namespace_name && namespace_name[0]) { - result->namespace_name = namespace_name; - } - } - return namespace_name; -} - -/* Re-home defs[start, end) into `arena`: they borrow strings from a loaded - * (spilled) result that is freed right after collection. def_module_qn is - * borrowed from def_modules[] and stays. */ -static void pxc_own_def_strings(CBMArena *arena, CBMLSPDef *defs, int start, int end) { - for (int i = start; i < end; i++) { - CBMLSPDef *d = &defs[i]; - d->qualified_name = cbm_arena_strdup(arena, d->qualified_name); - d->short_name = cbm_arena_strdup(arena, d->short_name); - d->label = cbm_arena_strdup(arena, d->label); - d->receiver_type = cbm_arena_strdup(arena, d->receiver_type); - d->return_types = cbm_arena_strdup(arena, d->return_types); - d->embedded_types = cbm_arena_strdup(arena, d->embedded_types); - d->field_defs = cbm_arena_strdup(arena, d->field_defs); - d->method_names_str = cbm_arena_strdup(arena, d->method_names_str); - d->namespace_name = cbm_arena_strdup(arena, d->namespace_name); - d->type_params = cbm_arena_strdup(arena, d->type_params); - } -} - -/* Collect a project-wide CBMLSPDef[] from all cached results. Returns a - * malloc'd array (caller frees) of length *out_count. String fields are - * borrowed from cache[i]->arena and from def_modules[i] (also borrowed); - * defs of a spilled result (ctx->spill) are copied into owned_arena, and - * spilled results are skipped when owned_arena is NULL. */ -CBMLSPDef *cbm_pxc_collect_all_defs(const cbm_pipeline_ctx_t *ctx, CBMFileResult **cache, - const cbm_file_info_t *files, int file_count, - const char *project_name, char **def_modules, - CBMArena *owned_arena, int *out_count) { - cbm_result_spill_t *spill = ctx && owned_arena ? ctx->spill : NULL; - int cap = 0; - for (int i = 0; i < file_count; i++) { - if (cache[i]) - cap += cache[i]->defs.count; - } - if (cap == 0 && !spill) { - *out_count = 0; - return NULL; - } - CBMLSPDef *defs = cap > 0 ? (CBMLSPDef *)calloc((size_t)cap, sizeof(CBMLSPDef)) : NULL; - if (cap > 0 && !defs) { - *out_count = 0; - return NULL; - } - int idx = 0; - for (int fi = 0; fi < file_count; fi++) { - const int file_start = idx; - CBMFileResult *result = cbm_result_cache_acquire(cache, spill, fi); - if (!result) - continue; - const bool loaded = result != cache[fi]; - int need = result->defs.count; - for (int di = 0; files[fi].language == CBM_LANG_ZIG && di < result->defs.count; di++) { - need += pxc_is_zig_type_fn(&result->defs.items[di]) ? 1 : 0; - } - if (idx + need > cap) { - int grown = cap * 2 > idx + need ? cap * 2 : idx + need; - CBMLSPDef *more = (CBMLSPDef *)realloc(defs, (size_t)grown * sizeof(CBMLSPDef)); - if (!more) { - cbm_result_cache_release(cache, fi, result); - continue; - } - defs = more; - cap = grown; - } - if (!def_modules[fi]) { - def_modules[fi] = cbm_pipeline_fqn_module_dir(project_name, files[fi].rel_path, - pxc_module_is_dir(files[fi].language)); - } - const char *namespace_name = cbm_pxc_apply_inferred_namespace(result, &files[fi]); - /* One import map per FILE (not per def, and not per base name): the - * cross-file base-class resolution needs the same local-name → - * import-QN view pass_semantic uses. Both drivers have already - * created this file's IMPORTS edges and filled the registry here. - * Built only for the languages that consume resolved base QNs (and - * for Zig, whose aliases resolve through it), and only when a caller - * supplied the pipeline context. */ - const cbm_registry_t *base_reg = NULL; - const char **imp_keys = NULL; - const char **imp_vals = NULL; - int imp_count = 0; - if (ctx && ctx->registry && ctx->gbuf && pxc_lang_resolves_base_qns(files[fi].language)) { - base_reg = ctx->registry; - } - if (ctx && ctx->gbuf && (base_reg || files[fi].language == CBM_LANG_ZIG)) { - pxc_build_import_map(ctx->gbuf, project_name, files[fi].rel_path, &imp_keys, - &imp_vals, &imp_count); - } - for (int di = 0; di < result->defs.count; di++) { - const CBMDefinition *src = &result->defs.items[di]; - int rc = files[fi].language == CBM_LANG_ZIG && src->label && - strcmp(src->label, "Variable") == 0 && src->qualified_name && - src->name - ? pxc_build_zig_alias(&result->arena, result, src, def_modules[fi], - &defs[idx], imp_keys, imp_vals, imp_count) - : pxc_build_lsp_def(&result->arena, src, def_modules[fi], - namespace_name, files[fi].language, &defs[idx], - base_reg, imp_keys, imp_vals, imp_count); - if (rc == 0) { - /* Zig: a function taking `anytype` carries its parameter kinds, - * so callers' argument types reach it (cbm_pxc_collect_zig_args). */ - if (files[fi].language == CBM_LANG_ZIG && src->signature && - strstr(src->signature, "anytype") && - (strcmp(defs[idx].label, "Function") == 0 || - strcmp(defs[idx].label, "Method") == 0)) { - defs[idx].type_params = cbm_zig_param_kinds(&result->arena, src->signature); - } - idx++; - } - if (files[fi].language == CBM_LANG_ZIG && pxc_is_zig_type_fn(src) && - pxc_build_zig_type_fn(&result->arena, src, def_modules[fi], &defs[idx]) == 0) { - idx++; - } - } - pxc_free_import_map(imp_keys, imp_vals, imp_count); /* NULL-safe */ - if (files[fi].language == CBM_LANG_GO || files[fi].language == CBM_LANG_ZIG) { - pxc_fold_go_struct_fields(&result->arena, result, defs, file_start, idx); - } - if (loaded) { - pxc_own_def_strings(owned_arena, defs, file_start, idx); - } - cbm_result_cache_release(cache, fi, result); - } - *out_count = idx; - return defs; -} - -/* Build per-file import map (local_name -> resolved module QN) from gbuf - * IMPORTS edges. Mirrors build_import_map() in pass_parallel.c. Returns 0 - * with *out_count = 0 when the file has no IMPORTS edges. Caller frees keys - * with pxc_free_import_map. */ -static int pxc_build_import_map(const cbm_gbuf_t *gbuf, const char *project_name, - const char *rel_path, const char ***out_keys, - const char ***out_vals, int *out_count) { - *out_keys = NULL; - *out_vals = NULL; - *out_count = 0; - - char *file_qn = cbm_pipeline_fqn_compute(project_name, rel_path, "__file__"); - if (!file_qn) - return 0; - const cbm_gbuf_node_t *file_node = cbm_gbuf_find_by_qn(gbuf, file_qn); - free(file_qn); - if (!file_node) - return 0; - - const cbm_gbuf_edge_t **edges = NULL; - int edge_count = 0; - int rc = - cbm_gbuf_find_edges_by_source_type(gbuf, file_node->id, "IMPORTS", &edges, &edge_count); - if (rc != 0 || edge_count == 0) - return 0; - - const char **keys = (const char **)calloc((size_t)edge_count, sizeof(const char *)); - const char **vals = (const char **)calloc((size_t)edge_count, sizeof(const char *)); - if (!keys || !vals) { - free(keys); - free(vals); - return 0; - } - int count = 0; - for (int i = 0; i < edge_count; i++) { - const cbm_gbuf_edge_t *e = edges[i]; - const cbm_gbuf_node_t *target = cbm_gbuf_find_by_id(gbuf, e->target_id); - if (!target || !e->properties_json) - continue; - const char *start = strstr(e->properties_json, "\"local_name\":\""); - if (!start) - continue; - start += strlen("\"local_name\":\""); - const char *end = strchr(start, '"'); - if (!end || end <= start) - continue; - size_t n = (size_t)(end - start); - char *local = (char *)malloc(n + 1); - if (!local) - continue; - memcpy(local, start, n); - local[n] = '\0'; - keys[count] = local; - vals[count] = target->qualified_name; /* borrowed from gbuf */ - count++; - } - *out_keys = keys; - *out_vals = vals; - *out_count = count; - return 0; -} - -static void pxc_free_import_map(const char **keys, const char **vals, int count) { - if (keys) { - for (int i = 0; i < count; i++) - free((void *)keys[i]); - free((void *)keys); - } - free((void *)vals); /* vals strings borrowed from gbuf — don't free elements */ -} - -/* Detect TS dialect flags from a relative path. */ -void cbm_pxc_ts_modes(CBMLanguage lang, const char *rel_path, bool *out_js, bool *out_jsx, - bool *out_dts) { - *out_js = (lang == CBM_LANG_JAVASCRIPT); - *out_jsx = (lang == CBM_LANG_TSX); - *out_dts = false; - if (!rel_path) - return; - size_t rl = strlen(rel_path); - if (lang == CBM_LANG_JAVASCRIPT && rl >= 4 && strcmp(rel_path + rl - 4, ".jsx") == 0) { - *out_jsx = true; - } - if (lang == CBM_LANG_TYPESCRIPT && rl >= 5 && strcmp(rel_path + rl - 5, ".d.ts") == 0) { - *out_dts = true; - } -} - -/* Returns true when this language has a cross-file LSP wired up. */ -bool cbm_pxc_has_cross_lsp(CBMLanguage lang) { - switch (lang) { - case CBM_LANG_GO: - case CBM_LANG_C: - case CBM_LANG_CPP: - case CBM_LANG_CUDA: - case CBM_LANG_PYTHON: - case CBM_LANG_JAVASCRIPT: - case CBM_LANG_TYPESCRIPT: - case CBM_LANG_TSX: - case CBM_LANG_PHP: - case CBM_LANG_CSHARP: /* tier-2 prebuilt registry path (pass_parallel.c) */ - case CBM_LANG_JAVA: /* fallback cbm_pxc_run_one path */ - case CBM_LANG_KOTLIN: /* fallback cbm_pxc_run_one path */ - case CBM_LANG_RUST: /* fallback cbm_pxc_run_one path (manifest-aware) */ - case CBM_LANG_ZIG: /* fallback cbm_pxc_run_one path */ - return true; - default: - return false; - } -} - -/* Append cross-file results from `src_out` (allocated in a scratch arena - * about to be destroyed) into `dst_calls` (lives in cache_entry->arena), - * copying every string field into dst_arena. Skips entries whose - * (caller_qn, callee_qn) is already present — avoids inflating the array with - * cross-file duplicates of per-file LSP output. - * - * Dedup uses a hash set keyed on "caller\x1fcallee", giving O(1) membership. - * The previous linear strcmp scan made each append O(n), so a file that - * resolved very many cross-calls turned the whole append into O(n^2) and could - * peg a core for minutes (observed: an index hung in pxc_append_results/strcmp). - * The key strings live in a scratch arena that is destroyed after the table. */ -/* Invocation rows dedup on (caller, callee). A reference row is tied to one - * source occurrence, so its site is part of the key and it never collides - * with an invocation of the same callee. An invocation row with a site span - * proves one call site (Zig), so its site is part of the key too. */ -static char *pxc_result_key(CBMArena *keys, const CBMResolvedCall *rc) { - if (rc->kind == CBM_RESOLVED_CALL_REFERENCE) { - return cbm_arena_sprintf(keys, "%s\x1f%s\x1fR%u:%u", rc->caller_qn, rc->callee_qn, - rc->site_start_byte, rc->site_end_byte); - } - if (rc->site_end_byte > rc->site_start_byte) { - return cbm_arena_sprintf(keys, "%s\x1f%s\x1fI%u", rc->caller_qn, rc->callee_qn, - rc->site_start_byte); - } - return cbm_arena_sprintf(keys, "%s\x1f%s", rc->caller_qn, rc->callee_qn); -} - -static void pxc_append_results(CBMArena *dst_arena, CBMResolvedCallArray *dst_calls, - const CBMResolvedCallArray *src_out) { - if (!dst_calls || !src_out) - return; - - CBMArena keys; - cbm_arena_init(&keys); - CBMHashTable *seen = cbm_ht_create((uint32_t)(dst_calls->count + src_out->count + 1)); - - for (int i = 0; i < dst_calls->count; i++) { - const CBMResolvedCall *rc = &dst_calls->items[i]; - if (rc->caller_qn && rc->callee_qn) { - char *k = pxc_result_key(&keys, rc); - if (k) { - cbm_ht_set(seen, k, (void *)1); - } - } - } - - for (int j = 0; j < src_out->count; j++) { - const CBMResolvedCall *src = &src_out->items[j]; - if (!src->caller_qn || !src->callee_qn) - continue; - char *k = pxc_result_key(&keys, src); - if (k && cbm_ht_has(seen, k)) - continue; - if (k) { - cbm_ht_set(seen, k, (void *)1); - } - CBMResolvedCall dst; - memset(&dst, 0, sizeof(dst)); - dst.caller_qn = cbm_arena_strdup(dst_arena, src->caller_qn); - dst.callee_qn = cbm_arena_strdup(dst_arena, src->callee_qn); - dst.strategy = src->strategy ? cbm_arena_strdup(dst_arena, src->strategy) : NULL; - dst.confidence = src->confidence; - dst.reason = src->reason ? cbm_arena_strdup(dst_arena, src->reason) : NULL; - dst.kind = src->kind; - dst.site_start_byte = src->site_start_byte; - dst.site_end_byte = src->site_end_byte; - cbm_resolvedcall_push(dst_calls, dst_arena, dst); - } - - cbm_ht_free(seen); - cbm_arena_destroy(&keys); -} - -/* ── Rust workspace manifest (Cargo.toml) for cross-CRATE resolution ── - * - * cbm_pxc_run_one's signature is shared with the parallel pass - * (pass_parallel.c) and cannot grow a manifest parameter without touching - * that file. We therefore pass the parsed workspace manifest to the Rust - * cross-file resolver through a thread-local borrowed pointer that each - * resolve worker sets from the manifest cbm_pxc_build_rust_manifest read - * once from the project's root Cargo.toml. The manifest's strings are - * owned by the caller's arena; the pointer is borrowed (NULL when - * the project has no Cargo.toml — single-crate / non-workspace projects, - * where in-file resolution needs no workspace metadata). */ -static _Thread_local const CBMCargoManifest *g_pxc_rust_manifest = NULL; - -void cbm_pxc_set_rust_manifest(const CBMCargoManifest *m) { - g_pxc_rust_manifest = m; -} - -const struct CBMCargoManifest *cbm_pxc_get_rust_manifest(void) { - return g_pxc_rust_manifest; -} - -/* Convert a CBMLSPDef array (the pipeline's lingua franca, go_lsp.h:73) - * into a CBMRustLSPDef array (rust_lsp.h) inside `arena`. The two structs - * share their first 9 string fields; CBMRustLSPDef adds `trait_qn` before - * `is_interface` whereas CBMLSPDef has `is_interface` followed by `lang`, - * so a memcpy is unsafe — copy field-by-field. trait_qn is left NULL - * because the pipeline's collect-all-defs step does not carry the - * impl-Trait-for-Type linkage; the resolver still recovers trait dispatch - * from the in-file walk (the cross-file path only needs receiver_type). */ -static CBMRustLSPDef *pxc_lspdefs_to_rust(CBMArena *arena, const CBMLSPDef *defs, int def_count) { - if (!defs || def_count <= 0) - return NULL; - CBMRustLSPDef *out = - (CBMRustLSPDef *)cbm_arena_alloc(arena, (size_t)def_count * sizeof(CBMRustLSPDef)); - if (!out) - return NULL; - for (int i = 0; i < def_count; i++) { - out[i].qualified_name = defs[i].qualified_name; - out[i].short_name = defs[i].short_name; - out[i].label = defs[i].label; - out[i].receiver_type = defs[i].receiver_type; - out[i].def_module_qn = defs[i].def_module_qn; - out[i].return_types = defs[i].return_types; - out[i].embedded_types = defs[i].embedded_types; - out[i].field_defs = defs[i].field_defs; - out[i].method_names_str = defs[i].method_names_str; - out[i].trait_qn = NULL; - out[i].is_interface = defs[i].is_interface; - } - return out; -} - -/* Run cross-file LSP for a single file inside a scratch arena that gets - * freed when the call returns. The LSP would otherwise allocate a fresh - * type registry + stdlib + all project defs into the supplied arena, and - * that adds up to O(N×project_size) memory if we used cache[i]->arena - * directly across N files (test_incremental.c saw 3.5 GB peak on a - * 1100-file repo before this fix). Output gets copied into the file's own - * arena and merged into result->resolved_calls. */ -void cbm_pxc_run_one(CBMLanguage lang, CBMFileResult *r, const char *source, int source_len, - const char *module_qn, CBMLSPDef *defs, int def_count, const char **imp_names, - const char **imp_qns, int imp_count) { - TSTree *tree = r->cached_tree; /* may be NULL — LSP re-parses then */ - - CBMArena scratch; - cbm_arena_init(&scratch); - CBMResolvedCallArray out; - memset(&out, 0, sizeof(out)); - - switch (lang) { - case CBM_LANG_GO: - cbm_run_go_lsp_cross(&scratch, source, source_len, module_qn, defs, def_count, imp_names, - imp_qns, imp_count, tree, &out); - break; - case CBM_LANG_C: - case CBM_LANG_CPP: - case CBM_LANG_CUDA: { - bool cpp_mode = (lang != CBM_LANG_C); - /* C/C++ cross LSP takes include_paths/include_ns_qns instead of - * imports — the existing pipeline doesn't carry C-style include - * resolution as a separate map, so pass NULL/0 and let the LSP - * fall back to its own #include scan. */ - CBMLSPStatus status = - cbm_run_c_lsp_cross_status(&scratch, source, source_len, module_qn, cpp_mode, defs, - def_count, NULL, NULL, 0, tree, &out); - if (status != CBM_LSP_COMPLETE) { - cbm_c_lsp_record_failure(r, status); - } - break; - } - case CBM_LANG_PYTHON: { - CBMLSPStatus status = - cbm_run_py_lsp_cross_status(&scratch, source, source_len, module_qn, defs, def_count, - imp_names, imp_qns, imp_count, tree, &out); - if (status != CBM_LSP_COMPLETE) { - cbm_py_lsp_record_failure(r, status); - } - break; - } - case CBM_LANG_PHP: - cbm_run_php_lsp_cross(&scratch, source, source_len, module_qn, defs, def_count, imp_names, - imp_qns, imp_count, tree, &out); - break; - case CBM_LANG_JAVA: - cbm_run_java_lsp_cross(&scratch, source, source_len, module_qn, defs, def_count, imp_names, - imp_qns, imp_count, tree, &out); - break; - case CBM_LANG_KOTLIN: - cbm_run_kotlin_lsp_cross(&scratch, source, source_len, module_qn, defs, def_count, - imp_names, imp_qns, imp_count, tree, &out); - break; - case CBM_LANG_RUST: { - /* The Rust resolver wants CBMRustLSPDef (rust_lsp.h), not the - * pipeline's CBMLSPDef — the structs share their first 9 fields - * but diverge after, so convert into the scratch arena. The - * workspace manifest (set by each resolve worker) lets - * `crate_a::foo` route across the crate boundary (#56). */ - CBMRustLSPDef *rdefs = pxc_lspdefs_to_rust(&scratch, defs, def_count); - cbm_run_rust_lsp_cross_with_manifest(&scratch, source, source_len, module_qn, rdefs, - def_count, imp_names, imp_qns, imp_count, tree, - g_pxc_rust_manifest, &out); - break; - } - case CBM_LANG_ZIG: - cbm_run_zig_lsp_cross(&scratch, source, source_len, module_qn, defs, def_count, imp_names, - imp_qns, imp_count, tree, &out); - break; - default: - break; - } - - pxc_append_results(&r->arena, &r->resolved_calls, &out); - cbm_arena_destroy(&scratch); -} - -/* cbm_pxc_run_one for Zig with the project's `anytype` argument types (args) - * and, when collect is set, collecting them instead: the file's resolved - * calls are then discarded. */ -static void pxc_run_one_zig(CBMFileResult *r, const char *source, int source_len, - const char *module_qn, CBMLSPDef *defs, int def_count, - const char **imp_names, const char **imp_qns, int imp_count, - const CBMZigArgType *args, int arg_count, - CBMZigArgTypeArray *collect) { - CBMArena scratch; - cbm_arena_init(&scratch); - CBMResolvedCallArray out; - memset(&out, 0, sizeof(out)); - cbm_run_zig_lsp_cross_ex(&scratch, source, source_len, module_qn, defs, def_count, imp_names, - imp_qns, imp_count, r->cached_tree, &out, args, arg_count, collect); - if (!collect) { - pxc_append_results(&r->arena, &r->resolved_calls, &out); - } - cbm_arena_destroy(&scratch); -} - -/* Variant of cbm_pxc_run_one for TS/JS/JSX/TSX with explicit dialect - * flags. Same scratch-arena lifecycle as cbm_pxc_run_one. */ -void cbm_pxc_run_one_ts(CBMFileResult *r, const char *source, int source_len, const char *module_qn, - CBMLSPDef *defs, int def_count, const char **imp_names, - const char **imp_qns, int imp_count, bool js_mode, bool jsx_mode, - bool dts_mode) { - CBMArena scratch; - cbm_arena_init(&scratch); - CBMResolvedCallArray out; - memset(&out, 0, sizeof(out)); - - cbm_run_ts_lsp_cross(&scratch, source, source_len, module_qn, js_mode, jsx_mode, dts_mode, defs, - def_count, imp_names, imp_qns, imp_count, r->cached_tree, &out); - - pxc_append_results(&r->arena, &r->resolved_calls, &out); - cbm_arena_destroy(&scratch); -} - -/* Parse the project's root Cargo.toml (if present) into `out_m`, using - * `marena` for the manifest's owned strings. Returns true when a manifest - * was parsed (a workspace root or any [package]/[dependencies]); false when - * there is no readable Cargo.toml, leaving *out_m untouched. The resulting - * manifest feeds cross-CRATE Rust resolution (#56): its [workspace].members - * map lets `crate_a::foo` route to the member crate's def. */ -/* Per-file cross-LSP dispatch for the parallel resolve worker: filter the global defs - * down to the file's own+imported modules via the module-def index, resolve - * through the shared prebuilt registry when the language has one (per-file - * OVERLAY pattern — no registry build, no finalize), and only fall back to - * the per-file registry build (with the FILTERED defs) for languages without - * a shared-registry variant. Before this helper existed the sequential - * driver fed the FULL def list into full per-file registry builds — - * O(files x defs), which ground an 81k-file TS corpus for hours. - * - * `rust_shared_get` supplies the lazily-built shared Rust all-defs registry - * (the parallel resolver owns its once-guard); NULL means "no shared rust - * registry available" and rust NULL-filter files take the per-file build. */ -/* Per-file registry-build cost counters (upstream cffb4e37), logged by - * pass_parallel at the end of resolve. */ -static _Atomic uint64_t g_pxc_defs_registered = 0; -static _Atomic uint64_t g_pxc_build_files = 0; -static _Atomic uint64_t g_pxc_filter_files = 0; -static _Atomic uint64_t g_pxc_filter_failed = 0; - -void cbm_pxc_filter_stats(uint64_t *defs_registered, uint64_t *build_files, uint64_t *filter_files, - uint64_t *filter_failed) { - *defs_registered = atomic_load_explicit(&g_pxc_defs_registered, memory_order_relaxed); - *build_files = atomic_load_explicit(&g_pxc_build_files, memory_order_relaxed); - *filter_files = atomic_load_explicit(&g_pxc_filter_files, memory_order_relaxed); - *filter_failed = atomic_load_explicit(&g_pxc_filter_failed, memory_order_relaxed); -} - -void cbm_pxc_filter_stats_reset(void) { - atomic_store_explicit(&g_pxc_defs_registered, 0, memory_order_relaxed); - atomic_store_explicit(&g_pxc_build_files, 0, memory_order_relaxed); - atomic_store_explicit(&g_pxc_filter_files, 0, memory_order_relaxed); - atomic_store_explicit(&g_pxc_filter_failed, 0, memory_order_relaxed); -} - -/* imp_vals plus every dotted prefix of each QN in the types bound to - * module_qn's `anytype` parameters (in arena): the modules among them hold - * the defs a bound walk resolves against. */ -static const char **pxc_zig_arg_modules(CBMArena *arena, const CBMCrossLspRegistries *r, - const char *module_qn, const char **imp_vals, - int imp_count, int *out_count) { - size_t mlen = strlen(module_qn); - int cap = imp_count; - for (int i = 0; i < r->zig_arg_count; i++) { - const char *q = r->zig_args[i].callee_qn; - if (strncmp(q, module_qn, mlen) == 0 && q[mlen] == '.') { - for (const char *t = r->zig_args[i].type; *t; t++) { - cap += *t == '.' ? 1 : 0; - } - } - } - const char **out = (const char **)cbm_arena_alloc(arena, (size_t)(cap + 1) * sizeof(char *)); - if (!out) { - *out_count = imp_count; - return imp_vals; - } - int n = 0; - for (int i = 0; i < imp_count; i++) { - out[n++] = imp_vals[i]; - } - for (int i = 0; i < r->zig_arg_count; i++) { - const char *q = r->zig_args[i].callee_qn; - if (strncmp(q, module_qn, mlen) != 0 || q[mlen] != '.') { - continue; - } - const char *t = r->zig_args[i].type; - size_t start = 0; - for (size_t j = 0;; j++) { - char ch = t[j]; - bool qn_char = ch && !strchr("[]()*?, ", ch); - if (ch == '.' && j > start && n < cap) { - out[n++] = cbm_arena_strndup(arena, t + start, j - start); - } - if (!qn_char) { - start = j + 1; - } - if (!ch) { - break; - } - } - } - *out_count = n; - return out; -} - -void cbm_pxc_dispatch_file(CBMLanguage lang, CBMFileResult *result, const char *source, - int source_len, const char *rel, const char *def_module, - const CBMCrossLspRegistries *cross_registries, - const CBMModuleDefIndex *module_def_index, CBMLSPDef *all_defs, - int all_def_count, const char **imp_keys, const char **imp_vals, - int imp_count, CBMTypeRegistry *(*rust_shared_get)(void *), - void *rust_shared_ctx) { - if (!result) { - return; - } - bool used_prebuilt = false; - CBMTypeRegistry *prebuilt = - cross_registries ? cbm_pxc_registry_for_lang(cross_registries, lang) : NULL; - if (prebuilt) { - switch (lang) { - case CBM_LANG_GO: - /* Tier 3 (metadata-driven): pure lookup over the Tier-1 - * lsp_unresolved entries — no parse, no AST walk. Then the - * AST walk on the shared Tier-2 registry (mirroring every - * other language) so NAMED receivers evaluated against - * project-wide defs also resolve. The walk variant below is - * read-only — the sealed registry is safe for parallel - * workers. */ - cbm_go_fast_resolve_qualified_calls(result, prebuilt, imp_keys, imp_vals, imp_count); - cbm_run_go_lsp_cross_with_registry(&result->arena, source, source_len, def_module, - prebuilt, imp_keys, imp_vals, imp_count, - result->cached_tree, &result->resolved_calls); - used_prebuilt = true; - break; - case CBM_LANG_PYTHON: { - CBMLSPStatus status = cbm_run_py_lsp_cross_with_registry_status( - &result->arena, source, source_len, def_module, prebuilt, imp_keys, imp_vals, - imp_count, result->cached_tree, &result->resolved_calls); - if (status != CBM_LSP_COMPLETE) { - cbm_py_lsp_record_failure(result, status); - } - used_prebuilt = true; - break; - } - case CBM_LANG_C: - case CBM_LANG_CPP: - case CBM_LANG_CUDA: { - CBMLSPStatus status = cbm_run_c_lsp_cross_with_registry_status( - &result->arena, source, source_len, def_module, (lang != CBM_LANG_C), prebuilt, - imp_keys, imp_vals, imp_count, result->cached_tree, &result->resolved_calls); - if (status != CBM_LSP_COMPLETE) { - cbm_c_lsp_record_failure(result, status); - } - used_prebuilt = true; - break; - } - case CBM_LANG_CSHARP: - cbm_run_cs_lsp_cross_with_registry(&result->arena, source, source_len, def_module, - prebuilt, imp_vals, imp_count, result->cached_tree, - &result->resolved_calls); - used_prebuilt = true; - break; - case CBM_LANG_JAVASCRIPT: - case CBM_LANG_TYPESCRIPT: - case CBM_LANG_TSX: { - /* TS: per-file OVERLAY chained to the shared base. Filter to - * own+imports so the overlay builder can pick out own-module - * defs without scanning the whole project. */ - bool js; - bool jsx; - bool dts; - cbm_pxc_ts_modes(lang, rel, &js, &jsx, &dts); - CBMLSPDef *ts_defs = all_defs; - int ts_def_count = all_def_count; - CBMLSPDef *ts_filtered = NULL; - if (module_def_index) { - int fc = 0; - ts_filtered = cbm_pxc_filter_defs_for_file(module_def_index, all_defs, lang, - result->namespace_name, def_module, - imp_vals, imp_count, &fc); - if (ts_filtered) { - ts_defs = ts_filtered; - ts_def_count = fc; - } - } - cbm_run_ts_lsp_cross_with_registry(&result->arena, source, source_len, def_module, js, - jsx, dts, prebuilt, ts_defs, ts_def_count, imp_keys, - imp_vals, imp_count, result->cached_tree, - &result->resolved_calls); - free(ts_filtered); - used_prebuilt = true; - break; - } - /* PHP falls through to the per-file build path below until its - * overlay variant lands. */ - default: - break; - } - } - - if (used_prebuilt) { - return; - } - /* Fallback: gopls per-file filter + per-file registry build. RUST is - * exempt from the module filter: its resolution is Cargo-manifest-aware - * and a `crate_a::foo` reference routes to defs in ANOTHER workspace - * crate — a module that is in neither own_module nor the import map, so - * the filter starves cross-crate resolution (#56 repro red). Rust - * therefore always resolves against the FULL def universe: the lazily - * built shared registry when available, else a full per-file build. */ - CBMLSPDef *filtered = NULL; - CBMLSPDef *file_defs = all_defs; - int file_def_count = all_def_count; - /* Zig: a function walked with an `anytype` parameter bound needs the defs - * of the bound types' modules too, which it need not import. */ - const char **filter_qns = imp_vals; - int filter_count = imp_count; - CBMArena zig_arena; - bool zig_arena_live = false; - if (lang == CBM_LANG_ZIG && cbm_pxc_zig_args_in_module(cross_registries, def_module)) { - cbm_arena_init(&zig_arena); - zig_arena_live = true; - filter_qns = pxc_zig_arg_modules(&zig_arena, cross_registries, def_module, imp_vals, - imp_count, &filter_count); - } - if (module_def_index && lang != CBM_LANG_RUST) { - int filtered_count = 0; - filtered = - cbm_pxc_filter_defs_for_file(module_def_index, all_defs, lang, result->namespace_name, - def_module, filter_qns, filter_count, &filtered_count); - if (filtered) { - file_defs = filtered; - file_def_count = filtered_count; - } - atomic_fetch_add_explicit(&g_pxc_filter_files, 1, memory_order_relaxed); - if (!filtered) { - atomic_fetch_add_explicit(&g_pxc_filter_failed, 1, memory_order_relaxed); - } - } - /* Per-file registry build cost is driven by this count. */ - atomic_fetch_add_explicit(&g_pxc_defs_registered, (uint64_t)file_def_count, - memory_order_relaxed); - atomic_fetch_add_explicit(&g_pxc_build_files, 1, memory_order_relaxed); - if (lang == CBM_LANG_RUST) { - CBMTypeRegistry *shared = rust_shared_get ? rust_shared_get(rust_shared_ctx) : NULL; - if (shared) { - cbm_run_rust_lsp_cross_with_registry(&result->arena, source, source_len, def_module, - shared, imp_keys, imp_vals, imp_count, - result->cached_tree, cbm_pxc_get_rust_manifest(), - &result->resolved_calls, - /*result=*/NULL); - } else { - cbm_pxc_run_one(lang, result, source, source_len, def_module, file_defs, file_def_count, - imp_keys, imp_vals, imp_count); - } - } else if (lang == CBM_LANG_JAVASCRIPT || lang == CBM_LANG_TYPESCRIPT || lang == CBM_LANG_TSX) { - bool js; - bool jsx; - bool dts; - cbm_pxc_ts_modes(lang, rel, &js, &jsx, &dts); - cbm_pxc_run_one_ts(result, source, source_len, def_module, file_defs, file_def_count, - imp_keys, imp_vals, imp_count, js, jsx, dts); - } else if (lang == CBM_LANG_ZIG) { - pxc_run_one_zig(result, source, source_len, def_module, file_defs, file_def_count, - imp_keys, imp_vals, imp_count, - cross_registries ? cross_registries->zig_args : NULL, - cross_registries ? cross_registries->zig_arg_count : 0, NULL); - } else { - cbm_pxc_run_one(lang, result, source, source_len, def_module, file_defs, file_def_count, - imp_keys, imp_vals, imp_count); - } - free(filtered); - if (zig_arena_live) { - cbm_arena_destroy(&zig_arena); - } -} - -static int pxc_zig_arg_cmp(const void *a, const void *b) { - const CBMZigArgType *x = (const CBMZigArgType *)a; - const CBMZigArgType *y = (const CBMZigArgType *)b; - int c = strcmp(x->callee_qn, y->callee_qn); - if (c == 0) { - c = x->param - y->param; - } - return c != 0 ? c : strcmp(x->type, y->type); -} - -CBMZigArgType *cbm_pxc_collect_zig_args(const cbm_pipeline_ctx_t *ctx, CBMFileResult **cache, - const cbm_file_info_t *files, int file_count, - char *const *def_modules, - const CBMModuleDefIndex *module_def_index, - CBMLSPDef *all_defs, int def_count, CBMArena *arena, - int *out_count) { - *out_count = 0; - bool any = false; - for (int i = 0; i < def_count && !any; i++) { - any = all_defs[i].lang == CBM_LANG_ZIG && all_defs[i].type_params && - strchr(all_defs[i].type_params, '*') && all_defs[i].label && - (strcmp(all_defs[i].label, "Function") == 0 || - strcmp(all_defs[i].label, "Method") == 0); - } - if (!any || !ctx || !cache || !def_modules) { - return NULL; - } - CBMZigArgTypeArray collect = {NULL, 0, 0, arena}; - for (int fi = 0; fi < file_count; fi++) { - if (files[fi].language != CBM_LANG_ZIG || !def_modules[fi]) { - continue; - } - CBMFileResult *result = cbm_result_cache_acquire(cache, ctx->spill, fi); - if (!result) { - continue; - } - char *owned = NULL; - const char *source = result->source; - int source_len = result->source_len; - if ((!source || source_len <= 0) && result->calls.count > 0) { - owned = pxc_read_file(files[fi].path, &source_len); - source = owned; - } - if (source && source_len > 0 && result->calls.count > 0) { - const char **imp_keys = NULL; - const char **imp_vals = NULL; - int imp_count = 0; - pxc_build_import_map(ctx->gbuf, ctx->project_name, files[fi].rel_path, &imp_keys, - &imp_vals, &imp_count); - int fc = 0; - CBMLSPDef *filtered = - module_def_index ? cbm_pxc_filter_defs_for_file( - module_def_index, all_defs, CBM_LANG_ZIG, - result->namespace_name, def_modules[fi], imp_vals, - imp_count, &fc) - : NULL; - pxc_run_one_zig(result, source, source_len, def_modules[fi], - filtered ? filtered : all_defs, filtered ? fc : def_count, imp_keys, - imp_vals, imp_count, NULL, 0, &collect); - free(filtered); - pxc_free_import_map(imp_keys, imp_vals, imp_count); - } - free(owned); - cbm_result_cache_release(cache, fi, result); - } - if (collect.count == 0) { - free(collect.items); - return NULL; - } - qsort(collect.items, (size_t)collect.count, sizeof(CBMZigArgType), pxc_zig_arg_cmp); - int n = 1; - for (int i = 1; i < collect.count; i++) { - if (pxc_zig_arg_cmp(&collect.items[i], &collect.items[n - 1]) != 0) { - collect.items[n++] = collect.items[i]; - } - } - cbm_log_info("lsp_cross.zig_anytype", "arg_types", cbm_log_itoa(n)); - *out_count = n; - return collect.items; -} - -bool cbm_pxc_zig_args_in_module(const CBMCrossLspRegistries *r, const char *module_qn) { - size_t n = module_qn ? strlen(module_qn) : 0; - for (int i = 0; r && n > 0 && i < r->zig_arg_count; i++) { - const char *q = r->zig_args[i].callee_qn; - if (strncmp(q, module_qn, n) == 0 && q[n] == '.') { - return true; - } - } - return false; -} - -bool cbm_pxc_build_rust_manifest(const cbm_pipeline_ctx_t *ctx, CBMArena *marena, - CBMCargoManifest *out_m) { - if (!ctx || !ctx->repo_path || !marena || !out_m) - return false; - char path[1024]; - int n = snprintf(path, sizeof(path), "%s/Cargo.toml", ctx->repo_path); - if (n <= 0 || (size_t)n >= sizeof(path)) - return false; - int toml_len = 0; - char *toml = pxc_read_file(path, &toml_len); - if (!toml || toml_len <= 0) { - free(toml); - return false; - } - memset(out_m, 0, sizeof(*out_m)); - cbm_cargo_parse(marena, toml, toml_len, out_m); - free(toml); /* cargo parser copies into marena */ - return true; -} - -/* ── Per-module def index (gopls "package summary" pattern) ──── */ - -typedef struct { - int count; - int cap; - int *indices; /* malloc'd; indices into the caller's all_defs[] */ -} pxc_module_entry_t; - -struct CBMModuleDefIndex { - CBMHashTable *ht; /* module_qn → pxc_module_entry_t* */ - CBMHashTable *namespace_ht; /* declared package/namespace → pxc_module_entry_t* */ - int def_count; /* total entries in the all_defs[] array */ -}; - -/* cbm_ht_foreach callback: free each pxc_module_entry_t. */ -static void pxc_module_entry_free_cb(const char *key, void *value, void *userdata) { - (void)key; - (void)userdata; - pxc_module_entry_t *e = (pxc_module_entry_t *)value; - if (!e) - return; - free(e->indices); - free(e); -} -static pxc_module_entry_t *pxc_module_entry_get_or_create(CBMHashTable *ht, const char *key) { - if (!ht || !key || !key[0]) { - return NULL; - } - pxc_module_entry_t *e = (pxc_module_entry_t *)cbm_ht_get(ht, key); - if (e) { - return e; - } - e = (pxc_module_entry_t *)calloc(1, sizeof(*e)); - if (!e) { - return NULL; - } - e->cap = 8; - e->indices = (int *)calloc((size_t)e->cap, sizeof(*e->indices)); - if (!e->indices) { - free(e); - return NULL; - } - cbm_ht_set(ht, key, e); - return e; -} - -static void pxc_module_entry_add_index(pxc_module_entry_t *e, int index) { - if (!e) { - return; - } - if (e->count >= e->cap) { - int new_cap = e->cap * 2; - int *new_indices = (int *)realloc(e->indices, (size_t)new_cap * sizeof(*new_indices)); - if (!new_indices) { - return; - } - e->indices = new_indices; - e->cap = new_cap; - } - e->indices[e->count++] = index; -} - -static bool pxc_is_jvm_lang(CBMLanguage lang); -static bool pxc_def_lang_matches(CBMLanguage caller_lang, CBMLanguage def_lang); - -static int pxc_mark_entry_defs(bool *selected, const pxc_module_entry_t *e, - const CBMLSPDef *all_defs, CBMLanguage caller_lang) { - if (!selected || !e) { - return 0; - } - int added = 0; - for (int j = 0; j < e->count; j++) { - int idx = e->indices[j]; - const CBMLSPDef *def = &all_defs[idx]; - if (!pxc_def_lang_matches(caller_lang, def->lang) || selected[idx]) { - continue; - } - selected[idx] = true; - added++; - } - return added; -} - -static bool pxc_is_jvm_lang(CBMLanguage lang) { - return lang == CBM_LANG_JAVA || lang == CBM_LANG_KOTLIN; -} - -static bool pxc_def_lang_matches(CBMLanguage caller_lang, CBMLanguage def_lang) { - if (pxc_is_jvm_lang(caller_lang)) { - return pxc_is_jvm_lang(def_lang); - } - return true; -} - -static void pxc_mark_module_defs(const CBMModuleDefIndex *idx, bool *selected, - const CBMLSPDef *all_defs, CBMLanguage caller_lang, - const char *module_qn, int *total) { - if (!idx || !idx->ht || !module_qn || !module_qn[0]) { - return; - } - pxc_module_entry_t *e = (pxc_module_entry_t *)cbm_ht_get(idx->ht, module_qn); - int added = pxc_mark_entry_defs(selected, e, all_defs, caller_lang); - if (total) { - *total += added; - } -} - -/* Zig: an alias names a module other than the file's imports (a re-export - * `pub const Database = zova.Database;`, or `const sqlite = @import(..)` that a - * return type text spells), so a Zig file also gets the defs of each module a - * selected alias names, transitively. Returns the number of defs added. */ -enum { PXC_ZIG_ALIAS_ROUNDS = 8 }; -static int pxc_mark_zig_alias_modules(const CBMModuleDefIndex *idx, bool *selected, - const CBMLSPDef *all_defs) { - int added_total = 0; - for (int round = 0; round < PXC_ZIG_ALIAS_ROUNDS; round++) { - int added = 0; - for (int i = 0; i < idx->def_count; i++) { - const CBMLSPDef *d = &all_defs[i]; - if (!selected[i] || d->lang != CBM_LANG_ZIG || !d->return_types || !d->label || - strcmp(d->label, "Variable") != 0) { - continue; - } - char buf[CBM_SZ_512]; - snprintf(buf, sizeof(buf), "%s", d->return_types); - for (char *dot = buf; dot;) { - pxc_module_entry_t *e = (pxc_module_entry_t *)cbm_ht_get(idx->ht, buf); - if (e) { - added += pxc_mark_entry_defs(selected, e, all_defs, CBM_LANG_ZIG); - break; - } - dot = strrchr(buf, '.'); - if (dot) { - *dot = '\0'; - } - } - } - added_total += added; - if (added == 0) { - break; - } - } - return added_total; -} - -CBMModuleDefIndex *cbm_pxc_build_module_def_index(CBMLSPDef *all_defs, int def_count) { - if (!all_defs || def_count <= 0) { - return NULL; - } - - CBMHashTable *ht = cbm_ht_create(64); - CBMHashTable *namespace_ht = cbm_ht_create(64); - if (!ht || !namespace_ht) { - cbm_ht_free(ht); - cbm_ht_free(namespace_ht); - return NULL; - } - - /* Single pass: index each def by file module and by declared package. - * JVM mixed roots (`src/main/java` + `src/main/kotlin`) share the - * declared package, not the path-derived module prefix. */ - for (int i = 0; i < def_count; i++) { - pxc_module_entry_add_index(pxc_module_entry_get_or_create(ht, all_defs[i].def_module_qn), - i); - pxc_module_entry_add_index( - pxc_module_entry_get_or_create(namespace_ht, all_defs[i].namespace_name), i); - } - - CBMModuleDefIndex *idx = (CBMModuleDefIndex *)calloc(1, sizeof(*idx)); - if (!idx) { - cbm_ht_foreach(ht, pxc_module_entry_free_cb, NULL); - cbm_ht_free(ht); - cbm_ht_foreach(namespace_ht, pxc_module_entry_free_cb, NULL); - cbm_ht_free(namespace_ht); - return NULL; - } - idx->ht = ht; - idx->namespace_ht = namespace_ht; - idx->def_count = def_count; - return idx; -} - -void cbm_pxc_free_module_def_index(CBMModuleDefIndex *idx) { - if (!idx) { - return; - } - if (idx->ht) { - cbm_ht_foreach(idx->ht, pxc_module_entry_free_cb, NULL); - cbm_ht_free(idx->ht); - } - if (idx->namespace_ht) { - cbm_ht_foreach(idx->namespace_ht, pxc_module_entry_free_cb, NULL); - cbm_ht_free(idx->namespace_ht); - } - free(idx); -} - -CBMLSPDef *cbm_pxc_filter_defs_for_file(const CBMModuleDefIndex *idx, CBMLSPDef *all_defs, - CBMLanguage caller_lang, const char *caller_namespace, - const char *own_module, const char *const *imp_qns, - int imp_count, int *out_count) { - if (out_count) { - *out_count = 0; - } - if (!idx || !idx->ht || !all_defs || !out_count || idx->def_count <= 0) { - return NULL; - } - - bool *selected = (bool *)calloc((size_t)idx->def_count, sizeof(*selected)); - if (!selected) { - return NULL; - } - - int total = 0; - pxc_mark_module_defs(idx, selected, all_defs, caller_lang, own_module, &total); - for (int i = 0; i < imp_count; i++) { - pxc_mark_module_defs(idx, selected, all_defs, caller_lang, imp_qns[i], &total); - } - if (caller_lang == CBM_LANG_ZIG) { - total += pxc_mark_zig_alias_modules(idx, selected, all_defs); - } - if (pxc_is_jvm_lang(caller_lang) && caller_namespace && caller_namespace[0] && - idx->namespace_ht) { - pxc_module_entry_t *e = - (pxc_module_entry_t *)cbm_ht_get(idx->namespace_ht, caller_namespace); - total += pxc_mark_entry_defs(selected, e, all_defs, caller_lang); - } - - if (total == 0) { - free(selected); - return NULL; - } - - CBMLSPDef *out = (CBMLSPDef *)malloc((size_t)total * sizeof(CBMLSPDef)); - if (!out) { - free(selected); - return NULL; - } - - int n = 0; - for (int i = 0; i < idx->def_count; i++) { - if (selected[i]) { - out[n++] = all_defs[i]; - } - } - *out_count = n; - free(selected); - return out; -} diff --git a/src/pipeline/pass_lsp_cross.zig b/src/pipeline/pass_lsp_cross.zig new file mode 100644 index 00000000..6d179b80 --- /dev/null +++ b/src/pipeline/pass_lsp_cross.zig @@ -0,0 +1,1567 @@ +//! pass_lsp_cross.zig — Cross-file LSP type-aware call resolution pass. +//! +//! See pass_lsp_cross.h for the high-level contract. This file is the +//! pipeline glue that converts the existing per-file extraction state +//! (CBMDefinition / CBMImport / IMPORTS-edge gbuf state) into the input +//! shape each language LSP's cbm_run_X_lsp_cross expects, then merges +//! the resulting CBMResolvedCall entries back into per-file results. +//! +//! The pass is a no-op for any file whose CBMFileResult is missing or +//! whose language has no cross-file LSP entry registered (e.g. Rust / +//! Java today). Per-LSP emit functions dedup against entries already in +//! resolved_calls, so this pass is also idempotent — safe to invoke +//! multiple times if the pipeline gains a re-run path later. + +const std = @import("std"); +const c = @import("c"); +const interop = @import("../foundation/interop.zig"); +const util = @import("util.zig"); + +const CStr = util.CStr; +const MutStr = util.MutStr; +const ix = util.ix; +const streq = util.streq; +const ptrDiff = util.ptrDiff; +const HashTable = c.CBMHashTable; +const Arena = c.CBMArena; +const Gbuf = c.cbm_gbuf_t; +const Node = c.cbm_gbuf_node_t; +const Edge = c.cbm_gbuf_edge_t; +const Ctx = c.cbm_pipeline_ctx_t; +const FileInfo = c.cbm_file_info_t; +const FileResult = c.CBMFileResult; +const Definition = c.CBMDefinition; +const LspDef = c.CBMLSPDef; +const Registry = c.cbm_registry_t; +const Language = c.CBMLanguage; +/// A NULL-terminated or counted array of borrowed C strings. +const StrList = [*c]CStr; + +// ── Constants ─────────────────────────────────────────────────────── + +/// source size cap, in MiB +const PXC_MAX_FILE_BYTES_FACTOR = 100; + +// ── Local helpers ─────────────────────────────────────────────────── + +/// strncmp(s, prefix, strlen(prefix)) == 0. +fn hasPrefix(s: CStr, comptime prefix: [:0]const u8) bool { + return c.strncmp(s, prefix.ptr, prefix.len) == 0; +} + +/// True for languages whose module QN is derived from the CONTAINING DIRECTORY +/// (Java package, Go package) rather than the filename stem. MUST match the +/// extraction-side cbm_lang_module_is_dir() in internal/cbm/helpers.c so the +/// cross-file LSP caller_qn agrees with the def-node QN (the lsp_resolve join +/// keys on exact equality). +fn pxcModuleIsDir(lang: Language) bool { + return lang == c.CBM_LANG_JAVA or lang == c.CBM_LANG_GO; +} + +/// Slurp a file into a malloc'd, NUL-terminated buffer. Mirrors the +/// read_file helper in pass_parallel (kept local so the +/// pipeline doesn't grow a public read-file API just for this pass). +fn pxcReadFile(path: CStr, out_len: *c_int) MutStr { + const f = c.cbm_fopen(path, "rb") orelse return null; + _ = c.fseek(f, 0, c.SEEK_END); + const size = c.ftell(f); + _ = c.fseek(f, 0, c.SEEK_SET); + if (size <= 0 or + size > @as(c_long, PXC_MAX_FILE_BYTES_FACTOR) * @as(c_long, c.CBM_SZ_1K) * @as(c_long, c.CBM_SZ_1K)) + { + _ = c.fclose(f); + return null; + } + // +pad: tree-sitter lexer lookahead reads past EOF; keep it in-bounds + const CBM_TS_LOOKAHEAD_PAD = 16; + const usize_size: usize = @intCast(size); + const buf: MutStr = @ptrCast(c.malloc(usize_size + CBM_TS_LOOKAHEAD_PAD)); + if (buf == null) { + _ = c.fclose(f); + return null; + } + var nread = c.fread(buf, 1, usize_size, f); + _ = c.fclose(f); + if (nread > usize_size) nread = usize_size; + @memset(buf[nread .. nread + CBM_TS_LOOKAHEAD_PAD], 0); + out_len.* = @intCast(nread); + return buf; +} + +/// Map a CBMDefinition.label to a CBMLSPDef.label. Per-language LSP registrars +/// only care about type-like containers (Class/Struct/Interface/Trait/Enum/Type) +/// plus Protocol/Function/Method — variables, modules, decorators, etc. are +/// skipped. Struct passes through so Rust/Go struct type-registration via the +/// cross-file LSP path is not dropped. +fn pxcMapLabel(label: CStr) CStr { + if (label == null) return null; + if (c.cbm_label_is_type_like(label) or streq(label, "Protocol") or + streq(label, "Function") or streq(label, "Method")) + { + return label; + } + return null; +} + +/// Build the embedded_types "|"-separated string from base_classes[]. +/// Returns NULL when there are no bases. Allocated in the supplied arena. +fn pxcJoinPipe(arena: *Arena, items: [*c]const CStr) CStr { + if (items == null or items[0] == null) return null; + var count: usize = 0; + var total: usize = 0; + while (items[count] != null) : (count += 1) { + total += c.strlen(items[count]); + } + if (count == 0) return null; + // count - 1 separators + NUL. + const bufsz = total + (count - 1) + 1; + const buf: MutStr = @ptrCast(c.cbm_arena_alloc(arena, bufsz)); + if (buf == null) return null; + var p: usize = 0; + for (0..count) |i| { + const n = c.strlen(items[i]); + @memcpy(buf[p .. p + n], items[i][0..n]); + p += n; + if (i + 1 < count) { + buf[p] = '|'; + p += 1; + } + } + buf[p] = 0; + return buf; +} + +// ── Cross-file base-class QN resolution ───────────────────────────── +// +// CBMDefinition.base_classes carries the SOURCE SPELLING of each base +// ("Base", "django.db.Model", "React.Component"): extraction strips +// keywords and generic arguments, but it cannot know WHERE the name is +// declared. The Python and TS cross-file registrars, however, consume +// CBMLSPDef.embedded_types as fully-qualified names — py_lookup_attribute +// and ts_lookup_member feed each entry straight into +// cbm_registry_lookup_type. An unqualified spelling therefore matched +// nothing declared in ANOTHER file: `class Child(Base)` in child.py never +// saw Base in base.py, so a call to an inherited method through a typed +// receiver had no member to find and fell through to the weak textual +// cascade (where the #592/#606 guard correctly kills it). +// +// Resolve each base name ONCE per definition here, from exactly the +// inputs pass_semantic uses to draw its INHERITS edge: the project +// registry, the declaring module, and the file's import map. Two +// properties follow. The LSP's inheritance view is the same relation the +// graph records, so the two cannot diverge. And the binding is +// import- or same-module-backed, not a short-name guess, so the CALLS +// edge it enables is a supported fact that the weak-member guard keeps. +// +// Cost: O(defs) hash lookups + ONE import map per file. No per-call-site +// hierarchy walk, no registry scan, no per-file registry rebuild. +fn pxcLangResolvesBaseQns(lang: Language) bool { + return switch (lang) { + c.CBM_LANG_PYTHON, c.CBM_LANG_JAVASCRIPT, c.CBM_LANG_TYPESCRIPT, c.CBM_LANG_TSX => true, + // Go / JVM / C# / C++ / Rust registrars qualify their own embedded + // types already (struct embedding, parent_class chains, impl-trait + // provenance). Re-resolving here would fight those paths, so they + // keep the raw join. + else => false, + }; +} + +/// True for the registry strategies that are pure short-name guesses. +/// EXPLICIT drop-list, mirroring cbm_tsjs_suppress_weak_method_match: a +/// base class bound by "some project type happens to share this name" is +/// exactly the fabricated relation #606 removed, and inheritance +/// multiplies it — every inherited member of the wrong base would become +/// a callable target. Import-, module- and suffix-aware strategies are +/// kept; anything unresolved simply retains its source spelling and the +/// behaviour that predates this resolution. +fn pxcBaseStrategyIsWeak(strategy: CStr) bool { + if (strategy == null or strategy[0] == 0) return true; + return streq(strategy, "suffix_match") or streq(strategy, "unique_name") or + streq(strategy, "field_type_hint") or streq(strategy, "fuzzy"); +} + +/// Resolve one base-class spelling to a project QN. Mirrors +/// pass_semantic's resolve_as_class — same registry, same type-like veto — +/// then additionally rejects weak short-name strategies (see above). +/// Returns NULL when the base is not a confidently-known project type; +/// stdlib and third-party bases land here and keep their raw spelling. +fn pxcResolveBaseQn(reg: ?*const Registry, raw: CStr, module_qn: CStr, imp_keys: StrList, imp_vals: StrList, imp_count: c_int) CStr { + if (reg == null or raw == null or raw[0] == 0) return null; + const res = c.cbm_registry_resolve(reg, raw, module_qn, imp_keys, imp_vals, imp_count); + if (res.qualified_name == null or res.qualified_name[0] == 0) return null; + if (pxcBaseStrategyIsWeak(res.strategy)) return null; + if (!c.cbm_label_is_type_like(c.cbm_registry_label_of(reg, res.qualified_name))) return null; + return res.qualified_name; +} + +/// pxcJoinPipe over base_classes, substituting each resolved QN for its +/// source spelling. Unresolved entries pass through verbatim so a base the +/// registry does not know keeps working exactly as before. +fn pxcJoinBaseQns( + arena: *Arena, + bases: [*c]const CStr, + reg: ?*const Registry, + module_qn: CStr, + imp_keys: StrList, + imp_vals: StrList, + imp_count: c_int, +) CStr { + if (bases == null or bases[0] == null) return null; + var count: usize = 0; + while (bases[count] != null) count += 1; + const resolved: StrList = @ptrCast(@alignCast(c.cbm_arena_alloc(arena, (count + 1) * @sizeOf(CStr)))); + if (resolved == null) return pxcJoinPipe(arena, bases); + for (0..count) |i| { + const qn = pxcResolveBaseQn(reg, bases[i], module_qn, imp_keys, imp_vals, imp_count); + resolved[i] = if (qn != null) qn else bases[i]; + } + resolved[count] = null; + return pxcJoinPipe(arena, resolved); +} + +fn pxcLastComponent(qn: CStr) CStr { + if (qn == null) return null; + const dot: CStr = c.strrchr(qn, '.'); + return if (dot != null) dot + 1 else qn; +} + +fn pxcJvmTypeQn(arena: ?*Arena, namespace_name: CStr, type_qn_or_name: CStr) CStr { + if (arena == null or namespace_name == null or namespace_name[0] == 0 or type_qn_or_name == null) { + return type_qn_or_name; + } + const short_name = pxcLastComponent(type_qn_or_name); + if (short_name == null or short_name[0] == 0) return type_qn_or_name; + return c.cbm_arena_sprintf(arena, "%s.%s", namespace_name, short_name); +} + +fn pxcJvmDefQn(arena: ?*Arena, src: ?*const Definition, namespace_name: CStr, label: CStr) CStr { + const s = src orelse return null; + if (arena == null or namespace_name == null or namespace_name[0] == 0) return s.qualified_name; + if (streq(label, "Method") or streq(label, "Function") or streq(label, "Constructor")) { + if (s.parent_class != null and s.parent_class[0] != 0) { + return c.cbm_arena_sprintf(arena, "%s.%s.%s", namespace_name, pxcLastComponent(s.parent_class), s.name); + } + return c.cbm_arena_sprintf(arena, "%s.%s", namespace_name, s.name); + } + return c.cbm_arena_sprintf(arena, "%s.%s", namespace_name, s.name); +} + +fn pxcInferJvmNamespace(arena: ?*Arena, rel_path: CStr, lang: Language) CStr { + if (arena == null or rel_path == null or !pxcIsJvmLang(lang)) return null; + var root: CStr = null; + const lang_root: CStr = if (lang == c.CBM_LANG_KOTLIN) "kotlin/" else "java/"; + const lang_root_len = c.strlen(lang_root); + if (hasPrefix(rel_path, "src/main/") and c.strncmp(rel_path + 9, lang_root, lang_root_len) == 0) { + root = rel_path + 9 + lang_root_len; + } else if (hasPrefix(rel_path, "src/test/") and + c.strncmp(rel_path + 9, lang_root, lang_root_len) == 0) + { + root = rel_path + 9 + lang_root_len; + } else { + const needle: CStr = if (lang == c.CBM_LANG_KOTLIN) "/kotlin/" else "/java/"; + root = c.strstr(rel_path, needle); + if (root != null) { + root += c.strlen(needle); + } else if (hasPrefix(rel_path, "src/")) { + root = rel_path + 4; + } else { + root = c.strstr(rel_path, "/src/"); + if (root != null) root += "/src/".len; + } + } + if (root == null or root[0] == 0) return null; + if (hasPrefix(root, "main/") or hasPrefix(root, "test/")) root += 5; + if (hasPrefix(root, "java/")) { + root += 5; + } else if (hasPrefix(root, "kotlin/")) { + root += 7; + } + const slash: CStr = c.strrchr(root, '/'); + if (slash == null or @intFromPtr(slash) <= @intFromPtr(root)) return null; + const len = ptrDiff(slash, root); + const ns: MutStr = @ptrCast(c.cbm_arena_alloc(arena, len + 1)); + if (ns == null) return null; + @memcpy(ns[0..len], root[0..len]); + ns[len] = 0; + for (0..len) |i| { + if (ns[i] == '/') ns[i] = '.'; + } + return ns; +} + +fn pxcImportLookup(local: CStr, len: usize, imp_keys: StrList, imp_vals: StrList, imp_count: c_int) CStr { + for (0..ix(imp_count)) |i| { + if (imp_keys[i] != null and imp_vals[i] != null and c.strlen(imp_keys[i]) == len and + c.strncmp(imp_keys[i], local, len) == 0) + { + return imp_vals[i]; + } + } + return null; +} + +/// The innermost container enclosing a container-level Zig alias (a QN prefix +/// of its parent_class, above the module) that declares `head` in this file, +/// as `.`. NULL when none does. +fn pxcZigContainerHead(arena: *Arena, file: *const FileResult, scope: CStr, module_qn: CStr, head: CStr, len: usize) CStr { + const mlen = c.strlen(module_qn); + var slen = c.strlen(scope); + while (slen > mlen) { + for (0..ix(file.defs.count)) |i| { + const qn: CStr = file.defs.items[i].qualified_name; + if (qn != null and c.strlen(qn) == slen + 1 + len and c.strncmp(qn, scope, slen) == 0 and + qn[slen] == '.' and c.strncmp(qn + slen + 1, head, len) == 0) + { + return c.cbm_arena_strdup(arena, qn); + } + } + while (slen > mlen and scope[slen - 1] != '.') slen -= 1; + slen = if (slen > 0) slen - 1 else 0; + } + return null; +} + +/// Zig: an alias (`const db = @import(..);` of "db.zig", `pub const Database = +/// zova.Database;`, alias_value = the value text), at module or container +/// level, becomes a "Variable" def whose return_types is the QN it names. The +/// head resolves lexically: a declaration in an enclosing container, then the +/// declaring file's imports (`const v = @import(..)` is imported as `v`, +/// the `.C` of an import of "x.zig" as `x`); any other head is a path from the declaring +/// module. Returns -1 for anything else. +fn pxcBuildZigAlias( + arena: *Arena, + file: *const FileResult, + src: *const Definition, + module_qn: CStr, + dst: *LspDef, + imp_keys: StrList, + imp_vals: StrList, + imp_count: c_int, +) c_int { + const text: CStr = src.alias_value; + if (text == null or text[0] == 0 or module_qn == null or c.strpbrk(text, " \t\r\n") != null) { + return -1; + } + var target: CStr = null; + const import_open = "@import(\""; + if (hasPrefix(text, import_open)) { + const path: CStr = text + import_open.len; + const close: CStr = c.strchr(path, '"'); + if (close == null or close[1] != ')') return -1; + const rest: CStr = close + 2; + if (ptrDiff(close, path) == "std".len and c.strncmp(path, "std", 3) == 0) { + target = c.cbm_arena_sprintf(arena, "std%s", rest); // the opaque std namespace + } else { + var local: CStr = src.name; + var len = c.strlen(local); + if (rest[0] != 0) { + local = path; + var p = path; + while (@intFromPtr(p) < @intFromPtr(close)) : (p += 1) { + if (p[0] == '/') local = p + 1; + } + len = ptrDiff(close, local); + if (len > 4 and c.strncmp(local + len - 4, ".zig", 4) == 0) len -= 4; + } + const head = pxcImportLookup(local, len, imp_keys, imp_vals, imp_count); + target = if (head != null) c.cbm_arena_sprintf(arena, "%s%s", head, rest) else null; + } + } else { + const len = c.strcspn(text, "."); + var head: CStr = if (src.parent_class != null) + pxcZigContainerHead(arena, file, src.parent_class, module_qn, text, len) + else + null; + if (head == null) head = pxcImportLookup(text, len, imp_keys, imp_vals, imp_count); + target = if (head != null) + c.cbm_arena_sprintf(arena, "%s%s", head, text + len) + else + c.cbm_arena_sprintf(arena, "%s.%s", module_qn, text); + } + if (target == null) return -1; + dst.* = std.mem.zeroes(LspDef); + dst.qualified_name = src.qualified_name; + dst.short_name = src.name; + dst.label = "Variable"; + dst.def_module_qn = module_qn; + dst.return_types = target; + dst.lang = c.CBM_LANG_ZIG; + return 0; +} + +/// Zig: a type function (`fn List(comptime T: type) type`) names the +/// containers it returns, whose members extraction scopes under its QN. The +/// resolver sees it as a type too: a "Struct" def of the same QN (its fields +/// fold in like any struct's) whose type_params lists its parameters' +/// kinds (cbm_zig_param_kinds of the signature text). +fn pxcIsZigTypeFn(src: *const Definition) bool { + return src.label != null and (streq(src.label, "Function") or streq(src.label, "Method")) and + src.return_type != null and streq(src.return_type, "type") and + src.qualified_name != null and src.name != null; +} + +fn pxcBuildZigTypeFn(arena: *Arena, src: *const Definition, module_qn: CStr, dst: *LspDef) c_int { + const out: CStr = c.cbm_zig_param_kinds(arena, if (src.signature != null) src.signature else ""); + if (out == null) return -1; + dst.* = std.mem.zeroes(LspDef); + dst.qualified_name = src.qualified_name; + dst.short_name = src.name; + dst.label = "Struct"; + dst.def_module_qn = module_qn; + dst.type_params = out; + dst.lang = c.CBM_LANG_ZIG; + return 0; +} + +/// Convert one CBMDefinition into a CBMLSPDef. Returns 0 on success, -1 +/// to skip (unsupported label or missing required field). dst gets borrowed +/// pointers into src and into `arena` for synthesised composites. +fn pxcBuildLspDef( + arena: *Arena, + src: *const Definition, + module_qn: CStr, + namespace_name: CStr, + lang: Language, + dst: *LspDef, + reg: ?*const Registry, + imp_keys: StrList, + imp_vals: StrList, + imp_count: c_int, +) c_int { + const label = pxcMapLabel(src.label); + if (label == null or src.qualified_name == null or src.name == null) return -1; + dst.* = std.mem.zeroes(LspDef); + if (pxcIsJvmLang(lang) and namespace_name != null and namespace_name[0] != 0) { + dst.qualified_name = pxcJvmDefQn(arena, src, namespace_name, label); + dst.receiver_type = pxcJvmTypeQn(arena, namespace_name, src.parent_class); + } else { + dst.qualified_name = src.qualified_name; + dst.receiver_type = src.parent_class; + } + dst.short_name = src.name; + dst.label = label; + dst.def_module_qn = module_qn; + dst.namespace_name = namespace_name; + dst.is_interface = streq(label, "Interface") or streq(label, "Protocol"); + // Single return-type string. The per-language registrars split on '|' + // for multi-return languages (Go); single-return languages just see one + // piece, which is what's already stored. + dst.return_types = src.return_type; + // Languages whose cross registrars read embedded_types as QNs get their + // bases resolved against the project registry; everyone else keeps the + // raw source spelling their own registrar already knows how to handle. + dst.embedded_types = if (reg != null and pxcLangResolvesBaseQns(lang)) + pxcJoinBaseQns(arena, src.base_classes, reg, module_qn, imp_keys, imp_vals, imp_count) + else + pxcJoinPipe(arena, src.base_classes); + dst.lang = lang; + return 0; +} + +/// A "Field" definition of the struct `struct_qn` that has a name and a type. +fn isFieldOf(fd: *const Definition, struct_qn: CStr) bool { + return !(fd.label == null or fd.parent_class == null or fd.name == null or fd.name[0] == 0 or + fd.return_type == null or fd.return_type[0] == 0 or !streq(fd.label, "Field") or + !streq(fd.parent_class, struct_qn)); +} + +/// Go and Zig: fold per-field "Field" definitions into their owning struct's +/// field_defs. extract_defs.c emits one flat CBMDefinition per struct field +/// (label "Field", parent_class = owning struct QN, name = field name, +/// return_type = raw type text). Those rows are dropped by pxcBuildLspDef +/// (pxcMapLabel excludes "Field"), so without this fold every Go struct +/// registers with zero fields and field-chain calls (h.svc.Handle) can +/// never resolve. Fields are always declared in the same file as their struct, +/// so scanning the file's own defs covers every case. Runs inside +/// cbm_pxc_collect_all_defs — one site covers both the prebuilt-registry path +/// and the per-file fallback, since both consume all_defs. +fn pxcFoldGoStructFields(arena: ?*Arena, result: ?*const FileResult, defs: [*c]LspDef, start: c_int, end: c_int) void { + const res = result orelse return; + if (arena == null or defs == null or start >= end) return; + for (ix(start)..ix(end)) |si| { + const dst = util.at(defs, si); + if (dst.label == null or !streq(dst.label, "Struct") or dst.qualified_name == null) continue; + var count: usize = 0; + var total: usize = 0; // "name:type" bytes; separators and NUL added below + for (0..ix(res.defs.count)) |di| { + const fd = util.at(res.defs.items, di); + if (!isFieldOf(fd, dst.qualified_name)) continue; + total += c.strlen(fd.name) + 1 + c.strlen(fd.return_type); + count += 1; + } + if (count == 0) continue; + // count - 1 separators + NUL. + const bufsz = total + (count - 1) + 1; + const buf: MutStr = @ptrCast(c.cbm_arena_alloc(arena, bufsz)); + if (buf == null) continue; + var p: usize = 0; + var written: usize = 0; + for (0..ix(res.defs.count)) |di| { + const fd = util.at(res.defs.items, di); + if (!isFieldOf(fd, dst.qualified_name)) continue; + var n = c.strlen(fd.name); + @memcpy(buf[p .. p + n], fd.name[0..n]); + p += n; + buf[p] = ':'; + p += 1; + n = c.strlen(fd.return_type); + @memcpy(buf[p .. p + n], fd.return_type[0..n]); + p += n; + if (written + 1 < count) { + buf[p] = '|'; + p += 1; + } + written += 1; + } + buf[p] = 0; + dst.field_defs = buf; + } +} + +/// A file's declared namespace, or for Java/Kotlin the one inferred from its +/// source-root path, stored back on the result (in its arena) so later readers +/// of result->namespace_name see the same value. NULL when there is none. +pub export fn cbm_pxc_apply_inferred_namespace(result: *FileResult, file: *const FileInfo) CStr { + var namespace_name: CStr = result.namespace_name; + if ((namespace_name == null or namespace_name[0] == 0) and file.rel_path != null) { + namespace_name = pxcInferJvmNamespace(&result.arena, file.rel_path, file.language); + if (namespace_name != null and namespace_name[0] != 0) { + result.namespace_name = namespace_name; + } + } + return namespace_name; +} + +/// Re-home defs[start, end) into `arena`: they borrow strings from a loaded +/// (spilled) result that is freed right after collection. def_module_qn is +/// borrowed from def_modules[] and stays. +fn pxcOwnDefStrings(arena: ?*Arena, defs: [*c]LspDef, start: c_int, end: c_int) void { + for (ix(start)..ix(end)) |i| { + const d = util.at(defs, i); + inline for (.{ + "qualified_name", "short_name", "label", "receiver_type", + "return_types", "embedded_types", "field_defs", "method_names_str", + "namespace_name", "type_params", + }) |field| { + @field(d, field) = c.cbm_arena_strdup(arena, @field(d, field)); + } + } +} + +/// Collect a project-wide CBMLSPDef[] from all cached results. Returns a +/// malloc'd array (caller frees) of length *out_count. String fields are +/// borrowed from cache[i]->arena and from def_modules[i] (also borrowed); +/// defs of a spilled result (ctx->spill) are copied into owned_arena, and +/// spilled results are skipped when owned_arena is NULL. +pub export fn cbm_pxc_collect_all_defs( + ctx: ?*const Ctx, + cache: [*c][*c]FileResult, + files: [*c]const FileInfo, + file_count: c_int, + project_name: CStr, + def_modules: [*c]MutStr, + owned_arena: ?*Arena, + out_count: *c_int, +) [*c]LspDef { + const spill = if (ctx != null and owned_arena != null) ctx.?.spill else null; + var cap: c_int = 0; + for (0..ix(file_count)) |i| { + if (cache[i] != null) cap += cache[i].*.defs.count; + } + if (cap == 0 and spill == null) { + out_count.* = 0; + return null; + } + var defs: [*c]LspDef = if (cap > 0) interop.callocN(LspDef, ix(cap)) else null; + if (cap > 0 and defs == null) { + out_count.* = 0; + return null; + } + var idx: c_int = 0; + for (0..ix(file_count)) |fi| { + const file_start = idx; + const file = util.at(files, fi); + const is_zig = file.language == c.CBM_LANG_ZIG; + const result: *FileResult = c.cbm_result_cache_acquire(cache, spill, @intCast(fi)) orelse continue; + const loaded = result != @as(?*FileResult, cache[fi]); + var need: c_int = result.defs.count; + if (is_zig) for (0..ix(result.defs.count)) |di| { + need += @intFromBool(pxcIsZigTypeFn(util.at(result.defs.items, di))); + }; + if (idx + need > cap) { + const grown: c_int = if (cap * 2 > idx + need) cap * 2 else idx + need; + const more = interop.reallocN(LspDef, defs, ix(grown)); + if (more == null) { + c.cbm_result_cache_release(cache, @intCast(fi), result); + continue; + } + defs = more; + cap = grown; + } + if (def_modules[fi] == null) { + def_modules[fi] = c.cbm_pipeline_fqn_module_dir(project_name, file.rel_path, pxcModuleIsDir(file.language)); + } + const namespace_name = cbm_pxc_apply_inferred_namespace(result, file); + // One import map per FILE (not per def, and not per base name): the + // cross-file base-class resolution needs the same local-name → + // import-QN view pass_semantic uses. Both drivers have already + // created this file's IMPORTS edges and filled the registry here. + // Built only for the languages that consume resolved base QNs (and + // for Zig, whose aliases resolve through it), and only when a caller + // supplied the pipeline context. + var base_reg: ?*const Registry = null; + var imp_keys: StrList = null; + var imp_vals: StrList = null; + var imp_count: c_int = 0; + if (ctx) |cx| { + if (cx.registry != null and cx.gbuf != null and pxcLangResolvesBaseQns(file.language)) { + base_reg = cx.registry; + } + if (cx.gbuf != null and (base_reg != null or is_zig)) { + _ = pxcBuildImportMap(cx.gbuf, project_name, file.rel_path, &imp_keys, &imp_vals, &imp_count); + } + } + for (0..ix(result.defs.count)) |di| { + const src = util.at(result.defs.items, di); + const dst = util.at(defs, ix(idx)); + const rc = if (is_zig and src.label != null and streq(src.label, "Variable") and + src.qualified_name != null and src.name != null) + pxcBuildZigAlias(&result.arena, result, src, def_modules[fi], dst, imp_keys, imp_vals, imp_count) + else + pxcBuildLspDef(&result.arena, src, def_modules[fi], namespace_name, file.language, dst, base_reg, imp_keys, imp_vals, imp_count); + if (rc == 0) { + // Zig: a function taking `anytype` carries its parameter kinds, + // so callers' argument types reach it (cbm_pxc_collect_zig_args). + if (is_zig and src.signature != null and c.strstr(src.signature, "anytype") != null and + (streq(dst.label, "Function") or streq(dst.label, "Method"))) + { + dst.type_params = c.cbm_zig_param_kinds(&result.arena, src.signature); + } + idx += 1; + } + if (is_zig and pxcIsZigTypeFn(src) and + pxcBuildZigTypeFn(&result.arena, src, def_modules[fi], util.at(defs, ix(idx))) == 0) + { + idx += 1; + } + } + pxcFreeImportMap(imp_keys, imp_vals, imp_count); // NULL-safe + if (file.language == c.CBM_LANG_GO or is_zig) { + pxcFoldGoStructFields(&result.arena, result, defs, file_start, idx); + } + if (loaded) pxcOwnDefStrings(owned_arena, defs, file_start, idx); + c.cbm_result_cache_release(cache, @intCast(fi), result); + } + out_count.* = idx; + return defs; +} + +/// Build per-file import map (local_name -> resolved module QN) from gbuf +/// IMPORTS edges. Mirrors build_import_map() in pass_parallel. Returns 0 +/// with *out_count = 0 when the file has no IMPORTS edges. Caller frees keys +/// with pxcFreeImportMap. +fn pxcBuildImportMap( + gbuf: ?*const Gbuf, + project_name: CStr, + rel_path: CStr, + out_keys: *StrList, + out_vals: *StrList, + out_count: *c_int, +) c_int { + out_keys.* = null; + out_vals.* = null; + out_count.* = 0; + + const file_qn: MutStr = c.cbm_pipeline_fqn_compute(project_name, rel_path, "__file__"); + if (file_qn == null) return 0; + const file_node: ?*const Node = c.cbm_gbuf_find_by_qn(gbuf, file_qn); + c.free(file_qn); + const fnode = file_node orelse return 0; + + var edges: [*c][*c]const Edge = null; + var edge_count: c_int = 0; + const rc = c.cbm_gbuf_find_edges_by_source_type(gbuf, fnode.id, "IMPORTS", &edges, &edge_count); + if (rc != 0 or edge_count == 0) return 0; + + const keys = interop.callocN(CStr, ix(edge_count)); + const vals = interop.callocN(CStr, ix(edge_count)); + if (keys == null or vals == null) { + c.free(@ptrCast(keys)); + c.free(@ptrCast(vals)); + return 0; + } + const marker = "\"local_name\":\""; + var count: usize = 0; + for (0..ix(edge_count)) |i| { + const e: *const Edge = edges[i]; + const target: ?*const Node = c.cbm_gbuf_find_by_id(gbuf, e.target_id); + if (target == null or e.properties_json == null) continue; + var start: CStr = c.strstr(e.properties_json, marker); + if (start == null) continue; + start += marker.len; + const end: CStr = c.strchr(start, '"'); + if (end == null or @intFromPtr(end) <= @intFromPtr(start)) continue; + const n = ptrDiff(end, start); + const local: MutStr = @ptrCast(c.malloc(n + 1)); + if (local == null) continue; + @memcpy(local[0..n], start[0..n]); + local[n] = 0; + keys[count] = local; + vals[count] = target.?.qualified_name; // borrowed from gbuf + count += 1; + } + out_keys.* = keys; + out_vals.* = vals; + out_count.* = @intCast(count); + return 0; +} + +fn pxcFreeImportMap(keys: StrList, vals: StrList, count: c_int) void { + if (keys != null) { + for (0..ix(count)) |i| c.free(@constCast(keys[i])); + c.free(@ptrCast(keys)); + } + c.free(@ptrCast(vals)); // vals strings borrowed from gbuf — don't free elements +} + +/// Detect TS dialect flags from a relative path. +pub export fn cbm_pxc_ts_modes(lang: Language, rel_path: CStr, out_js: *bool, out_jsx: *bool, out_dts: *bool) void { + out_js.* = lang == c.CBM_LANG_JAVASCRIPT; + out_jsx.* = lang == c.CBM_LANG_TSX; + out_dts.* = false; + if (rel_path == null) return; + const rl = c.strlen(rel_path); + if (lang == c.CBM_LANG_JAVASCRIPT and rl >= 4 and streq(rel_path + rl - 4, ".jsx")) { + out_jsx.* = true; + } + if (lang == c.CBM_LANG_TYPESCRIPT and rl >= 5 and streq(rel_path + rl - 5, ".d.ts")) { + out_dts.* = true; + } +} + +/// Returns true when this language has a cross-file LSP wired up. +pub export fn cbm_pxc_has_cross_lsp(lang: Language) bool { + return switch (lang) { + c.CBM_LANG_GO, + c.CBM_LANG_C, + c.CBM_LANG_CPP, + c.CBM_LANG_CUDA, + c.CBM_LANG_PYTHON, + c.CBM_LANG_JAVASCRIPT, + c.CBM_LANG_TYPESCRIPT, + c.CBM_LANG_TSX, + c.CBM_LANG_PHP, + c.CBM_LANG_CSHARP, // tier-2 prebuilt registry path (pass_parallel) + c.CBM_LANG_JAVA, // fallback cbm_pxc_run_one path + c.CBM_LANG_KOTLIN, // fallback cbm_pxc_run_one path + c.CBM_LANG_RUST, // fallback cbm_pxc_run_one path (manifest-aware) + c.CBM_LANG_ZIG, // fallback cbm_pxc_run_one path + => true, + else => false, + }; +} + +// Append cross-file results from `src_out` (allocated in a scratch arena +// about to be destroyed) into `dst_calls` (lives in cache_entry->arena), +// copying every string field into dst_arena. Skips entries whose +// (caller_qn, callee_qn) is already present — avoids inflating the array with +// cross-file duplicates of per-file LSP output. +// +// Dedup uses a hash set keyed on "caller\x1fcallee", giving O(1) membership. +// The previous linear strcmp scan made each append O(n), so a file that +// resolved very many cross-calls turned the whole append into O(n^2) and could +// peg a core for minutes (observed: an index hung in pxc_append_results/strcmp). +// The key strings live in a scratch arena that is destroyed after the table. +/// Invocation rows dedup on (caller, callee). A reference row is tied to one +/// source occurrence, so its site is part of the key and it never collides +/// with an invocation of the same callee. An invocation row with a site span +/// proves one call site (Zig), so its site is part of the key too. +fn pxcResultKey(keys: *Arena, rc: *const c.CBMResolvedCall) MutStr { + if (rc.kind == c.CBM_RESOLVED_CALL_REFERENCE) { + return c.cbm_arena_sprintf(keys, "%s\x1f%s\x1fR%u:%u", rc.caller_qn, rc.callee_qn, rc.site_start_byte, rc.site_end_byte); + } + if (rc.site_end_byte > rc.site_start_byte) { + return c.cbm_arena_sprintf(keys, "%s\x1f%s\x1fI%u", rc.caller_qn, rc.callee_qn, rc.site_start_byte); + } + return c.cbm_arena_sprintf(keys, "%s\x1f%s", rc.caller_qn, rc.callee_qn); +} + +fn pxcAppendResults(dst_arena: *Arena, dst_calls: *c.CBMResolvedCallArray, src_out: *const c.CBMResolvedCallArray) void { + var keys: Arena = undefined; + c.cbm_arena_init(&keys); + const seen = c.cbm_ht_create(@as(u32, @bitCast(dst_calls.count + src_out.count + 1))); + + for (0..ix(dst_calls.count)) |i| { + const rc = util.at(dst_calls.items, i); + if (rc.caller_qn != null and rc.callee_qn != null) { + const k = pxcResultKey(&keys, rc); + if (k != null) _ = c.cbm_ht_set(seen, k, @ptrFromInt(1)); + } + } + + for (0..ix(src_out.count)) |j| { + const src = util.at(src_out.items, j); + if (src.caller_qn == null or src.callee_qn == null) continue; + const k = pxcResultKey(&keys, src); + if (k != null and c.cbm_ht_has(seen, k)) continue; + if (k != null) _ = c.cbm_ht_set(seen, k, @ptrFromInt(1)); + var dst = std.mem.zeroes(c.CBMResolvedCall); + dst.caller_qn = c.cbm_arena_strdup(dst_arena, src.caller_qn); + dst.callee_qn = c.cbm_arena_strdup(dst_arena, src.callee_qn); + dst.strategy = if (src.strategy != null) c.cbm_arena_strdup(dst_arena, src.strategy) else null; + dst.confidence = src.confidence; + dst.reason = if (src.reason != null) c.cbm_arena_strdup(dst_arena, src.reason) else null; + dst.kind = src.kind; + dst.site_start_byte = src.site_start_byte; + dst.site_end_byte = src.site_end_byte; + c.cbm_resolvedcall_push(dst_calls, dst_arena, dst); + } + + c.cbm_ht_free(seen); + c.cbm_arena_destroy(&keys); +} + +// ── Rust workspace manifest (Cargo.toml) for cross-CRATE resolution ── +// +// cbm_pxc_run_one's signature is shared with the parallel pass +// (pass_parallel) and cannot grow a manifest parameter without touching +// that file. We therefore pass the parsed workspace manifest to the Rust +// cross-file resolver through a thread-local borrowed pointer that each +// resolve worker sets from the manifest cbm_pxc_build_rust_manifest read +// once from the project's root Cargo.toml. The manifest's strings are +// owned by the caller's arena; the pointer is borrowed (NULL when +// the project has no Cargo.toml — single-crate / non-workspace projects, +// where in-file resolution needs no workspace metadata). +threadlocal var g_pxc_rust_manifest: ?*const c.CBMCargoManifest = null; + +pub export fn cbm_pxc_set_rust_manifest(m: ?*const c.CBMCargoManifest) void { + g_pxc_rust_manifest = m; +} + +pub export fn cbm_pxc_get_rust_manifest() ?*const c.CBMCargoManifest { + return g_pxc_rust_manifest; +} + +/// Convert a CBMLSPDef array (the pipeline's lingua franca, go_lsp.h:73) +/// into a CBMRustLSPDef array (rust_lsp.h) inside `arena`. The two structs +/// share their first 9 string fields; CBMRustLSPDef adds `trait_qn` before +/// `is_interface` whereas CBMLSPDef has `is_interface` followed by `lang`, +/// so a memcpy is unsafe — copy field-by-field. trait_qn is left NULL +/// because the pipeline's collect-all-defs step does not carry the +/// impl-Trait-for-Type linkage; the resolver still recovers trait dispatch +/// from the in-file walk (the cross-file path only needs receiver_type). +fn pxcLspdefsToRust(arena: *Arena, defs: [*c]const LspDef, def_count: c_int) [*c]c.CBMRustLSPDef { + if (defs == null or def_count <= 0) return null; + const out: [*c]c.CBMRustLSPDef = + @ptrCast(@alignCast(c.cbm_arena_alloc(arena, ix(def_count) * @sizeOf(c.CBMRustLSPDef)))); + if (out == null) return null; + for (0..ix(def_count)) |i| { + inline for (.{ + "qualified_name", "short_name", "label", "receiver_type", "def_module_qn", + "return_types", "embedded_types", "field_defs", "method_names_str", + }) |field| { + @field(out[i], field) = @field(defs[i], field); + } + out[i].trait_qn = null; + out[i].is_interface = defs[i].is_interface; + } + return out; +} + +/// Run cross-file LSP for a single file inside a scratch arena that gets +/// freed when the call returns. The LSP would otherwise allocate a fresh +/// type registry + stdlib + all project defs into the supplied arena, and +/// that adds up to O(N×project_size) memory if we used cache[i]->arena +/// directly across N files (test_incremental.c saw 3.5 GB peak on a +/// 1100-file repo before this fix). Output gets copied into the file's own +/// arena and merged into result->resolved_calls. +pub export fn cbm_pxc_run_one( + lang: Language, + r: *FileResult, + source: CStr, + source_len: c_int, + module_qn: CStr, + defs: [*c]LspDef, + def_count: c_int, + imp_names: StrList, + imp_qns: StrList, + imp_count: c_int, +) void { + const tree = r.cached_tree; // may be NULL — LSP re-parses then + + var scratch: Arena = undefined; + c.cbm_arena_init(&scratch); + var out = std.mem.zeroes(c.CBMResolvedCallArray); + + switch (lang) { + c.CBM_LANG_GO => c.cbm_run_go_lsp_cross(&scratch, source, source_len, module_qn, defs, def_count, imp_names, imp_qns, imp_count, tree, &out), + c.CBM_LANG_C, c.CBM_LANG_CPP, c.CBM_LANG_CUDA => { + const cpp_mode = lang != c.CBM_LANG_C; + // C/C++ cross LSP takes include_paths/include_ns_qns instead of + // imports — the existing pipeline doesn't carry C-style include + // resolution as a separate map, so pass NULL/0 and let the LSP + // fall back to its own #include scan. + const status = c.cbm_run_c_lsp_cross_status(&scratch, source, source_len, module_qn, cpp_mode, defs, def_count, null, null, 0, tree, &out); + if (status != c.CBM_LSP_COMPLETE) c.cbm_c_lsp_record_failure(r, status); + }, + c.CBM_LANG_PYTHON => { + const status = c.cbm_run_py_lsp_cross_status(&scratch, source, source_len, module_qn, defs, def_count, imp_names, imp_qns, imp_count, tree, &out); + if (status != c.CBM_LSP_COMPLETE) c.cbm_py_lsp_record_failure(r, status); + }, + c.CBM_LANG_PHP => c.cbm_run_php_lsp_cross(&scratch, source, source_len, module_qn, defs, def_count, imp_names, imp_qns, imp_count, tree, &out), + c.CBM_LANG_JAVA => c.cbm_run_java_lsp_cross(&scratch, source, source_len, module_qn, defs, def_count, imp_names, imp_qns, imp_count, tree, &out), + c.CBM_LANG_KOTLIN => c.cbm_run_kotlin_lsp_cross(&scratch, source, source_len, module_qn, defs, def_count, imp_names, imp_qns, imp_count, tree, &out), + c.CBM_LANG_RUST => { + // The Rust resolver wants CBMRustLSPDef (rust_lsp.h), not the + // pipeline's CBMLSPDef — the structs share their first 9 fields + // but diverge after, so convert into the scratch arena. The + // workspace manifest (set by each resolve worker) lets + // `crate_a::foo` route across the crate boundary (#56). + const rdefs = pxcLspdefsToRust(&scratch, defs, def_count); + c.cbm_run_rust_lsp_cross_with_manifest(&scratch, source, source_len, module_qn, rdefs, def_count, imp_names, imp_qns, imp_count, tree, g_pxc_rust_manifest, &out); + }, + c.CBM_LANG_ZIG => c.cbm_run_zig_lsp_cross(&scratch, source, source_len, module_qn, defs, def_count, imp_names, imp_qns, imp_count, tree, &out), + else => {}, + } + + pxcAppendResults(&r.arena, &r.resolved_calls, &out); + c.cbm_arena_destroy(&scratch); +} + +/// cbm_pxc_run_one for Zig with the project's `anytype` argument types (args) +/// and, when collect is set, collecting them instead: the file's resolved +/// calls are then discarded. +fn pxcRunOneZig( + r: *FileResult, + source: CStr, + source_len: c_int, + module_qn: CStr, + defs: [*c]LspDef, + def_count: c_int, + imp_names: StrList, + imp_qns: StrList, + imp_count: c_int, + args: [*c]const c.CBMZigArgType, + arg_count: c_int, + collect: ?*c.CBMZigArgTypeArray, +) void { + var scratch: Arena = undefined; + c.cbm_arena_init(&scratch); + var out = std.mem.zeroes(c.CBMResolvedCallArray); + c.cbm_run_zig_lsp_cross_ex(&scratch, source, source_len, module_qn, defs, def_count, imp_names, imp_qns, imp_count, r.cached_tree, &out, args, arg_count, collect); + if (collect == null) pxcAppendResults(&r.arena, &r.resolved_calls, &out); + c.cbm_arena_destroy(&scratch); +} + +/// Variant of cbm_pxc_run_one for TS/JS/JSX/TSX with explicit dialect +/// flags. Same scratch-arena lifecycle as cbm_pxc_run_one. +pub export fn cbm_pxc_run_one_ts( + r: *FileResult, + source: CStr, + source_len: c_int, + module_qn: CStr, + defs: [*c]LspDef, + def_count: c_int, + imp_names: StrList, + imp_qns: StrList, + imp_count: c_int, + js_mode: bool, + jsx_mode: bool, + dts_mode: bool, +) void { + var scratch: Arena = undefined; + c.cbm_arena_init(&scratch); + var out = std.mem.zeroes(c.CBMResolvedCallArray); + + c.cbm_run_ts_lsp_cross(&scratch, source, source_len, module_qn, js_mode, jsx_mode, dts_mode, defs, def_count, imp_names, imp_qns, imp_count, r.cached_tree, &out); + + pxcAppendResults(&r.arena, &r.resolved_calls, &out); + c.cbm_arena_destroy(&scratch); +} + +// Per-file cross-LSP dispatch for the parallel resolve worker: filter the global defs +// down to the file's own+imported modules via the module-def index, resolve +// through the shared prebuilt registry when the language has one (per-file +// OVERLAY pattern — no registry build, no finalize), and only fall back to +// the per-file registry build (with the FILTERED defs) for languages without +// a shared-registry variant. Before this helper existed the sequential +// driver fed the FULL def list into full per-file registry builds — +// O(files x defs), which ground an 81k-file TS corpus for hours. +// +// `rust_shared_get` supplies the lazily-built shared Rust all-defs registry +// (the parallel resolver owns its once-guard); NULL means "no shared rust +// registry available" and rust NULL-filter files take the per-file build. +// Per-file registry-build cost counters (upstream cffb4e37), logged by +// pass_parallel at the end of resolve. +var g_pxc_defs_registered: std.atomic.Value(u64) = .init(0); +var g_pxc_build_files: std.atomic.Value(u64) = .init(0); +var g_pxc_filter_files: std.atomic.Value(u64) = .init(0); +var g_pxc_filter_failed: std.atomic.Value(u64) = .init(0); + +pub export fn cbm_pxc_filter_stats(defs_registered: *u64, build_files: *u64, filter_files: *u64, filter_failed: *u64) void { + defs_registered.* = g_pxc_defs_registered.load(.monotonic); + build_files.* = g_pxc_build_files.load(.monotonic); + filter_files.* = g_pxc_filter_files.load(.monotonic); + filter_failed.* = g_pxc_filter_failed.load(.monotonic); +} + +pub export fn cbm_pxc_filter_stats_reset() void { + g_pxc_defs_registered.store(0, .monotonic); + g_pxc_build_files.store(0, .monotonic); + g_pxc_filter_files.store(0, .monotonic); + g_pxc_filter_failed.store(0, .monotonic); +} + +/// imp_vals plus every dotted prefix of each QN in the types bound to +/// module_qn's `anytype` parameters (in arena): the modules among them hold +/// the defs a bound walk resolves against. +fn pxcZigArgModules( + arena: *Arena, + r: *const c.CBMCrossLspRegistries, + module_qn: CStr, + imp_vals: StrList, + imp_count: c_int, + out_count: *c_int, +) StrList { + const mlen = c.strlen(module_qn); + var cap: c_int = imp_count; + for (0..ix(r.zig_arg_count)) |i| { + const q: CStr = r.zig_args[i].callee_qn; + if (c.strncmp(q, module_qn, mlen) == 0 and q[mlen] == '.') { + var t: CStr = r.zig_args[i].type; + while (t[0] != 0) : (t += 1) { + cap += @intFromBool(t[0] == '.'); + } + } + } + const out: StrList = @ptrCast(@alignCast(c.cbm_arena_alloc(arena, ix(cap + 1) * @sizeOf(CStr)))); + if (out == null) { + out_count.* = imp_count; + return imp_vals; + } + var n: c_int = 0; + for (0..ix(imp_count)) |i| { + out[ix(n)] = imp_vals[i]; + n += 1; + } + for (0..ix(r.zig_arg_count)) |i| { + const q: CStr = r.zig_args[i].callee_qn; + if (c.strncmp(q, module_qn, mlen) != 0 or q[mlen] != '.') continue; + const t: CStr = r.zig_args[i].type; + var start: usize = 0; + var j: usize = 0; + while (true) : (j += 1) { + const ch = t[j]; + const qn_char = ch != 0 and c.strchr("[]()*?, ", ch) == null; + if (ch == '.' and j > start and n < cap) { + out[ix(n)] = c.cbm_arena_strndup(arena, t + start, j - start); + n += 1; + } + if (!qn_char) start = j + 1; + if (ch == 0) break; + } + } + out_count.* = n; + return out; +} + +const RustSharedGet = ?*const fn (?*anyopaque) callconv(.c) [*c]c.CBMTypeRegistry; + +pub export fn cbm_pxc_dispatch_file( + lang: Language, + result_c: ?*FileResult, + source: CStr, + source_len: c_int, + rel: CStr, + def_module: CStr, + cross_registries: ?*const c.CBMCrossLspRegistries, + module_def_index: ?*const ModuleDefIndex, + all_defs: [*c]LspDef, + all_def_count: c_int, + imp_keys: StrList, + imp_vals: StrList, + imp_count: c_int, + rust_shared_get: RustSharedGet, + rust_shared_ctx: ?*anyopaque, +) void { + const result = result_c orelse return; + var used_prebuilt = false; + const prebuilt: ?*c.CBMTypeRegistry = + if (cross_registries != null) c.cbm_pxc_registry_for_lang(cross_registries, lang) else null; + if (prebuilt != null) { + switch (lang) { + c.CBM_LANG_GO => { + // Tier 3 (metadata-driven): pure lookup over the Tier-1 + // lsp_unresolved entries — no parse, no AST walk. Then the + // AST walk on the shared Tier-2 registry (mirroring every + // other language) so NAMED receivers evaluated against + // project-wide defs also resolve. The walk variant below is + // read-only — the sealed registry is safe for parallel + // workers. + _ = c.cbm_go_fast_resolve_qualified_calls(result, prebuilt, imp_keys, imp_vals, imp_count); + c.cbm_run_go_lsp_cross_with_registry(&result.arena, source, source_len, def_module, prebuilt, imp_keys, imp_vals, imp_count, result.cached_tree, &result.resolved_calls); + used_prebuilt = true; + }, + c.CBM_LANG_PYTHON => { + const status = c.cbm_run_py_lsp_cross_with_registry_status(&result.arena, source, source_len, def_module, prebuilt, imp_keys, imp_vals, imp_count, result.cached_tree, &result.resolved_calls); + if (status != c.CBM_LSP_COMPLETE) c.cbm_py_lsp_record_failure(result, status); + used_prebuilt = true; + }, + c.CBM_LANG_C, c.CBM_LANG_CPP, c.CBM_LANG_CUDA => { + const status = c.cbm_run_c_lsp_cross_with_registry_status(&result.arena, source, source_len, def_module, lang != c.CBM_LANG_C, prebuilt, imp_keys, imp_vals, imp_count, result.cached_tree, &result.resolved_calls); + if (status != c.CBM_LSP_COMPLETE) c.cbm_c_lsp_record_failure(result, status); + used_prebuilt = true; + }, + c.CBM_LANG_CSHARP => { + c.cbm_run_cs_lsp_cross_with_registry(&result.arena, source, source_len, def_module, prebuilt, imp_vals, imp_count, result.cached_tree, &result.resolved_calls); + used_prebuilt = true; + }, + c.CBM_LANG_JAVASCRIPT, c.CBM_LANG_TYPESCRIPT, c.CBM_LANG_TSX => { + // TS: per-file OVERLAY chained to the shared base. Filter to + // own+imports so the overlay builder can pick out own-module + // defs without scanning the whole project. + var js: bool = undefined; + var jsx: bool = undefined; + var dts: bool = undefined; + cbm_pxc_ts_modes(lang, rel, &js, &jsx, &dts); + var ts_defs = all_defs; + var ts_def_count = all_def_count; + var ts_filtered: [*c]LspDef = null; + if (module_def_index != null) { + var fc: c_int = 0; + ts_filtered = cbm_pxc_filter_defs_for_file(module_def_index, all_defs, lang, result.namespace_name, def_module, imp_vals, imp_count, &fc); + if (ts_filtered != null) { + ts_defs = ts_filtered; + ts_def_count = fc; + } + } + c.cbm_run_ts_lsp_cross_with_registry(&result.arena, source, source_len, def_module, js, jsx, dts, prebuilt, ts_defs, ts_def_count, imp_keys, imp_vals, imp_count, result.cached_tree, &result.resolved_calls); + c.free(ts_filtered); + used_prebuilt = true; + }, + // PHP falls through to the per-file build path below until its + // overlay variant lands. + else => {}, + } + } + + if (used_prebuilt) return; + // Fallback: gopls per-file filter + per-file registry build. RUST is + // exempt from the module filter: its resolution is Cargo-manifest-aware + // and a `crate_a::foo` reference routes to defs in ANOTHER workspace + // crate — a module that is in neither own_module nor the import map, so + // the filter starves cross-crate resolution (#56 repro red). Rust + // therefore always resolves against the FULL def universe: the lazily + // built shared registry when available, else a full per-file build. + var filtered: [*c]LspDef = null; + var file_defs = all_defs; + var file_def_count = all_def_count; + // Zig: a function walked with an `anytype` parameter bound needs the defs + // of the bound types' modules too, which it need not import. + var filter_qns = imp_vals; + var filter_count = imp_count; + var zig_arena: Arena = undefined; + var zig_arena_live = false; + if (lang == c.CBM_LANG_ZIG and cbm_pxc_zig_args_in_module(cross_registries, def_module)) { + c.cbm_arena_init(&zig_arena); + zig_arena_live = true; + filter_qns = pxcZigArgModules(&zig_arena, cross_registries.?, def_module, imp_vals, imp_count, &filter_count); + } + if (module_def_index != null and lang != c.CBM_LANG_RUST) { + var filtered_count: c_int = 0; + filtered = cbm_pxc_filter_defs_for_file(module_def_index, all_defs, lang, result.namespace_name, def_module, filter_qns, filter_count, &filtered_count); + if (filtered != null) { + file_defs = filtered; + file_def_count = filtered_count; + } + _ = g_pxc_filter_files.fetchAdd(1, .monotonic); + if (filtered == null) _ = g_pxc_filter_failed.fetchAdd(1, .monotonic); + } + // Per-file registry build cost is driven by this count. + _ = g_pxc_defs_registered.fetchAdd(@bitCast(@as(i64, file_def_count)), .monotonic); + _ = g_pxc_build_files.fetchAdd(1, .monotonic); + if (lang == c.CBM_LANG_RUST) { + const shared: ?*c.CBMTypeRegistry = if (rust_shared_get) |get| get(rust_shared_ctx) else null; + if (shared != null) { + c.cbm_run_rust_lsp_cross_with_registry(&result.arena, source, source_len, def_module, shared, imp_keys, imp_vals, imp_count, result.cached_tree, cbm_pxc_get_rust_manifest(), &result.resolved_calls, null); + } else { + cbm_pxc_run_one(lang, result, source, source_len, def_module, file_defs, file_def_count, imp_keys, imp_vals, imp_count); + } + } else if (lang == c.CBM_LANG_JAVASCRIPT or lang == c.CBM_LANG_TYPESCRIPT or lang == c.CBM_LANG_TSX) { + var js: bool = undefined; + var jsx: bool = undefined; + var dts: bool = undefined; + cbm_pxc_ts_modes(lang, rel, &js, &jsx, &dts); + cbm_pxc_run_one_ts(result, source, source_len, def_module, file_defs, file_def_count, imp_keys, imp_vals, imp_count, js, jsx, dts); + } else if (lang == c.CBM_LANG_ZIG) { + pxcRunOneZig( + result, + source, + source_len, + def_module, + file_defs, + file_def_count, + imp_keys, + imp_vals, + imp_count, + if (cross_registries) |r| r.zig_args else null, + if (cross_registries) |r| r.zig_arg_count else 0, + null, + ); + } else { + cbm_pxc_run_one(lang, result, source, source_len, def_module, file_defs, file_def_count, imp_keys, imp_vals, imp_count); + } + c.free(filtered); + if (zig_arena_live) c.cbm_arena_destroy(&zig_arena); +} + +fn pxcZigArgCmp(a: ?*const anyopaque, b: ?*const anyopaque) callconv(.c) c_int { + const x: *const c.CBMZigArgType = @ptrCast(@alignCast(a)); + const y: *const c.CBMZigArgType = @ptrCast(@alignCast(b)); + var cmp = c.strcmp(x.callee_qn, y.callee_qn); + if (cmp == 0) cmp = x.param - y.param; + return if (cmp != 0) cmp else c.strcmp(x.type, y.type); +} + +pub export fn cbm_pxc_collect_zig_args( + ctx_c: ?*const Ctx, + cache: [*c][*c]FileResult, + files: [*c]const FileInfo, + file_count: c_int, + def_modules: [*c]const MutStr, + module_def_index: ?*const ModuleDefIndex, + all_defs: [*c]LspDef, + def_count: c_int, + arena: ?*Arena, + out_count: *c_int, +) [*c]c.CBMZigArgType { + out_count.* = 0; + var any = false; + for (0..ix(def_count)) |i| { + if (any) break; + const d = util.at(all_defs, i); + any = d.lang == c.CBM_LANG_ZIG and d.type_params != null and + c.strchr(d.type_params, '*') != null and d.label != null and + (streq(d.label, "Function") or streq(d.label, "Method")); + } + const ctx = ctx_c orelse return null; + if (!any or cache == null or def_modules == null) return null; + var collect = std.mem.zeroes(c.CBMZigArgTypeArray); + collect.arena = arena; + for (0..ix(file_count)) |fi| { + if (files[fi].language != c.CBM_LANG_ZIG or def_modules[fi] == null) continue; + const result: *FileResult = c.cbm_result_cache_acquire(cache, ctx.spill, @intCast(fi)) orelse continue; + var owned: MutStr = null; + var source: CStr = result.source; + var source_len: c_int = result.source_len; + if ((source == null or source_len <= 0) and result.calls.count > 0) { + owned = pxcReadFile(files[fi].path, &source_len); + source = owned; + } + if (source != null and source_len > 0 and result.calls.count > 0) { + var imp_keys: StrList = null; + var imp_vals: StrList = null; + var imp_count: c_int = 0; + _ = pxcBuildImportMap(ctx.gbuf, ctx.project_name, files[fi].rel_path, &imp_keys, &imp_vals, &imp_count); + var fc: c_int = 0; + const filtered: [*c]LspDef = if (module_def_index != null) + cbm_pxc_filter_defs_for_file(module_def_index, all_defs, c.CBM_LANG_ZIG, result.namespace_name, def_modules[fi], imp_vals, imp_count, &fc) + else + null; + pxcRunOneZig( + result, + source, + source_len, + def_modules[fi], + if (filtered != null) filtered else all_defs, + if (filtered != null) fc else def_count, + imp_keys, + imp_vals, + imp_count, + null, + 0, + &collect, + ); + c.free(filtered); + pxcFreeImportMap(imp_keys, imp_vals, imp_count); + } + c.free(owned); + c.cbm_result_cache_release(cache, @intCast(fi), result); + } + if (collect.count == 0) { + c.free(collect.items); + return null; + } + c.qsort(collect.items, ix(collect.count), @sizeOf(c.CBMZigArgType), pxcZigArgCmp); + var n: usize = 1; + for (1..ix(collect.count)) |i| { + if (pxcZigArgCmp(util.at(collect.items, i), util.at(collect.items, n - 1)) != 0) { + collect.items[n] = collect.items[i]; + n += 1; + } + } + interop.logInfo("lsp_cross.zig_anytype", .{ "arg_types", c.cbm_log_itoa(@intCast(n)) }); + out_count.* = @intCast(n); + return collect.items; +} + +pub export fn cbm_pxc_zig_args_in_module(r_c: ?*const c.CBMCrossLspRegistries, module_qn: CStr) bool { + const r = r_c orelse return false; + const n = if (module_qn != null) c.strlen(module_qn) else 0; + if (n == 0) return false; + for (0..ix(r.zig_arg_count)) |i| { + const q: CStr = r.zig_args[i].callee_qn; + if (c.strncmp(q, module_qn, n) == 0 and q[n] == '.') return true; + } + return false; +} + +/// Parse the project's root Cargo.toml (if present) into `out_m`, using +/// `marena` for the manifest's owned strings. Returns true when a manifest +/// was parsed (a workspace root or any [package]/[dependencies]); false when +/// there is no readable Cargo.toml, leaving *out_m untouched. The resulting +/// manifest feeds cross-CRATE Rust resolution (#56): its [workspace].members +/// map lets `crate_a::foo` route to the member crate's def. +pub export fn cbm_pxc_build_rust_manifest(ctx_c: ?*const Ctx, marena: ?*Arena, out_m: ?*c.CBMCargoManifest) bool { + const ctx = ctx_c orelse return false; + const out = out_m orelse return false; + if (ctx.repo_path == null or marena == null) return false; + var path: [1024]u8 = undefined; + const n = c.snprintf(&path, path.len, "%s/Cargo.toml", ctx.repo_path); + if (n <= 0 or ix(n) >= path.len) return false; + var toml_len: c_int = 0; + const toml = pxcReadFile(&path, &toml_len); + if (toml == null or toml_len <= 0) { + c.free(toml); + return false; + } + out.* = std.mem.zeroes(c.CBMCargoManifest); + c.cbm_cargo_parse(marena, toml, toml_len, out); + c.free(toml); // cargo parser copies into marena + return true; +} + +// ── Per-module def index (gopls "package summary" pattern) ────────── + +const ModuleEntry = extern struct { + count: c_int, + cap: c_int, + /// malloc'd; indices into the caller's all_defs[] + indices: [*c]c_int, +}; + +/// struct CBMModuleDefIndex. The header only names it as an opaque type. +const ModuleDefIndex = extern struct { + /// module_qn → ModuleEntry* + ht: ?*HashTable, + /// declared package/namespace → ModuleEntry* + namespace_ht: ?*HashTable, + /// total entries in the all_defs[] array + def_count: c_int, +}; + +/// cbm_ht_foreach callback: free each ModuleEntry. +fn pxcModuleEntryFreeCb(key: CStr, value: ?*anyopaque, userdata: ?*anyopaque) callconv(.c) void { + _ = key; + _ = userdata; + const e: *ModuleEntry = @ptrCast(@alignCast(value orelse return)); + c.free(e.indices); + c.free(e); +} + +fn pxcModuleEntryGetOrCreate(ht: ?*HashTable, key: CStr) ?*ModuleEntry { + if (ht == null or key == null or key[0] == 0) return null; + if (c.cbm_ht_get(ht, key)) |found| return @ptrCast(@alignCast(found)); + const e: *ModuleEntry = @ptrCast(@alignCast(c.calloc(1, @sizeOf(ModuleEntry)) orelse return null)); + e.cap = 8; + e.indices = interop.callocN(c_int, ix(e.cap)); + if (e.indices == null) { + c.free(e); + return null; + } + _ = c.cbm_ht_set(ht, key, e); + return e; +} + +fn pxcModuleEntryAddIndex(entry: ?*ModuleEntry, index: c_int) void { + const e = entry orelse return; + if (e.count >= e.cap) { + const new_cap = e.cap * 2; + const new_indices = interop.reallocN(c_int, e.indices, ix(new_cap)); + if (new_indices == null) return; + e.indices = new_indices; + e.cap = new_cap; + } + e.indices[ix(e.count)] = index; + e.count += 1; +} + +fn pxcMarkEntryDefs(selected: [*c]bool, entry: ?*const ModuleEntry, all_defs: [*c]const LspDef, caller_lang: Language) c_int { + const e = entry orelse return 0; + if (selected == null) return 0; + var added: c_int = 0; + for (0..ix(e.count)) |j| { + const idx = ix(e.indices[j]); + if (!pxcDefLangMatches(caller_lang, all_defs[idx].lang) or selected[idx]) continue; + selected[idx] = true; + added += 1; + } + return added; +} + +fn pxcIsJvmLang(lang: Language) bool { + return lang == c.CBM_LANG_JAVA or lang == c.CBM_LANG_KOTLIN; +} + +fn pxcDefLangMatches(caller_lang: Language, def_lang: Language) bool { + if (pxcIsJvmLang(caller_lang)) return pxcIsJvmLang(def_lang); + return true; +} + +fn pxcMarkModuleDefs( + idx: *const ModuleDefIndex, + selected: [*c]bool, + all_defs: [*c]const LspDef, + caller_lang: Language, + module_qn: CStr, + total: *c_int, +) void { + if (idx.ht == null or module_qn == null or module_qn[0] == 0) return; + const e: ?*ModuleEntry = @ptrCast(@alignCast(c.cbm_ht_get(idx.ht, module_qn))); + total.* += pxcMarkEntryDefs(selected, e, all_defs, caller_lang); +} + +/// Zig: an alias names a module other than the file's imports (a re-export +/// `pub const Database = zova.Database;`, or `const sqlite = @import(..)` that a +/// return type text spells), so a Zig file also gets the defs of each module a +/// selected alias names, transitively. Returns the number of defs added. +const PXC_ZIG_ALIAS_ROUNDS = 8; +fn pxcMarkZigAliasModules(idx: *const ModuleDefIndex, selected: [*c]bool, all_defs: [*c]const LspDef) c_int { + var added_total: c_int = 0; + for (0..PXC_ZIG_ALIAS_ROUNDS) |_| { + var added: c_int = 0; + for (0..ix(idx.def_count)) |i| { + const d = util.at(all_defs, i); + if (!selected[i] or d.lang != c.CBM_LANG_ZIG or d.return_types == null or + d.label == null or !streq(d.label, "Variable")) + { + continue; + } + var buf: [c.CBM_SZ_512]u8 = undefined; + _ = c.snprintf(&buf, buf.len, "%s", d.return_types); + while (true) { + const e: ?*ModuleEntry = @ptrCast(@alignCast(c.cbm_ht_get(idx.ht, &buf))); + if (e != null) { + added += pxcMarkEntryDefs(selected, e, all_defs, c.CBM_LANG_ZIG); + break; + } + const dot: MutStr = c.strrchr(&buf, '.'); + if (dot == null) break; + dot[0] = 0; + } + } + added_total += added; + if (added == 0) break; + } + return added_total; +} + +pub export fn cbm_pxc_build_module_def_index(all_defs: [*c]LspDef, def_count: c_int) ?*ModuleDefIndex { + if (all_defs == null or def_count <= 0) return null; + + const ht = c.cbm_ht_create(64); + const namespace_ht = c.cbm_ht_create(64); + if (ht == null or namespace_ht == null) { + c.cbm_ht_free(ht); + c.cbm_ht_free(namespace_ht); + return null; + } + + // Single pass: index each def by file module and by declared package. + // JVM mixed roots (`src/main/java` + `src/main/kotlin`) share the + // declared package, not the path-derived module prefix. + for (0..ix(def_count)) |i| { + pxcModuleEntryAddIndex(pxcModuleEntryGetOrCreate(ht, all_defs[i].def_module_qn), @intCast(i)); + pxcModuleEntryAddIndex(pxcModuleEntryGetOrCreate(namespace_ht, all_defs[i].namespace_name), @intCast(i)); + } + + const idx: ?*ModuleDefIndex = @ptrCast(@alignCast(c.calloc(1, @sizeOf(ModuleDefIndex)))); + if (idx == null) { + c.cbm_ht_foreach(ht, pxcModuleEntryFreeCb, null); + c.cbm_ht_free(ht); + c.cbm_ht_foreach(namespace_ht, pxcModuleEntryFreeCb, null); + c.cbm_ht_free(namespace_ht); + return null; + } + idx.?.ht = ht; + idx.?.namespace_ht = namespace_ht; + idx.?.def_count = def_count; + return idx; +} + +pub export fn cbm_pxc_free_module_def_index(idx_c: ?*ModuleDefIndex) void { + const idx = idx_c orelse return; + if (idx.ht != null) { + c.cbm_ht_foreach(idx.ht, pxcModuleEntryFreeCb, null); + c.cbm_ht_free(idx.ht); + } + if (idx.namespace_ht != null) { + c.cbm_ht_foreach(idx.namespace_ht, pxcModuleEntryFreeCb, null); + c.cbm_ht_free(idx.namespace_ht); + } + c.free(idx); +} + +pub export fn cbm_pxc_filter_defs_for_file( + idx_c: ?*const ModuleDefIndex, + all_defs: [*c]LspDef, + caller_lang: Language, + caller_namespace: CStr, + own_module: CStr, + imp_qns: [*c]const CStr, + imp_count: c_int, + out_count: ?*c_int, +) [*c]LspDef { + if (out_count) |n| n.* = 0; + const idx = idx_c orelse return null; + const out_n = out_count orelse return null; + if (idx.ht == null or all_defs == null or idx.def_count <= 0) return null; + + const selected = interop.callocN(bool, ix(idx.def_count)); + if (selected == null) return null; + + var total: c_int = 0; + pxcMarkModuleDefs(idx, selected, all_defs, caller_lang, own_module, &total); + for (0..ix(imp_count)) |i| { + pxcMarkModuleDefs(idx, selected, all_defs, caller_lang, imp_qns[i], &total); + } + if (caller_lang == c.CBM_LANG_ZIG) total += pxcMarkZigAliasModules(idx, selected, all_defs); + if (pxcIsJvmLang(caller_lang) and caller_namespace != null and caller_namespace[0] != 0 and + idx.namespace_ht != null) + { + const e: ?*ModuleEntry = @ptrCast(@alignCast(c.cbm_ht_get(idx.namespace_ht, caller_namespace))); + total += pxcMarkEntryDefs(selected, e, all_defs, caller_lang); + } + + if (total == 0) { + c.free(selected); + return null; + } + + const out = interop.mallocN(LspDef, ix(total)); + if (out == null) { + c.free(selected); + return null; + } + + var n: usize = 0; + for (0..ix(idx.def_count)) |i| { + if (selected[i]) { + out[n] = all_defs[i]; + n += 1; + } + } + out_n.* = @intCast(n); + c.free(selected); + return out; +} diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c deleted file mode 100644 index c4040f5d..00000000 --- a/src/pipeline/pass_parallel.c +++ /dev/null @@ -1,3808 +0,0 @@ -/* - * pass_parallel.c — Three-phase parallel pipeline. - * - * Phase 3A: Parallel extract + create definition nodes (per-worker gbufs) - * Phase 3B: Serial registry build + edge creation from cached results - * Phase 4: Parallel call/usage/semantic resolution (per-worker edge bufs) - * - * Each file is read and parsed ONCE (Phase 3A). The CBMFileResult is cached - * and reused for resolution (Phase 4), eliminating 3x redundant I/O + parsing. - * - * Depends on: worker_pool, graph_buffer (shared IDs + merge), extraction (cbm.h) - */ -#include "foundation/constants.h" - -enum { - PP_RESOLVE_COUNTERS = 5, /* int fields of resolve_worker_state_t */ - PP_JSON_MARGIN = 10, - PP_ESC_MARGIN = 3, - PP_ESC_SPACE = 2, - /* Fixed bytes around a serialized JSON field: ,"key":"value" / ,"key":[...] - * -> comma + 2 key quotes + colon + 2 value quotes (resp. brackets). */ - PP_JSON_FIELD_OVERHEAD = 6, - PP_ARGS_MARGIN = 20, - /* ,"line": -> comma + key (7) + colon + up to 10 digits + NUL. */ - PP_LINE_MARGIN = 24, - PP_LOG_THRESH = 24, - PP_LOG_INTERVAL = 10, - PP_TIMER_THRESH = 1000, - /* Extraction memory back-pressure: when the process is over its RSS budget, - * a worker reclaims + naps before pulling another file so peers can finish - * and return pages. Bounded spins avoid deadlock when the resident graph - * itself is near budget: then ONE confirmation cycle decides — drained - * means continue, still over means the attempt fails whole (upstream - * decision A, #1997 #832) instead of the former open-ended soft overshoot. */ - PP_BACKPRESSURE_MAX_SPINS = 40, - PP_BACKPRESSURE_NAP_NS = 3000000, /* 3 ms */ -}; -#define PP_NSEC_PER_SEC 1000000000ULL -#define PP_USEC_PER_MS 1000000ULL -#define PP_HALF_CONF 0.5 -#define PP_FIELD_HINT_CONF 0.85 -enum { PP_CSHARP_M_PREFIX_LEN = 2 }; - -/* Absolute source-retention ceilings for the parallel extract pipeline. - * - * The extract worker copies each file's source bytes into result->arena so - * the fused cross-file LSP step in resolve_worker can re-parse without - * re-opening the file. That retention is TRANSIENT (freed at run end) but it - * is a PEAK-RSS driver: every retained byte is resident at once across the - * extract→resolve handoff. - * - * A cap here is a FLOOR as much as a ceiling: whatever total we allow, we - * WILL hold that much resident at peak on a large repo. rust-analyzer bounds - * retained file *text* to a small fixed budget and re-reads source on a miss - * rather than scaling the retained set with host RAM — because the re-read is - * cheap relative to holding tens of GB resident. We follow the same model: - * - derive the total budget from the process memory budget (budget/8), - * - BUT clamp the RAM-derived DEFAULT to a small absolute ceiling (1 GiB) - * so a 512 GiB host does not retain 64 GiB of source it would re-read - * cheaply anyway, and keep the per-file cap modest (32 MiB) so one - * pathological generated blob cannot monopolise the budget. - * A file dropped from retention is NOT lost to cross-file resolution: - * resolve_worker re-reads it on demand, bounded and freed immediately - * (source_reread fallback). Both caps are env-overridable — CBM_RETAIN_TOTAL_MB - * / CBM_RETAIN_PER_FILE_MB raise or lower the auto-derived defaults directly - * (the hard ceilings bound only the RAM-derived default, never a deliberate - * operator/caller choice). */ -#define PP_RETAIN_TOTAL_HARD_MAX_BYTES (1024ULL * 1024 * 1024) /* 1 GiB default ceiling */ -#define PP_RETAIN_PER_FILE_HARD_MAX_BYTES (32ULL * 1024 * 1024) /* 32 MiB per file */ -#include "pipeline/pipeline.h" -#include "pipeline/pipeline_internal.h" -#include "pipeline/pass_lsp_cross.h" /* cbm_pxc_* helpers for fused cross-file LSP */ -#include "pipeline/lsp_resolve.h" -#include "pipeline/laravel_routing.h" -#include "lsp/rust_cargo.h" -#include "helpers.h" /* cbm_kind_in_set_free_cache — per-worker-thread cache teardown */ -#include "pipeline/worker_pool.h" -#include "foundation/compat.h" -#include "foundation/compat_fs.h" -#include "foundation/compat_thread.h" -#include "graph_buffer/graph_buffer.h" -#include "service_patterns.h" -#include "foundation/platform.h" -#include "foundation/hash_table.h" -#include "foundation/log.h" -#include "foundation/slab_alloc.h" -#include "foundation/mem.h" -#include "foundation/str_util.h" -#include "foundation/profile.h" -#include "foundation/compat_regex.h" -#include "foundation/limits.h" -#include "cbm.h" -#include "simhash/minhash.h" -#include "semantic/ast_profile.h" - -#include -#include -#include -#include -#include -#include -#include - -/* Back-pressure nap-cycle counter (test observability): each execution of the - * over-budget collect+nap gate counts one cycle. Lets tests assert the gate does - * not re-pay the full nap tax on every file pull when napping cannot reclaim - * memory (the resident floor, not in-flight transients, holds the budget). */ -static _Atomic long g_bp_nap_cycles = 0; -/* Defined here, declared in lsp_resolve.h: the uncapped tail-match scan's - * cost, logged at the end of parallel resolve. */ -_Atomic uint64_t g_lsp_tail_lookups = 0; -_Atomic uint64_t g_lsp_tail_candidates = 0; - -long cbm_pp_bp_nap_cycles(void) { - return atomic_load_explicit(&g_bp_nap_cycles, memory_order_relaxed); -} - -void cbm_pp_bp_nap_cycles_reset(void) { - atomic_store_explicit(&g_bp_nap_cycles, 0, memory_order_relaxed); -} - -/* Test seam: when set, a parallel extract with a spill store spills from its - * first file instead of waiting for the budget signal. */ -static _Atomic int g_force_spill = 0; -/* Results parked by the parallel extract (test observability). */ -static _Atomic long g_spill_parked = 0; - -void cbm_parallel_force_spill_for_test(int on) { - atomic_store_explicit(&g_force_spill, on ? 1 : 0, memory_order_relaxed); -} - -long cbm_pp_spill_parked(void) { - return atomic_load_explicit(&g_spill_parked, memory_order_relaxed); -} - -void cbm_pp_spill_parked_reset(void) { - atomic_store_explicit(&g_spill_parked, 0, memory_order_relaxed); -} - -/* Parse a positive MB-valued retention env knob (CBM_RETAIN_*_MB) into bytes. - * Follows the limits.c strtol convention: unset / unparseable / non-positive - * → return 0 so the caller keeps its derived default. */ -static size_t cbm_retain_env_bytes(const char *name) { - const char *raw = getenv(name); - if (!raw || !raw[0]) { - return 0; - } - errno = 0; - char *end = NULL; - long v = strtol(raw, &end, 10); - if (errno != 0 || end == raw || *end != '\0' || v <= 0) { - return 0; - } - return (size_t)v * 1024 * 1024; -} - -/* Auto-derived TOTAL retention budget: a fraction of the process memory - * budget, clamped to the absolute ceiling. The clamp bounds ONLY this - * RAM-derived default — a huge-RAM host must not retain tens of GB it would - * re-read cheaply. When the budget is unset (tests / no cgroup) fall back to - * the ceiling. */ -static size_t cbm_parallel_extract_default_total_cap(void) { - size_t cap = cbm_mem_budget(); - if (cap > 0) { - cap /= 8; - } else { - cap = PP_RETAIN_TOTAL_HARD_MAX_BYTES; - } - if (cap > PP_RETAIN_TOTAL_HARD_MAX_BYTES) { - cap = PP_RETAIN_TOTAL_HARD_MAX_BYTES; - } - return cap; -} - -/* Auto-derived PER-FILE cap: modest absolute ceiling, never above the total. */ -static size_t cbm_parallel_extract_default_per_file_cap(size_t total_cap) { - size_t cap = PP_RETAIN_PER_FILE_HARD_MAX_BYTES; - if (cap > total_cap) { - cap = total_cap; - } - return cap; -} - -/* Resolve the effective retention options from (in precedence order): - * explicit caller opts > CBM_RETAIN_*_MB env knobs > RAM-derived defaults. - * The absolute hard ceilings apply only to the RAM-derived defaults; an - * operator/caller that sets a value explicitly is trusted. The only invariant - * enforced unconditionally is per-file ≤ total. */ -static cbm_parallel_extract_opts_t cbm_parallel_extract_resolve_opts( - const cbm_parallel_extract_opts_t *opts) { - cbm_parallel_extract_opts_t resolved = { - .retain_sources = true, - .retain_sources_set = true, - .retain_total_budget_bytes = cbm_parallel_extract_default_total_cap(), - .retain_per_file_max_bytes = 0, - }; - - size_t env_total = cbm_retain_env_bytes("CBM_RETAIN_TOTAL_MB"); - if (env_total > 0) { - resolved.retain_total_budget_bytes = env_total; - } - - resolved.retain_per_file_max_bytes = - cbm_parallel_extract_default_per_file_cap(resolved.retain_total_budget_bytes); - size_t env_per_file = cbm_retain_env_bytes("CBM_RETAIN_PER_FILE_MB"); - if (env_per_file > 0) { - resolved.retain_per_file_max_bytes = env_per_file; - } - - if (opts) { - if (opts->retain_sources_set) { - resolved.retain_sources = opts->retain_sources; - } - if (opts->retain_total_budget_bytes > 0) { - resolved.retain_total_budget_bytes = opts->retain_total_budget_bytes; - } - if (opts->retain_per_file_max_bytes > 0) { - resolved.retain_per_file_max_bytes = opts->retain_per_file_max_bytes; - } - } - - /* Correctness invariant: a single file can never exceed the total budget. */ - if (resolved.retain_per_file_max_bytes > resolved.retain_total_budget_bytes) { - resolved.retain_per_file_max_bytes = resolved.retain_total_budget_bytes; - } - return resolved; -} - -static uint64_t extract_now_ns(void) { - struct timespec ts; - cbm_clock_gettime(CLOCK_MONOTONIC, &ts); - return ((uint64_t)ts.tv_sec * PP_NSEC_PER_SEC) + (uint64_t)ts.tv_nsec; -} - -/* ── Helpers (duplicated from pass files — kept static for isolation) ── */ - -/* Read file into a malloc'd buffer (= mimalloc in production). - * *out_size receives the on-disk size and *out_status the failure reason so the - * caller can attribute a skip to the right phase (read vs oversized) instead of - * a silent drop. Both out params may be NULL. */ -static char *read_file(const char *path, int *out_len, long *out_size, - cbm_read_status_t *out_status) { - if (out_size) { - *out_size = 0; - } - if (out_status) { - *out_status = CBM_READ_OK; - } - FILE *f = cbm_fopen(path, "rb"); - if (!f) { - if (out_status) { - *out_status = CBM_READ_OPEN_FAIL; - } - return NULL; - } - (void)fseek(f, 0, SEEK_END); - long size = ftell(f); - (void)fseek(f, 0, SEEK_SET); - if (out_size) { - *out_size = size; - } - if (size <= 0) { - (void)fclose(f); - if (out_status) { - *out_status = CBM_READ_EMPTY; - } - return NULL; - } - if (size > cbm_max_file_bytes()) { /* generous, env-configurable cap (B4) */ - (void)fclose(f); - if (out_status) { - *out_status = CBM_READ_OVERSIZED; - } - return NULL; - } - /* +pad: tree-sitter lexer lookahead reads past EOF; keep it in-bounds */ - enum { CBM_TS_LOOKAHEAD_PAD = 16 }; - char *buf = (char *)malloc((size_t)size + CBM_TS_LOOKAHEAD_PAD); - if (!buf) { - (void)fclose(f); - if (out_status) { - *out_status = CBM_READ_OOM; - } - return NULL; - } - size_t nread = fread(buf, SKIP_ONE, (size_t)size, f); - (void)fclose(f); - if (nread > (size_t)size) { - nread = (size_t)size; - } - memset(buf + nread, 0, CBM_TS_LOOKAHEAD_PAD); - *out_len = (int)nread; - return buf; -} - -/* ── Per-worker skip list (Stage 2 / Track B) ─────────────────────── - * Each extract worker appends read/extract/oversized skips into its OWN list - * (no lock on the hot path); the lists are merged into the pipeline's - * cbm_file_error_t array in the existing sequential merge loop. */ -typedef struct { - cbm_file_error_t *items; - int count; - int cap; -} pp_err_list_t; - -/* NULL-safe heap strdup. */ -static char *pp_err_dup(const char *s) { - if (!s) { - return NULL; - } - size_t n = strlen(s) + 1; - char *d = (char *)malloc(n); - if (d) { - memcpy(d, s, n); - } - return d; -} - -static void pp_err_add(pp_err_list_t *list, const char *path, const char *reason, - const char *phase) { - if (!list) { - return; - } - if (list->count >= list->cap) { - int ncap = list->cap ? list->cap * 2 : 8; - cbm_file_error_t *grown = - (cbm_file_error_t *)realloc(list->items, (size_t)ncap * sizeof(*grown)); - if (!grown) { - return; /* drop on OOM — never fail extraction to record a skip */ - } - list->items = grown; - list->cap = ncap; - } - list->items[list->count].path = pp_err_dup(path); - list->items[list->count].reason = pp_err_dup(reason); - list->items[list->count].phase = pp_err_dup(phase); - list->count++; -} - -/* Free source buffer. */ -static void free_source(char *buf) { - free(buf); -} - -/* Append a JSON-escaped string value to buf at position *pos. */ -/* Escape one character for JSON. Returns bytes written (1 or 2). */ -static int json_escape_char(char *buf, size_t avail, char ch) { - char esc = 0; - switch (ch) { - case '"': - esc = '"'; - break; - case '\\': - esc = '\\'; - break; - case '\n': - esc = 'n'; - break; - case '\r': - esc = 'r'; - break; - case '\t': - esc = 't'; - break; - default: - if (avail >= SKIP_ONE) { - /* Any other raw control byte (e.g. form feed) is invalid inside a - * JSON string — degrade to a space. */ - buf[0] = ((unsigned char)ch < 0x20) ? ' ' : ch; - } - return SKIP_ONE; - } - if (avail >= PP_ESC_SPACE) { - buf[0] = '\\'; - buf[SKIP_ONE] = esc; - } - return PP_ESC_SPACE; -} - -/* Escaped length of a string under json_escape_char's rules: escaped - * characters expand to 2 bytes, everything else stays 1. */ -static size_t pp_json_escaped_len(const char *s) { - size_t n = 0; - for (; *s; s++) { - switch (*s) { - case '"': - case '\\': - case '\n': - case '\r': - case '\t': - n += PP_ESC_SPACE; - break; - default: - n += SKIP_ONE; - } - } - return n; -} - -/* Appends are ATOMIC: a field is emitted only if the WHOLE serialized form - * fits (with PP_ESC_SPACE bytes reserved for the closing '}' + NUL). Cutting a - * field mid-value produced unterminated strings/arrays — malformed properties - * JSON that aborts every json_extract()-based consumer downstream (seen on the - * Linux kernel: 50-param functions truncated at the 2 KB cap). Dropping an - * oversized optional field whole keeps the JSON valid. */ -static void append_json_string(char *buf, size_t bufsize, size_t *pos, const char *key, - const char *val) { - if (!val || val[0] == '\0') { - return; - } - size_t required = strlen(key) + pp_json_escaped_len(val) + PP_JSON_FIELD_OVERHEAD; - if (*pos + required + PP_ESC_SPACE > bufsize) { - return; /* whole field would not fit — skip it atomically */ - } - size_t p = *pos; - int w = snprintf(buf + p, bufsize - p, ",\"%s\":\"", key); - if (w <= 0 || (size_t)w >= bufsize - p) { - return; - } - p += (size_t)w; - for (const char *s = val; *s && p < bufsize - PP_ESC_MARGIN; s++) { - int n = json_escape_char(buf + p, bufsize - p - PP_ESC_SPACE, *s); - p += (size_t)n; - } - if (p < bufsize - SKIP_ONE) { - buf[p++] = '"'; - } - buf[p] = '\0'; - *pos = p; -} - -/* Append a JSON array of strings: ,"key":["a","b","c"]. Atomic like - * append_json_string: emitted only if the whole array fits. */ -static void append_json_str_array(char *buf, size_t bufsize, size_t *pos, const char *key, - const char **arr) { - if (!arr || !arr[0] || *pos >= bufsize - PP_JSON_MARGIN) { - return; - } - /* ,"key":[ + per item "" + separating commas + ] */ - size_t required = strlen(key) + PP_JSON_FIELD_OVERHEAD; - for (int i = 0; arr[i]; i++) { - required += pp_json_escaped_len(arr[i]) + PP_ESC_SPACE + (i > 0 ? SKIP_ONE : 0); - } - if (*pos + required + PP_ESC_SPACE > bufsize) { - return; /* whole array would not fit — skip it atomically */ - } - size_t p = *pos; - int n = snprintf(buf + p, bufsize - p, ",\"%s\":[", key); - if (n <= 0 || p + (size_t)n >= bufsize - PP_ESC_SPACE) { - return; - } - p += (size_t)n; - for (int i = 0; arr[i]; i++) { - if (i > 0 && p < bufsize - SKIP_ONE) { - buf[p++] = ','; - } - if (p < bufsize - SKIP_ONE) { - buf[p++] = '"'; - } - /* Full escaping (not just quote/backslash): items like C param types - * sliced from multi-line declarations carry raw \n/\t bytes, which are - * invalid inside JSON strings. */ - for (const char *s = arr[i]; *s && p < bufsize - PP_ESC_SPACE; s++) { - p += (size_t)json_escape_char(buf + p, bufsize - p - PP_ESC_SPACE, *s); - } - if (p < bufsize - SKIP_ONE) { - buf[p++] = '"'; - } - } - if (p < bufsize - SKIP_ONE) { - buf[p++] = ']'; - } - buf[p] = '\0'; - *pos = p; -} - -/* Append an already-serialized JSON value verbatim: ,"key":. Atomic like - * append_json_string. For `variants`, which extraction builds as a JSON array - * of objects (cbm.h). */ -static void append_json_raw(char *buf, size_t bufsize, size_t *pos, const char *key, - const char *json) { - if (!json || json[0] == '\0') { - return; - } - size_t required = strlen(key) + strlen(json) + PP_JSON_FIELD_OVERHEAD; - if (*pos + required + PP_ESC_SPACE > bufsize) { - return; /* whole field would not fit — skip it atomically */ - } - int w = snprintf(buf + *pos, bufsize - *pos, ",\"%s\":%s", key, json); - if (w > 0 && (size_t)w < bufsize - *pos) { - *pos += (size_t)w; - } -} - -static void build_def_props(char *buf, size_t bufsize, const CBMDefinition *def) { - /* Complexity/loop/recursion metrics are meaningful only for Function/Method. - * Gate the block so the millions of Macro/Field/Variable/Class/Enum nodes - * keep a lean properties blob (lossless — those fields are always zero for - * non-functions). Cuts RAM, gbuf-merge copy and dump volume. */ - const bool is_fn = - def->label && (strcmp(def->label, "Function") == 0 || strcmp(def->label, "Method") == 0); - int n; - if (is_fn) { - n = snprintf(buf, bufsize, - "{\"complexity\":%d,\"cognitive\":%d,\"loop_count\":%d,\"loop_depth\":%d," - "\"self_recursive\":%s,\"param_count\":%d,\"max_access_depth\":%d," - "\"linear_scan_in_loop\":%d,\"alloc_in_loop\":%d,\"recursion_in_loop\":%s," - "\"unguarded_recursion\":%s," - "\"lines\":%d,\"is_exported\":%s,\"is_test\":%s,\"is_entry_point\":%s", - def->complexity, def->cognitive, def->loop_count, def->loop_depth, - def->is_recursive ? "true" : "false", def->param_count, def->max_access_depth, - def->linear_scan_in_loop, def->alloc_in_loop, - def->recursion_in_loop ? "true" : "false", - def->unguarded_recursion ? "true" : "false", def->lines, - def->is_exported ? "true" : "false", def->is_test ? "true" : "false", - def->is_entry_point ? "true" : "false"); - } else { - n = snprintf(buf, bufsize, - "{\"complexity\":%d,\"lines\":%d,\"is_exported\":%s,\"is_test\":%s," - "\"is_entry_point\":%s", - def->complexity, def->lines, def->is_exported ? "true" : "false", - def->is_test ? "true" : "false", def->is_entry_point ? "true" : "false"); - } - if (n <= 0 || (size_t)n >= bufsize) { - buf[0] = '\0'; - return; - } - size_t pos = (size_t)n; - append_json_string(buf, bufsize, &pos, "docstring", def->docstring); - /* Right after the docstring, ahead of the bounded fields, so a long - * signature cannot squeeze it out of the fixed buffer. */ - append_json_raw(buf, bufsize, &pos, "variants", def->variants); - append_json_string(buf, bufsize, &pos, "signature", def->signature); - append_json_string(buf, bufsize, &pos, "return_type", def->return_type); - append_json_string(buf, bufsize, &pos, "parent_class", def->parent_class); - append_json_str_array(buf, bufsize, &pos, "decorators", def->decorators); - append_json_str_array(buf, bufsize, &pos, "base_classes", def->base_classes); - append_json_str_array(buf, bufsize, &pos, "param_names", def->param_names); - append_json_str_array(buf, bufsize, &pos, "param_types", def->param_types); - append_json_string(buf, bufsize, &pos, "route_path", def->route_path); - append_json_string(buf, bufsize, &pos, "route_method", def->route_method); - append_json_string(buf, bufsize, &pos, "http_client", def->http_client); - append_json_string(buf, bufsize, &pos, "http_base_url", def->http_base_url); - - /* MinHash fingerprint — append if present and buffer has room. - * Hex-encoded K=64 uint32 = 512 chars + key/quotes ≈ 520 chars. */ - if (def->fingerprint && def->fingerprint_k > 0 && - pos + CBM_MINHASH_HEX_LEN + CBM_MINHASH_JSON_OVERHEAD < bufsize) { - char fp_hex[CBM_MINHASH_HEX_BUF]; - cbm_minhash_to_hex((const cbm_minhash_t *)def->fingerprint, fp_hex, sizeof(fp_hex)); - append_json_string(buf, bufsize, &pos, "fp", fp_hex); - } - - /* AST structural profile — append if present and buffer has room. */ - if (def->structural_profile && pos + CBM_AST_PROFILE_BUF < bufsize) { - append_json_string(buf, bufsize, &pos, "sp", def->structural_profile); - } - - /* Body tokens — raw identifiers from function body AST for semantic search. */ - if (def->body_tokens && pos + CBM_SZ_512 < bufsize) { - append_json_string(buf, bufsize, &pos, "bt", def->body_tokens); - } - - if (pos < bufsize - SKIP_ONE) { - buf[pos] = '}'; - buf[pos + SKIP_ONE] = '\0'; - } -} - -/* Build import map from graph buffer IMPORTS edges (read-only access to gbuf). */ -static int build_import_map(const cbm_gbuf_t *gbuf, const char *project_name, const char *rel_path, - const char ***out_keys, const char ***out_vals, int *out_count) { - *out_keys = NULL; - *out_vals = NULL; - *out_count = 0; - - char *file_qn = cbm_pipeline_fqn_compute(project_name, rel_path, "__file__"); - const cbm_gbuf_node_t *file_node = cbm_gbuf_find_by_qn(gbuf, file_qn); - free(file_qn); - if (!file_node) { - return 0; - } - - const cbm_gbuf_edge_t **edges = NULL; - int edge_count = 0; - int rc = - cbm_gbuf_find_edges_by_source_type(gbuf, file_node->id, "IMPORTS", &edges, &edge_count); - if (rc != 0 || edge_count == 0) { - return 0; - } - - const char **keys = calloc(edge_count, sizeof(const char *)); - const char **vals = calloc(edge_count, sizeof(const char *)); - int count = 0; - - for (int i = 0; i < edge_count; i++) { - const cbm_gbuf_edge_t *e = edges[i]; - const cbm_gbuf_node_t *target = cbm_gbuf_find_by_id(gbuf, e->target_id); - if (!target || !e->properties_json) { - continue; - } - const char *start = strstr(e->properties_json, "\"local_name\":\""); - if (start) { - start += strlen("\"local_name\":\""); - const char *end = strchr(start, '"'); - if (end && end > start) { - keys[count] = cbm_strndup(start, end - start); - vals[count] = target->qualified_name; - count++; - } - } - } - - *out_keys = keys; - *out_vals = vals; - *out_count = count; - return 0; -} - -static void free_import_map(const char **keys, const char **vals, int count) { - if (keys) { - for (int i = 0; i < count; i++) { - free((void *)keys[i]); - } - free((void *)keys); - } - if (vals) { - free((void *)vals); - } -} - -/* True for languages whose module QN derives from the CONTAINING DIRECTORY - * (Java/Go package). MUST match cbm_lang_module_is_dir() (internal/cbm/helpers.c) - * and pxc_module_is_dir() (pass_lsp_cross.c) so same-module callee resolution - * keys against the directory-based def-node QNs in the registry. */ -static bool pp_module_is_dir(CBMLanguage lang) { - return lang == CBM_LANG_JAVA || lang == CBM_LANG_GO; -} - -static bool is_checked_exception(const char *name) { - if (!name) { - return false; - } - if (strstr(name, "Error") || strstr(name, "Panic") || strstr(name, "error") || - strstr(name, "panic")) { - return false; - } - return true; -} - -static const char *resolve_as_class(const cbm_registry_t *reg, const char *name, - const char *module_qn, const char **imp_keys, - const char **imp_vals, int imp_count) { - cbm_resolution_t res = - cbm_registry_resolve(reg, name, module_qn, imp_keys, imp_vals, imp_count); - if (!res.qualified_name || res.qualified_name[0] == '\0') { - return NULL; - } - /* Accept any type-like container (Class/Struct/Interface/Enum/Type/Trait): - * base classes, Rust `impl Trait for S` struct receivers, and Go struct - * embedding all resolve through here. Struct included so the struct receiver - * of an IMPLEMENTS edge is not dropped. */ - const char *label = cbm_registry_label_of(reg, res.qualified_name); - if (!cbm_label_is_type_like(label)) { - return NULL; - } - return res.qualified_name; -} - -static void extract_decorator_func(const char *dec, char *out, size_t outsz) { - out[0] = '\0'; - if (!dec) { - return; - } - const char *start = dec; - if (*start == '@') { - start++; - } - const char *paren = strchr(start, '('); - size_t len = paren ? (size_t)(paren - start) : strlen(start); - if (len == 0 || len >= outsz) { - return; - } - memcpy(out, start, len); - out[len] = '\0'; -} - -/* ── File sort for tail-latency reduction ────────────────────────── */ - -typedef struct { - int idx; - int64_t size; -} file_sort_entry_t; - -static int compare_by_size_desc(const void *a, const void *b) { - const file_sort_entry_t *fa = a; - const file_sort_entry_t *fb = b; - if (fb->size > fa->size) { - return SKIP_ONE; - } - if (fb->size < fa->size) { - return CBM_NOT_FOUND; - } - return 0; -} - -/* ── Phase 3A: Parallel Extract ──────────────────────────────────── */ - -#define CBM_CACHE_LINE CBM_SZ_128 - -typedef struct __attribute__((aligned(CBM_CACHE_LINE))) { - cbm_gbuf_t *local_gbuf; - int nodes_created; - int errors; - char _pad[CBM_CACHE_LINE - sizeof(cbm_gbuf_t *) - (PP_ESC_SPACE * sizeof(int))]; -} extract_worker_state_t; - -typedef struct { - const cbm_file_info_t *files; - file_sort_entry_t *sorted; - int file_count; - const char *project_name; - const char *repo_path; - - extract_worker_state_t *workers; - int max_workers; - _Atomic int next_worker_id; - - CBMFileResult **result_cache; - _Atomic int64_t *shared_ids; - _Atomic int *cancelled; - _Atomic int next_file_idx; - - cbm_pkg_entries_t *pkg_entries; /* per-worker manifest arrays (separate allocation) */ - const cbm_stem_marks_t *stem_marks; /* which Module definitions take `qn#ext` */ - - bool retain_sources; /* copy source into result->arena for cross-file LSP */ - size_t retain_total_budget_bytes; /* project-wide retention cap (peak-RSS bound) */ - size_t retain_per_file_max_bytes; /* per-file retention cap */ - _Atomic int64_t retained_bytes; /* total source bytes copied into result arenas */ - _Atomic int retain_cap_warned; /* WARN index.retain_capped emitted once per run */ - - /* Per-worker skip lists (separate allocation, indexed by worker_id — no hot- - * path lock). Merged into the pipeline in the sequential merge loop. */ - pp_err_list_t *err_lists; - _Atomic int oversized_warned; /* throttle for the index.file_oversized WARN */ - - /* Back-pressure futility latch: set when a full collect+nap cycle ended - * still over budget — the resident floor (graph + retained sources), not - * in-flight transients, holds the memory, so napping cannot reclaim it. - * While set, pulls skip the nap (the designed soft overshoot); the cheap - * over-budget probe re-arms the gate once RSS drains under budget. */ - _Atomic int bp_futile; - /* Set once futility was CONFIRMED by a second full cycle that still ended - * over budget. Every pull loop breaks on it and extract returns - * CBM_PIPELINE_ABORT_OVER_BUDGET. Deliberately not the shared cancel - * token — that one belongs to client cancellation. */ - _Atomic int over_budget_abort; - - /* Spill (F1). With a store, the first futility latch turns spilling on - * instead of failing the attempt: every finished result is then parked on - * disk and its result_cache slot cleared. result_owner[file_idx] is the - * extracting worker id + 1, so each worker parks its own backlog once - * (its own allocations, which its next cbm_mem_collect returns). Only a - * latch that fires again while spilling fails the attempt whole. */ - cbm_result_spill_t *spill; - _Atomic int *result_owner; - _Atomic int spilling; -} extract_ctx_t; - -/* Cap on the number of index.file_oversized WARN lines (the full list still goes - * to the response/logfile — this only throttles the stderr noise). */ -enum { PP_OVERSIZED_WARN_MAX = 32 }; - -/* Insert one definition node (and its route if present) into the local gbuf. */ -static void insert_def_into_gbuf(extract_worker_state_t *ws, const cbm_stem_marks_t *marks, - const cbm_file_info_t *fi, CBMDefinition *def) { - char props[CBM_SZ_2K]; - build_def_props(props, sizeof(props), def); - const char *rel = def->file_path ? def->file_path : fi->rel_path; - char marked[CBM_SZ_4K]; - const char *qn = def->label && strcmp(def->label, "Module") == 0 - ? cbm_stem_marks_module_qn(marks, def->qualified_name, rel, fi->language, - marked, sizeof(marked)) - : def->qualified_name; - int64_t func_id = - cbm_gbuf_upsert_node(ws->local_gbuf, def->label ? def->label : "Function", def->name, qn, - rel, (int)def->start_line, (int)def->end_line, props); - ws->nodes_created++; - if (def->route_path && def->route_path[0] != '\0') { - const char *rm = def->route_method ? def->route_method : "ANY"; - char route_qn[CBM_ROUTE_QN_SIZE]; - char cpath[CBM_SZ_256]; - snprintf(route_qn, sizeof(route_qn), "__route__%s__%s", rm, - cbm_route_canon_path(def->route_path, cpath, sizeof(cpath))); - char rprops[CBM_SZ_256]; - snprintf(rprops, sizeof(rprops), "{\"method\":\"%s\",\"source\":\"decorator\"}", rm); - int64_t route_id = - cbm_gbuf_upsert_node(ws->local_gbuf, "Route", def->route_path, route_qn, - def->file_path ? def->file_path : fi->rel_path, 0, 0, rprops); - char hprops[CBM_SZ_512]; - char esc_h[CBM_SZ_512]; - cbm_json_escape(esc_h, sizeof(esc_h), def->qualified_name); - snprintf(hprops, sizeof(hprops), "{\"handler\":\"%s\"}", esc_h); - cbm_gbuf_insert_edge(ws->local_gbuf, func_id, route_id, "HANDLES", hprops); - } -} - -static void log_extract_fail(int pos, uint64_t ms, const char *path) { - if (pos < PP_LOG_THRESH) { - cbm_log_warn("parallel.extract.file.fail", "pos", cbm_log_itoa(pos), "elapsed_ms", - cbm_log_itoa((int)ms), "path", path); - } -} - -static void log_extract_done(int pos, uint64_t ms, int defs, const char *path) { - if (pos < PP_LOG_THRESH || ms > PP_TIMER_THRESH) { - cbm_log_info("parallel.extract.file.done", "pos", cbm_log_itoa(pos), "elapsed_ms", - cbm_log_itoa((int)ms), "defs", cbm_log_itoa(defs), "path", path); - } -} - -/* One back-pressure cycle: reclaim this thread's freed pages, then nap in - * bounded 3 ms steps while the process stays over budget. Counted for test - * observability. Returns true when the FULL cycle elapsed still over budget — - * the resident floor, not in-flight transients, holds the memory. */ -static bool pp_backpressure_cycle_still_over(extract_ctx_t *ec) { - cbm_mem_collect(); - atomic_fetch_add_explicit(&g_bp_nap_cycles, SKIP_ONE, memory_order_relaxed); - int bp = 0; - for (; bp < PP_BACKPRESSURE_MAX_SPINS && cbm_mem_over_budget() && - !atomic_load_explicit(ec->cancelled, memory_order_relaxed) && - !atomic_load_explicit(&ec->over_budget_abort, memory_order_relaxed); - bp++) { - struct timespec nap = {0, PP_BACKPRESSURE_NAP_NS}; - cbm_nanosleep(&nap, NULL); - } - return bp == PP_BACKPRESSURE_MAX_SPINS && cbm_mem_over_budget(); -} - -/* Back-pressure was futile and the confirmation cycle still ended over budget. - * The budget stops being advisory here: flag the attempt so every worker stops - * pulling and extract returns CBM_PIPELINE_ABORT_OVER_BUDGET. Nothing is - * published, so the previously served Zova generation keeps answering. */ -static void pp_fail_whole_over_budget(extract_ctx_t *ec) { - if (atomic_exchange_explicit(&ec->over_budget_abort, 1, memory_order_relaxed) == 0) { - cbm_log_error("mem.budget.exceeded", "rss_mb", - cbm_log_itoa((int)(cbm_mem_rss() / (1024 * 1024))), "budget_mb", - cbm_log_itoa((int)(cbm_mem_budget() / (1024 * 1024))), "phase", - "parallel_extract", "action", "fail_whole"); - } -} - -static bool pp_park(extract_ctx_t *ec, int file_idx, CBMFileResult *result) { - if (!cbm_result_spill_park(ec->spill, file_idx, result)) { - return false; - } - atomic_fetch_add_explicit(&g_spill_parked, SKIP_ONE, memory_order_relaxed); - return true; -} - -/* Park every result this worker extracted that is still in memory. */ -static void pp_spill_backlog(extract_ctx_t *ec, int worker_id) { - for (int i = 0; i < ec->file_count; i++) { - if (atomic_load_explicit(&ec->result_owner[i], memory_order_relaxed) != worker_id + 1) { - continue; - } - if (ec->result_cache[i] && pp_park(ec, i, ec->result_cache[i])) { - ec->result_cache[i] = NULL; - } - } - cbm_mem_collect(); -} - -/* The budget gate's futility latch fired. With a spill store that is not yet - * spilling, turn spilling on and re-arm the gate; otherwise run the B3 - * confirmation cycle and fail the attempt whole if it still ends over. */ -static void pp_backpressure_futile(extract_ctx_t *ec, int worker_id) { - if (ec->spill && atomic_exchange_explicit(&ec->spilling, 1, memory_order_relaxed) == 0) { - cbm_log_warn("mem.spill.engaged", "rss_mb", - cbm_log_itoa((int)(cbm_mem_rss() / (1024 * 1024))), "budget_mb", - cbm_log_itoa((int)(cbm_mem_budget() / (1024 * 1024)))); - pp_spill_backlog(ec, worker_id); - atomic_store_explicit(&ec->bp_futile, 0, memory_order_relaxed); - return; - } - cbm_log_warn("mem.backpressure.futile", "action", "confirm"); - if (pp_backpressure_cycle_still_over(ec)) { - pp_fail_whole_over_budget(ec); - } else { - atomic_store_explicit(&ec->bp_futile, 0, memory_order_relaxed); - } -} - -static void extract_worker(int worker_id, void *ctx_ptr) { - extract_ctx_t *ec = ctx_ptr; - extract_worker_state_t *ws = &ec->workers[worker_id]; - bool backlog_spilled = false; - - /* Lazy gbuf creation */ - if (!ws->local_gbuf) { - ws->local_gbuf = cbm_gbuf_new_shared_ids(ec->project_name, ec->repo_path, ec->shared_ids); - } - - /* Pull files from shared atomic counter */ - while (SKIP_ONE) { - int sort_pos = - atomic_fetch_add_explicit(&ec->next_file_idx, SKIP_ONE, memory_order_relaxed); - if (sort_pos >= ec->file_count) { - break; - } - if (atomic_load_explicit(ec->cancelled, memory_order_relaxed) || - atomic_load_explicit(&ec->over_budget_abort, memory_order_relaxed)) { - break; - } - - /* Memory back-pressure (large repos): if the process is over its RSS - * budget, reclaim this thread's freed pages and nap so peer workers can - * finish their current file and return memory before this worker adds - * another parse working set. Caps the concurrent extraction transient - * near the budget instead of letting all workers parse their biggest - * files at once. Self-disabling when the budget is unset (tests) or RSS - * is under budget; bounded spins avoid deadlock when the resident graph - * is itself near budget. - * - * Futility latch: when a FULL nap cycle ends still over budget, the - * resident floor — not transients — holds the memory; napping again on - * the next pull cannot reclaim it and only idles workers (linux kernel: - * one full cycle per pull ≈ 390 s at 79% avg CPU). The worker that - * latches bp_futile (0→1) runs ONE confirmation cycle while its peers - * skip the nap. A confirmation that ends under budget re-arms the gate - * (RSS drained after all); one that still ends over budget fails the - * attempt whole: every worker stops pulling, extract returns - * CBM_PIPELINE_ABORT_OVER_BUDGET, nothing is published and the - * previously served generation keeps answering. */ - if (cbm_mem_budget() > 0) { - bool over = cbm_mem_over_budget(); - bool futile = atomic_load_explicit(&ec->bp_futile, memory_order_relaxed) != 0; - if (over && !futile) { - /* Act only on the 0→1 transition: all workers race into the - * gate before anyone latches, so a plain store would confirm - * (and WARN) once per worker. */ - if (pp_backpressure_cycle_still_over(ec) && - atomic_exchange_explicit(&ec->bp_futile, 1, memory_order_relaxed) == 0) { - pp_backpressure_futile(ec, worker_id); - } - } else if (!over && futile) { - atomic_store_explicit(&ec->bp_futile, 0, memory_order_relaxed); - } - } - if (atomic_load_explicit(&ec->over_budget_abort, memory_order_relaxed)) { - break; - } - const bool spilling = atomic_load_explicit(&ec->spilling, memory_order_relaxed) != 0; - if (spilling && !backlog_spilled) { - pp_spill_backlog(ec, worker_id); - backlog_spilled = true; - } - - int file_idx = ec->sorted[sort_pos].idx; - const cbm_file_info_t *fi = &ec->files[file_idx]; - pp_err_list_t *errs = ec->err_lists ? &ec->err_lists[worker_id] : NULL; - - /* Crash-quarantine skip (Stage 3c): a file the supervisor pinned as a - * crasher must never be extracted again. Record it as a phase="crash" - * skip in this worker's list (merged into the pipeline's file-error list - * later, surfacing in skipped[]) and move on — the good files still - * index and status stays "indexed". No-op unless the supervisor set - * CBM_INDEX_QUARANTINE_FILE. cbm_extract_file's hard guard backstops - * this check. */ - if (cbm_index_is_quarantined(fi->rel_path)) { - const char *phase = cbm_index_quarantine_phase(fi->rel_path); - if (!phase) { - phase = "crash"; - } - const char *reason = - (strcmp(phase, "hang") == 0) ? "quarantined after hang" : "quarantined after crash"; - pp_err_add(errs, fi->rel_path, reason, phase); - ws->errors++; - continue; - } - - /* Read + extract */ - int source_len = 0; - long file_size = 0; - cbm_read_status_t rst = CBM_READ_OK; - char *source = read_file(fi->path, &source_len, &file_size, &rst); - if (!source) { - ws->errors++; - if (rst == CBM_READ_OVERSIZED) { - /* Never a silent drop: record the oversized skip + a throttled - * WARN so the file surfaces in the response/logfile. */ - long cap = cbm_max_file_bytes(); - char reason[96]; - snprintf(reason, sizeof(reason), "oversized (%lld MB > %lld MB)", - (long long)(file_size / (CBM_SZ_1K * CBM_SZ_1K)), - (long long)(cap / (CBM_SZ_1K * CBM_SZ_1K))); - pp_err_add(errs, fi->rel_path, reason, "oversized"); - if (atomic_fetch_add_explicit(&ec->oversized_warned, SKIP_ONE, - memory_order_relaxed) < PP_OVERSIZED_WARN_MAX) { - cbm_log_warn("index.file_oversized", "path", fi->rel_path, "size_mb", - cbm_log_itoa((int)(file_size / (CBM_SZ_1K * CBM_SZ_1K))), "cap_mb", - cbm_log_itoa((int)(cap / (CBM_SZ_1K * CBM_SZ_1K)))); - } - } else if (rst == CBM_READ_OPEN_FAIL || rst == CBM_READ_OOM) { - pp_err_add(errs, fi->rel_path, "read failed", "read"); - } - /* CBM_READ_EMPTY: benign 0-byte file — not reported. */ - continue; - } - - /* Per-file start log: shows which file each worker is processing. - * Critical for diagnosing stuck workers on large vendored files. */ - if (sort_pos < PP_LOG_THRESH) { /* first 2 rounds of workers = most interesting */ - cbm_log_info("parallel.extract.file.start", "pos", cbm_log_itoa(sort_pos), "size_kb", - cbm_log_itoa(source_len / CBM_SZ_1K), "path", fi->rel_path); - } - - uint64_t file_t0 = extract_now_ns(); - - CBMFileResult *result = cbm_extract_file(source, source_len, fi->language, ec->project_name, - fi->rel_path, CBM_EXTRACT_BUDGET, NULL, NULL); - - uint64_t file_elapsed_ms = (extract_now_ns() - file_t0) / PP_USEC_PER_MS; - - if (!result) { - log_extract_fail(sort_pos, file_elapsed_ms, fi->rel_path); - free_source(source); - ws->errors++; - pp_err_add(errs, fi->rel_path, "extract failed", "extract"); - continue; - } - log_extract_done(sort_pos, file_elapsed_ms, result->defs.count, fi->rel_path); - - /* Consume the previously-ignored has_error flag: a parse timeout / parse - * failure / unsupported-grammar result carries no defs but must still be - * reported (phase "extract", reason = the extractor's message). The empty - * result flows through unchanged below (the defs loop is a no-op). */ - if (result->has_error) { - pp_err_add(errs, fi->rel_path, result->error_msg ? result->error_msg : "extract failed", - "extract"); - ws->errors++; - } else if (result->parse_incomplete) { - /* Indexed, but with unrecovered ERROR/MISSING regions. Not a skip: - * recorded under the distinct "parse_partial" phase with the - * line-range list as the reason, or "parse_unusable" when one range - * covers most of the file (see parse_unusable in cbm.h). */ - pp_err_add(errs, fi->rel_path, result->error_ranges ? result->error_ranges : "", - result->parse_unusable ? "parse_unusable" : "parse_partial"); - } - - /* Create definition nodes in local gbuf */ - for (int d = 0; d < result->defs.count; d++) { - CBMDefinition *def = &result->defs.items[d]; - if (def->qualified_name && def->name) { - insert_def_into_gbuf(ws, ec->stem_marks, fi, def); - } - } - - /* Free TSTree immediately — arena strings survive for registry+resolve. - * This makes slab reset safe: tree-sitter's internal nodes (in slab) - * are released before the slab is bulk-reclaimed. */ - cbm_free_tree(result); - - /* Detect and parse manifest files for package map */ - { - const char *bn = strrchr(fi->rel_path, '/'); - cbm_pkgmap_try_parse(bn ? bn + SKIP_ONE : fi->rel_path, fi->rel_path, source, - source_len, &ec->pkg_entries[worker_id]); - } - - /* Retain source bytes in result->arena so the fused cross-file LSP - * step in resolve_worker can re-parse without re-reading from disk. - * Bounded per-file (retain_per_file_max_bytes) and by a project-wide - * budget (retain_total_budget_bytes) to bound peak RSS — see the - * retention-cap comment at the top of this file. A file DROPPED here - * is NOT lost to cross-file resolution: resolve_worker re-reads it on - * demand (bounded, freed immediately). WARN once per run so the - * operator knows retention was capped (index.retain_capped). */ - if (ec->retain_sources && source_len > 0) { - bool dropped = false; - if ((size_t)source_len > ec->retain_per_file_max_bytes) { - dropped = true; /* over the per-file cap */ - } else { - int64_t prior = atomic_fetch_add_explicit(&ec->retained_bytes, (int64_t)source_len, - memory_order_relaxed); - if ((size_t)(prior + (int64_t)source_len) <= ec->retain_total_budget_bytes) { - char *copy = (char *)cbm_arena_alloc(&result->arena, (size_t)source_len + 1); - if (copy) { - memcpy(copy, source, (size_t)source_len); - copy[source_len] = '\0'; - result->source = copy; - result->source_len = source_len; - } else { - /* Arena OOM — not a cap drop; re-read still covers - * cross-file resolution, so don't emit the WARN. */ - atomic_fetch_sub_explicit(&ec->retained_bytes, (int64_t)source_len, - memory_order_relaxed); - } - } else { - atomic_fetch_sub_explicit(&ec->retained_bytes, (int64_t)source_len, - memory_order_relaxed); - dropped = true; /* project-wide budget exhausted */ - } - } - if (dropped && - atomic_exchange_explicit(&ec->retain_cap_warned, 1, memory_order_relaxed) == 0) { - cbm_log_warn("index.retain_capped", "path", fi->rel_path ? fi->rel_path : "?", - "bytes", cbm_log_itoa((int)source_len)); - } - } - - /* Free source buffer — extraction captured everything needed, - * and the retention copy (if any) lives in result->arena. */ - free_source(source); - - /* Cache result (arena + extracted data, no tree) for Phase 3B and Phase 4. - * While spilling it goes straight to disk; a failed park keeps it. */ - if (!spilling || !pp_park(ec, file_idx, result)) { - ec->result_cache[file_idx] = result; - if (ec->result_owner) { - atomic_store_explicit(&ec->result_owner[file_idx], worker_id + 1, - memory_order_relaxed); - } - } - - /* Progress logging: log every 10 files (atomic read, no contention) */ - if ((sort_pos + SKIP_ONE) % PP_LOG_INTERVAL == 0 || sort_pos + SKIP_ONE == ec->file_count) { - cbm_log_info("parallel.extract.progress", "done", cbm_log_itoa(sort_pos + SKIP_ONE), - "total", cbm_log_itoa(ec->file_count)); - } - - /* Reclaim all slab + tier2 memory between files. - * - * After cbm_free_tree(result), all tree nodes are on free lists. - * We then destroy the parser (frees its internal allocations too), - * leaving ZERO live slab/tier2 pointers. At that point, we can - * safely munmap/free every page, bounding peak memory per-file - * instead of accumulating across all 644 files. - * - * get_thread_parser() in cbm_extract_file will create a fresh - * parser for the next file — cost is microseconds vs seconds - * for parsing. This prevents unbounded memory accumulation and works - * identically on macOS, Linux, and Windows. */ - cbm_destroy_thread_parser(); - cbm_slab_reclaim(); - cbm_mem_collect(); - } - - /* Final cleanup (parser already destroyed in loop, just slab state) */ - cbm_slab_destroy_thread(); - cbm_kind_in_set_free_cache(); /* free this worker thread's node-type bitset cache */ -} - -static void merge_pkg_entries(cbm_pipeline_ctx_t *ctx, cbm_pkg_entries_t *pkg_entries, - int worker_count) { - if (!pkg_entries) { - return; - } - /* Supplement with a repo-wide filesystem walk so manifests filtered - * by the main discoverer (package.json, composer.json — in - * IGNORED_JSON_FILES) still feed pkgmap. Append into worker 0's - * array so the existing merge below sees them. */ - cbm_pkgmap_scan_repo(ctx->repo_path, &pkg_entries[0], ctx->excluded_dirs, ctx->excluded_count); - ctx->pkgmap = cbm_pkgmap_build(pkg_entries, worker_count, ctx->project_name); - cbm_pipeline_set_pkgmap(ctx->pkgmap); - for (int i = 0; i < worker_count; i++) { - cbm_pkg_entries_free(&pkg_entries[i]); - } - free(pkg_entries); -} - -static void log_extract_mem_stats(int worker_count) { - if (cbm_mem_budget() > 0) { - size_t mb = (size_t)CBM_SZ_1K * CBM_SZ_1K; - cbm_log_info("parallel.extract.mem", "rss_mb", cbm_log_itoa((int)(cbm_mem_rss() / mb)), - "peak_mb", cbm_log_itoa((int)(cbm_mem_peak_rss() / mb)), "budget_mb", - cbm_log_itoa((int)(cbm_mem_budget() / mb)), "per_worker_mb", - cbm_log_itoa((int)(cbm_mem_worker_budget(worker_count) / mb))); - } -} - -int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count, - CBMFileResult **result_cache, _Atomic int64_t *shared_ids, - int worker_count, const cbm_parallel_extract_opts_t *opts) { - cbm_parallel_extract_opts_t resolved_opts = cbm_parallel_extract_resolve_opts(opts); - - if (file_count == 0) { - return 0; - } - - cbm_log_info("parallel.extract.start", "files", cbm_log_itoa(file_count), "workers", - cbm_log_itoa(worker_count)); - { - size_t mb = (size_t)CBM_SZ_1K * CBM_SZ_1K; - cbm_log_info("parallel.extract.retention", "retain_sources", - resolved_opts.retain_sources ? "true" : "false", "total_mb", - cbm_log_itoa((int)(resolved_opts.retain_total_budget_bytes / mb)), - "per_file_mb", - cbm_log_itoa((int)(resolved_opts.retain_per_file_max_bytes / mb))); - } - - /* Log per-worker memory budget */ - if (cbm_mem_budget() > 0) { - size_t worker_budget = cbm_mem_worker_budget(worker_count); - cbm_log_info("parallel.mem.budget", "total_mb", - cbm_log_itoa((int)(cbm_mem_budget() / ((size_t)CBM_SZ_1K * CBM_SZ_1K))), - "per_worker_mb", - cbm_log_itoa((int)(worker_budget / ((size_t)CBM_SZ_1K * CBM_SZ_1K)))); - } - - /* Sub-phase: Ensure extraction library is initialized */ - CBM_PROF_START(t_init); - cbm_init(); - - /* Slab allocator for tree-sitter (thread-safe via TLS). */ - cbm_slab_install(); - CBM_PROF_END("parallel_extract", "1_init_libs", t_init); - - /* Sub-phase: Sort files by descending size for tail-latency reduction */ - CBM_PROF_START(t_sort); - file_sort_entry_t *sorted = malloc((size_t)file_count * sizeof(file_sort_entry_t)); - if (!sorted) { - return CBM_NOT_FOUND; - } - for (int i = 0; i < file_count; i++) { - sorted[i].idx = i; - sorted[i].size = files[i].size; - } - if (file_count > 1) { - qsort(sorted, file_count, sizeof(file_sort_entry_t), compare_by_size_desc); - } - CBM_PROF_END_N("parallel_extract", "2_sort_files", t_sort, file_count); - - /* Allocate per-worker state (cache-line aligned via posix_memalign) */ - extract_worker_state_t *workers = NULL; - if (cbm_aligned_alloc((void **)&workers, CBM_CACHE_LINE, - (size_t)worker_count * sizeof(extract_worker_state_t)) != 0) { - free(sorted); - return CBM_NOT_FOUND; - } - memset(workers, 0, (size_t)worker_count * sizeof(extract_worker_state_t)); - - /* Per-worker manifest entry arrays (separate from cache-line-aligned worker state) */ - cbm_pkg_entries_t *pkg_entries = calloc((size_t)worker_count, sizeof(cbm_pkg_entries_t)); - if (!pkg_entries) { - cbm_aligned_free(workers); - free(sorted); - return CBM_NOT_FOUND; - } - - /* Per-worker skip lists (separate allocation; merged into the pipeline in the - * sequential merge loop below). */ - pp_err_list_t *err_lists = calloc((size_t)worker_count, sizeof(pp_err_list_t)); - - extract_ctx_t ec = { - .files = files, - .sorted = sorted, - .file_count = file_count, - .project_name = ctx->project_name, - .repo_path = ctx->repo_path, - .workers = workers, - .max_workers = worker_count, - .result_cache = result_cache, - .shared_ids = shared_ids, - .cancelled = ctx->cancelled, - .pkg_entries = pkg_entries, - .stem_marks = ctx->stem_marks, - .err_lists = err_lists, - .retain_sources = resolved_opts.retain_sources, - .retain_total_budget_bytes = resolved_opts.retain_total_budget_bytes, - .retain_per_file_max_bytes = resolved_opts.retain_per_file_max_bytes, - }; - atomic_init(&ec.next_worker_id, 0); - atomic_init(&ec.next_file_idx, 0); - atomic_init(&ec.retained_bytes, 0); - atomic_init(&ec.retain_cap_warned, 0); - atomic_init(&ec.oversized_warned, 0); - atomic_init(&ec.bp_futile, 0); - atomic_init(&ec.over_budget_abort, 0); - ec.spill = ctx->spill; - ec.result_owner = - ec.spill ? (_Atomic int *)calloc((size_t)file_count, sizeof(_Atomic int)) : NULL; - if (!ec.result_owner) { - ec.spill = NULL; /* no owner map, no spilling: the B3 path stays */ - } - atomic_init(&ec.spilling, - ec.spill && atomic_load_explicit(&g_force_spill, memory_order_relaxed) ? 1 : 0); - - /* Sub-phase: Dispatch workers (parse + extract per file, PARALLEL) */ - CBM_PROF_START(t_dispatch); - cbm_parallel_for_opts_t parallel_opts = {.max_workers = worker_count, .force_pthreads = false}; - cbm_parallel_for(worker_count, extract_worker, &ec, parallel_opts); - CBM_PROF_END_N("parallel_extract", "3_dispatch_workers_parallel", t_dispatch, file_count); - - /* Sub-phase: Merge all local gbufs into main gbuf (SEQUENTIAL, gbuf not thread-safe) */ - CBM_PROF_START(t_merge); - int total_nodes = 0; - int total_errors = 0; - for (int i = 0; i < worker_count; i++) { - if (workers[i].local_gbuf) { - cbm_gbuf_merge(ctx->gbuf, workers[i].local_gbuf); - total_nodes += workers[i].nodes_created; - total_errors += workers[i].errors; - cbm_gbuf_free(workers[i].local_gbuf); - } - } - CBM_PROF_END_N("parallel_extract", "4_merge_gbufs_seq", t_merge, total_nodes); - - /* Merge per-worker skip lists into the pipeline (SEQUENTIAL — no lock). - * Runs unconditionally (not gated on local_gbuf) so a worker whose files all - * failed still surfaces its skips. */ - if (err_lists) { - for (int i = 0; i < worker_count; i++) { - for (int j = 0; j < err_lists[i].count; j++) { - cbm_pipeline_add_file_error(ctx->pipeline, err_lists[i].items[j].path, - err_lists[i].items[j].reason, - err_lists[i].items[j].phase); - free(err_lists[i].items[j].path); - free(err_lists[i].items[j].reason); - free(err_lists[i].items[j].phase); - } - free(err_lists[i].items); - } - free(err_lists); - } - - merge_pkg_entries(ctx, pkg_entries, worker_count); - - cbm_aligned_free(workers); - free(sorted); - free(ec.result_owner); - - /* The over-budget verdict outranks the cancel sentinel: a caller must be - * able to name the cause. Every non-zero code stops before publication. */ - if (atomic_load(&ec.over_budget_abort)) { - return CBM_PIPELINE_ABORT_OVER_BUDGET; - } - if (atomic_load(ctx->cancelled)) { - return CBM_NOT_FOUND; - } - - log_extract_mem_stats(worker_count); - - cbm_log_info("parallel.extract.done", "nodes", cbm_log_itoa(total_nodes), "errors", - cbm_log_itoa(total_errors)); - return 0; -} - -int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count, - CBMFileResult **result_cache, _Atomic int64_t *shared_ids, - int worker_count) { - return cbm_parallel_extract_ex(ctx, files, file_count, result_cache, shared_ids, worker_count, - NULL); -} - -/* ── Phase 3B: Serial Registry Build ─────────────────────────────── */ - -/* Register one definition and create DEFINES + DEFINES_METHOD edges. Returns edge count. */ -static int register_and_link_def(cbm_pipeline_ctx_t *ctx, const CBMDefinition *def, const char *rel, - CBMLanguage lang, int *reg_entries) { - int edges = 0; - if (!def->name || !def->qualified_name || !def->label) { - return 0; - } - /* Register callable symbols + every type-like container (Class/Struct/ - * Interface/Enum/Type/Trait). Type-like defs must be in the registry so - * `class Foo : IBar` (INHERITS), `impl Trait for S` (IMPLEMENTS), and - * method/field resolution can reach them — Struct included so Rust/Go/ - * Swift/D structs resolve as type targets. Variable/Field defs are - * registered too so READS/WRITES can resolve. - * KEEP IN SYNC with pipeline_incremental.c's seed set. */ - if (strcmp(def->label, "Function") == 0 || strcmp(def->label, "Method") == 0 || - cbm_label_is_type_like(def->label) || strcmp(def->label, "Variable") == 0 || - strcmp(def->label, "Field") == 0) { - cbm_registry_add(ctx->registry, def->name, def->qualified_name, def->label); - (*reg_entries)++; - } - char *file_qn = cbm_pipeline_fqn_compute(ctx->project_name, rel, "__file__"); - const cbm_gbuf_node_t *file_node = cbm_gbuf_find_by_qn(ctx->gbuf, file_qn); - char marked[CBM_SZ_4K]; - const char *qn = strcmp(def->label, "Module") == 0 - ? cbm_stem_marks_module_qn(ctx->stem_marks, def->qualified_name, - def->file_path ? def->file_path : rel, lang, - marked, sizeof(marked)) - : def->qualified_name; - const cbm_gbuf_node_t *def_node = cbm_gbuf_find_by_qn(ctx->gbuf, qn); - if (file_node && def_node) { - cbm_gbuf_insert_edge(ctx->gbuf, file_node->id, def_node->id, "DEFINES", "{}"); - edges++; - } - free(file_qn); - if (def->parent_class && strcmp(def->label, "Method") == 0) { - const cbm_gbuf_node_t *parent = cbm_gbuf_find_by_qn(ctx->gbuf, def->parent_class); - if (parent && def_node) { - cbm_gbuf_insert_edge(ctx->gbuf, parent->id, def_node->id, "DEFINES_METHOD", "{}"); - } - } - return edges; -} - -/* Create IMPORTS edges for one file's imports (parallel path). */ -static int create_imports_edges(cbm_pipeline_ctx_t *ctx, const CBMFileResult *result, - const char *rel, CBMHashTable *namespace_map) { - int count = 0; - char *file_qn = cbm_pipeline_fqn_compute(ctx->project_name, rel, "__file__"); - const cbm_gbuf_node_t *source_node = cbm_gbuf_find_by_qn(ctx->gbuf, file_qn); - if (!source_node) { - free(file_qn); - return 0; - } - for (int j = 0; j < result->imports.count; j++) { - CBMImport *imp = &result->imports.items[j]; - if (!imp->module_path) { - continue; - } - const cbm_gbuf_node_t *target = - cbm_pipeline_resolve_import_node(ctx, rel, file_qn, imp, namespace_map); - if (target && target->id != source_node->id) { - char esc_ln[CBM_SZ_128]; - cbm_json_escape(esc_ln, sizeof(esc_ln), imp->local_name ? imp->local_name : ""); - char imp_props[CBM_SZ_256]; - snprintf(imp_props, sizeof(imp_props), "{\"local_name\":\"%s\"}", esc_ln); - cbm_gbuf_insert_edge(ctx->gbuf, source_node->id, target->id, "IMPORTS", imp_props); - count++; - } - } - free(file_qn); - return count; -} - -/* Find channel source node (enclosing function or file). */ -static const cbm_gbuf_node_t *find_channel_src(cbm_pipeline_ctx_t *ctx, const CBMChannel *ch, - const char *rel) { - const cbm_gbuf_node_t *node = NULL; - if (ch->enclosing_func_qn && ch->enclosing_func_qn[0]) { - node = cbm_gbuf_find_by_qn(ctx->gbuf, ch->enclosing_func_qn); - } - if (!node) { - char *file_qn = cbm_pipeline_fqn_compute(ctx->project_name, rel, "__file__"); - node = cbm_gbuf_find_by_qn(ctx->gbuf, file_qn); - free(file_qn); - } - return node; -} - -/* Create Channel nodes + EMITS/LISTENS_ON edges for one file. */ -static void create_channel_edges(cbm_pipeline_ctx_t *ctx, const CBMFileResult *result, - const char *rel) { - for (int j = 0; j < result->channels.count; j++) { - CBMChannel *ch = &result->channels.items[j]; - if (!ch->channel_name || !ch->channel_name[0]) { - continue; - } - char channel_qn[CBM_SZ_512]; - snprintf(channel_qn, sizeof(channel_qn), "__channel__%s__%s", - ch->transport ? ch->transport : "unknown", ch->channel_name); - char esc_cn[CBM_SZ_256]; - cbm_json_escape(esc_cn, sizeof(esc_cn), ch->channel_name); - char channel_props[CBM_SZ_512]; - snprintf(channel_props, sizeof(channel_props), "{\"transport\":\"%s\",\"name\":\"%s\"}", - ch->transport ? ch->transport : "unknown", esc_cn); - int64_t channel_id = cbm_gbuf_upsert_node(ctx->gbuf, "Channel", ch->channel_name, - channel_qn, "", 0, 0, channel_props); - const cbm_gbuf_node_t *src_node = find_channel_src(ctx, ch, rel); - if (src_node && channel_id > 0) { - const char *edge_type = ch->direction == CBM_CHANNEL_EMIT ? "EMITS" : "LISTENS_ON"; - char edge_props[CBM_SZ_128]; - snprintf(edge_props, sizeof(edge_props), "{\"transport\":\"%s\"}", - ch->transport ? ch->transport : "unknown"); - cbm_gbuf_insert_edge(ctx->gbuf, src_node->id, channel_id, edge_type, edge_props); - } - } -} - -/* Create CONFIGURES edges for one file's env accesses. */ -static void create_env_configures(cbm_pipeline_ctx_t *ctx, const CBMFileResult *result, - const char *rel) { - char *file_qn = NULL; - const cbm_gbuf_node_t *file_node = NULL; - for (int j = 0; j < result->env_accesses.count; j++) { - const CBMEnvAccess *ea = &result->env_accesses.items[j]; - if (!ea->env_key || !ea->env_key[0]) { - continue; - } - char env_qn[CBM_SZ_512]; - snprintf(env_qn, sizeof(env_qn), "__env__%s", ea->env_key); - char env_props[CBM_SZ_512]; - snprintf(env_props, sizeof(env_props), "{\"env_key\":\"%s\"}", ea->env_key); - int64_t env_id = - cbm_gbuf_upsert_node(ctx->gbuf, "EnvVar", ea->env_key, env_qn, "", 0, 0, env_props); - if (env_id <= 0) { - continue; - } - const cbm_gbuf_node_t *src = NULL; - if (ea->enclosing_func_qn && ea->enclosing_func_qn[0]) { - src = cbm_gbuf_find_by_qn(ctx->gbuf, ea->enclosing_func_qn); - } - if (!src) { - if (!file_qn) { - file_qn = cbm_pipeline_fqn_compute(ctx->project_name, rel, "__file__"); - file_node = cbm_gbuf_find_by_qn(ctx->gbuf, file_qn); - } - src = file_node; - } - if (src && src->id != env_id) { - cbm_gbuf_insert_edge(ctx->gbuf, src->id, env_id, "CONFIGURES", - "{\"strategy\":\"env_access\"}"); - } - } - free(file_qn); -} - -int cbm_build_registry_from_cache(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, - int file_count, CBMFileResult **result_cache) { - cbm_log_info("parallel.registry.start", "files", cbm_log_itoa(file_count)); - - int reg_entries = 0; - int defines_edges = 0; - int imports_edges = 0; - - /* Namespace/package → File-QN map for namespace imports (C# `using`, - * Java/Kotlin `import`, PHP `use`). Built from the full result cache so - * every declaring file is visible regardless of loop order. */ - const char **rels = (const char **)calloc((size_t)file_count, sizeof(char *)); - if (rels) { - for (int i = 0; i < file_count; i++) { - rels[i] = files[i].rel_path; - } - } - CBMHashTable *namespace_map = - cbm_pipeline_namespace_map_build(ctx->project_name, result_cache, ctx->spill, rels, - file_count); - free(rels); - - for (int i = 0; i < file_count; i++) { - if (cbm_pipeline_check_cancel(ctx)) { - cbm_pipeline_namespace_map_free(namespace_map); - return CBM_NOT_FOUND; - } - - CBMFileResult *result = cbm_result_cache_acquire(result_cache, ctx->spill, i); - if (!result) { - continue; - } - - const char *rel = files[i].rel_path; - - /* Register callable symbols + DEFINES/DEFINES_METHOD edges */ - for (int d = 0; d < result->defs.count; d++) { - defines_edges += register_and_link_def(ctx, &result->defs.items[d], rel, - files[i].language, ®_entries); - } - - imports_edges += create_imports_edges(ctx, result, rel, namespace_map); - create_channel_edges(ctx, result, rel); - create_env_configures(ctx, result, rel); - cbm_result_cache_release(result_cache, i, result); - } - - cbm_pipeline_namespace_map_free(namespace_map); - - cbm_log_info("parallel.registry.done", "entries", cbm_log_itoa(reg_entries), "defines", - cbm_log_itoa(defines_edges), "imports", cbm_log_itoa(imports_edges)); - return 0; -} - -/* ── Phase 4: Parallel Resolution ────────────────────────────────── */ - -typedef struct __attribute__((aligned(CBM_CACHE_LINE))) { - cbm_gbuf_t *local_edge_buf; - int calls_resolved; - int usages_resolved; - int semantic_resolved; - int errors; - /* Subset of calls_resolved that were attributed via the LSP-override - * path (cbm_pipeline_find_lsp_resolution hit) rather than the - * registry's textual matcher. Surfaced in the parallel.resolve.done - * log line so divergence between pipelines becomes observable. */ - int lsp_overrides; - char _pad[CBM_CACHE_LINE - sizeof(cbm_gbuf_t *) - (PP_RESOLVE_COUNTERS * sizeof(int))]; -} resolve_worker_state_t; - -typedef struct { - const cbm_file_info_t *files; - int file_count; - const char *project_name; - const char *repo_path; - - resolve_worker_state_t *workers; - int max_workers; - - CBMFileResult **result_cache; - cbm_result_spill_t *spill; /* parked results (F1); NULL when nothing spilled */ - const cbm_gbuf_t *main_gbuf; /* READ-ONLY during Phase 4 */ - const cbm_registry_t *registry; /* READ-ONLY during Phase 4 */ - _Atomic int64_t *shared_ids; - _Atomic int *cancelled; - _Atomic int next_file_idx; - - /* Cross-file LSP inputs — pre-built once by the caller in pipeline.c - * and shared read-only by usage across workers (typed non-const to - * match the existing cbm_run_X_lsp_cross callee signatures the - * worker forwards them to). NULL/0 → cross-LSP no-ops. */ - CBMLSPDef *all_defs; - int def_count; - char *const *def_modules; /* per-file module QN; def_modules[i] for files[i] */ - /* Optional inverted index for per-file def filtering (gopls pattern). - * When non-NULL, the fused worker calls cbm_pxc_filter_defs_for_file - * to shrink the def array passed to the LSP from O(all_defs) to - * O(relevant_defs). NULL → each file sees the full all_defs[]. */ - struct CBMModuleDefIndex *module_def_index; - /* Tier 2 full: pre-built per-language registries (project-wide, - * finalized, READ-ONLY). When non-NULL for a lang, the worker uses - * cbm_run_X_lsp_cross_with_registry — skip per-file build entirely. - * Stored as CBMCrossLspRegistries* (typedef from pass_lsp_cross.h). */ - CBMCrossLspRegistries *cross_registries; - - /* Parsed once on the coordinator thread and borrowed read-only by resolve - * workers. The pointer is installed into each worker's TLS slot so Rust's - * manifest-aware cross-file resolver sees the same Cargo context. */ - const CBMCargoManifest *rust_manifest; - - /* F4: LAZILY-built shared Rust registry (built ONCE, on the first NULL-filter - * rust file — the ~all_defs amplifier files). Not eager: repos whose rust files - * all filter to subsets never pay the O(all_defs) build + multi-GB RSS. Built - * under rust_shared_mu into rust_shared_arena, published via rust_shared_reg; - * torn down after the worker dispatch. */ - _Atomic(CBMTypeRegistry *) rust_shared_reg; - cbm_mutex_t rust_shared_mu; - CBMArena rust_shared_arena; - bool rust_shared_arena_live; - - /* Counters for parallel.resolve.lsp_cross_done summary. */ - _Atomic int lsp_cross_processed; - _Atomic int lsp_cross_skipped_no_source; - /* Error message of a cross-file LSP walk that stopped incomplete, per - * file index. Each file is owned by one worker; the coordinator reports - * them after join, so the pipeline's error list is never mutated - * concurrently. */ - const char **lsp_failures; - - /* Per-sub-phase timing (ns aggregated across workers) — surfaces - * exactly where parallel_resolve's wall time is spent so we stop - * guessing about hot paths. Logged once at the end of - * cbm_parallel_resolve. */ - _Atomic uint64_t time_ns_import_map; - _Atomic uint64_t time_ns_cross_lsp; - _Atomic uint64_t time_ns_calls; - _Atomic uint64_t time_ns_usages; - _Atomic uint64_t time_ns_throws; - _Atomic uint64_t time_ns_rw; - _Atomic uint64_t time_ns_semantic; - /* Whole-iteration timer — captures everything from atomic file_idx - * pickup through cleanup. If this >> sum of sub-phases, the - * unmeasured cost is either in skip-eligibility checks, gbuf - * setup, or — most likely — workers waiting on the - * cbm_parallel_for synchronization barrier at the end. */ - _Atomic uint64_t time_ns_total_loop; - _Atomic int total_files_visited; - /* Sub-breakdowns inside resolve_file_calls — finds the 553µs-per- - * iteration hot path that the high-level resolve_calls counter - * doesn't pinpoint. */ - _Atomic uint64_t time_ns_rc_lsp_lookup; /* lsp_idx + fallback scan */ - _Atomic uint64_t time_ns_rc_resolve; /* lsp_target_node OR registry_resolve */ - _Atomic uint64_t time_ns_rc_hint; /* try_field_type_hint */ - _Atomic uint64_t time_ns_rc_target; /* gbuf_find_by_qn for target */ - _Atomic uint64_t time_ns_rc_emit; /* emit_service_edge */ - _Atomic uint64_t time_ns_rc_source; /* find_source_node */ -} resolve_ctx_t; - -/* Minimum buffer space needed per arg JSON object */ -#define CBM_ARG_JSON_GUARD CBM_SZ_32 - -/* Append arg data as JSON to edge properties: ,"args":[{"i":0,"e":"x","v":"val"},...] - * Returns new position in buffer. */ -/* Sanitize expression string for JSON (in-place). */ -static void sanitize_expr(char *expr_buf, const char *expr) { - if (expr) { - snprintf(expr_buf, 128, "%.*s", 120, expr); - for (char *p = expr_buf; *p; p++) { - if (*p == '"') { - *p = '\''; - } - if (*p == '\n' || *p == '\r') { - *p = ' '; - } - } - } else { - expr_buf[0] = '\0'; - } -} - -/* Format one call arg as JSON. Returns snprintf result. */ -static int format_call_arg(char *buf, size_t bufsize, const CBMCallArg *a, const char *expr) { - char esc_k[CBM_SZ_128]; - char esc_e[CBM_SZ_128]; - char esc_v[CBM_SZ_128]; - cbm_json_escape(esc_e, sizeof(esc_e), expr); - if (a->keyword && a->value) { - cbm_json_escape(esc_k, sizeof(esc_k), a->keyword); - cbm_json_escape(esc_v, sizeof(esc_v), a->value); - return snprintf(buf, bufsize, "{\"i\":%d,\"k\":\"%s\",\"e\":\"%s\",\"v\":\"%s\"}", a->index, - esc_k, esc_e, esc_v); - } - if (a->keyword) { - cbm_json_escape(esc_k, sizeof(esc_k), a->keyword); - return snprintf(buf, bufsize, "{\"i\":%d,\"k\":\"%s\",\"e\":\"%s\"}", a->index, esc_k, - esc_e); - } - if (a->value) { - cbm_json_escape(esc_v, sizeof(esc_v), a->value); - return snprintf(buf, bufsize, "{\"i\":%d,\"e\":\"%s\",\"v\":\"%s\"}", a->index, esc_e, - esc_v); - } - return snprintf(buf, bufsize, "{\"i\":%d,\"e\":\"%s\"}", a->index, esc_e); -} - -static size_t append_args_json(char *buf, size_t bufsize, size_t pos, const CBMCall *call) { - if (call->arg_count == 0 || pos >= bufsize - PP_ARGS_MARGIN) { - return pos; - } - int n = snprintf(buf + pos, bufsize - pos, ",\"args\":["); - if (n <= 0) { - return pos; - } - pos += (size_t)n; - for (int i = 0; i < call->arg_count && pos < bufsize - CBM_ARG_JSON_GUARD; i++) { - const CBMCallArg *a = &call->args[i]; - size_t mark = pos; /* rollback point (before the separator) */ - if (i > 0 && pos < bufsize - SKIP_ONE) { - buf[pos++] = ','; - } - char expr_buf[CBM_SZ_128]; - sanitize_expr(expr_buf, a->expr); - n = format_call_arg(buf + pos, bufsize - pos, a, expr_buf); - /* snprintf returns the UNtruncated length: if the arg did not fully - * fit, advancing pos by n would push it past buf and the buf[pos] - * writes below would overflow. Drop the arg whole (atomic field — - * keeps the array valid) and stop appending. */ - if (n <= 0 || (size_t)n >= bufsize - pos) { - pos = mark; - break; - } - pos += (size_t)n; - } - if (pos < bufsize - SKIP_ONE) { - buf[pos++] = ']'; - } - buf[pos] = '\0'; - return pos; -} - -/* Scan call args for a URL-like route path and handler reference. */ -static bool is_path_keyword(const char *keyword) { - static const char *path_keywords[] = {"prefix", "path", "route", "pattern", - "url", "endpoint", "rule", "mount_path", - "route_path", "url_path", NULL}; - for (const char **kw = path_keywords; *kw; kw++) { - if (strcmp(keyword, *kw) == 0) { - return true; - } - } - return false; -} - -static const char *find_route_path_in_args(const CBMCall *call, const char **out_handler) { - *out_handler = NULL; - /* 1. First string arg starting with / */ - if (call->first_string_arg && call->first_string_arg[0] == '/') { - *out_handler = call->second_arg_name; - return call->first_string_arg; - } - /* 2. Keyword args (prefix=, path=, route=, etc.) */ - const char *found = NULL; - for (int ai = 0; ai < call->arg_count && !found; ai++) { - const CBMCallArg *ca = &call->args[ai]; - const char *val = ca->value ? ca->value : ca->expr; - if (!val || val[0] != '/') { - continue; - } - if ((ca->keyword && is_path_keyword(ca->keyword)) || (!ca->keyword && ca->index == 0)) { - found = val; - } - } - if (!found) { - return NULL; - } - /* 3. Handler: first identifier arg that's not a path/keyword */ - for (int ai = 0; ai < call->arg_count; ai++) { - const CBMCallArg *ca = &call->args[ai]; - if (!ca->expr || ca->expr[0] == '/' || ca->expr[0] == '"' || ca->expr[0] == '\'') { - continue; - } - if (ca->keyword && (strcmp(ca->keyword, "prefix") == 0 || - strcmp(ca->keyword, "name") == 0 || strcmp(ca->keyword, "tags") == 0)) { - continue; - } - *out_handler = ca->expr; - break; - } - return found; -} - -/* Build props JSON, append args, close brace, emit edge. */ -static void finalize_and_emit(cbm_gbuf_t *gbuf, int64_t src_id, int64_t tgt_id, - const char *edge_type, char *props, int n, const CBMCall *call) { - if (n > 0 && (size_t)n < CBM_SZ_2K - PP_ESC_SPACE) { - size_t pos = append_args_json(props, CBM_SZ_2K, (size_t)n, call); - if (call->start_line > 0 && strcmp(edge_type, "CALLS") == 0 && - pos < CBM_SZ_2K - PP_LINE_MARGIN) { - int ln = snprintf(props + pos, CBM_SZ_2K - pos, ",\"line\":%d", call->start_line); - if (ln > 0) { - pos += (size_t)ln; - } - } - if (pos < CBM_SZ_2K - SKIP_ONE) { - props[pos] = '}'; - props[pos + SKIP_ONE] = '\0'; - } - } - cbm_gbuf_insert_edge(gbuf, src_id, tgt_id, edge_type, props); -} - -/* Build Route node QN and properties for HTTP/async service edges. */ -static int64_t build_service_route(cbm_gbuf_t *gbuf, const char *arg, const char *method, - const char *broker, cbm_svc_kind_t svc) { - char route_qn[CBM_ROUTE_QN_SIZE]; - const char *prefix; - char cpath[CBM_SZ_256]; - const char *qpath = arg; - if (svc == CBM_SVC_HTTP) { - prefix = method ? method : "ANY"; - qpath = cbm_route_canon_path(arg, cpath, sizeof(cpath)); - } else { - prefix = broker ? broker : "async"; - } - snprintf(route_qn, sizeof(route_qn), "__route__%s__%s", prefix, qpath); - char route_props[CBM_SZ_256]; - if (method) { - snprintf(route_props, sizeof(route_props), "{\"method\":\"%s\"}", method); - } else if (broker) { - snprintf(route_props, sizeof(route_props), "{\"broker\":\"%s\"}", broker); - } else { - snprintf(route_props, sizeof(route_props), "{}"); - } - return cbm_gbuf_upsert_node(gbuf, "Route", arg, route_qn, "", 0, 0, route_props); -} - -/* Emit HTTP_CALLS or ASYNC_CALLS edge via Route node. */ -static void emit_http_async_service_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, - const CBMCall *call, const cbm_resolution_t *res, - cbm_svc_kind_t svc, const char *arg) { - const char *edge_type = (svc == CBM_SVC_HTTP) ? "HTTP_CALLS" : "ASYNC_CALLS"; - const char *method = - (svc == CBM_SVC_HTTP) ? cbm_service_pattern_http_method(call->callee_name) : NULL; - const char *broker = - (svc == CBM_SVC_ASYNC) ? cbm_service_pattern_broker(res->qualified_name) : NULL; - - int64_t route_id = build_service_route(gbuf, arg, method, broker, svc); - - char esc_c[CBM_SZ_256]; - char esc_a[CBM_SZ_256]; - cbm_json_escape(esc_c, sizeof(esc_c), call->callee_name); - cbm_json_escape(esc_a, sizeof(esc_a), arg); - char props[CBM_SZ_2K]; - int n = snprintf(props, sizeof(props), "{\"callee\":\"%s\",\"url_path\":\"%s\"", esc_c, esc_a); - if (method) { - n += snprintf(props + n, sizeof(props) - (size_t)n, ",\"method\":\"%s\"", method); - } - if (broker) { - n += snprintf(props + n, sizeof(props) - (size_t)n, ",\"broker\":\"%s\"", broker); - } - finalize_and_emit(gbuf, source->id, route_id, edge_type, props, n, call); -} - -/* Emit CONFIGURES edge. */ -static void emit_config_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, - const cbm_gbuf_node_t *target, const CBMCall *call, - const cbm_resolution_t *res, const char *arg) { - /* emit_service_edge may be reached with target==NULL on the HTTP/ASYNC - * external-client bypass (#523); a CONFIGURES edge needs a real target, so - * never deref a NULL target here. */ - if (!target) { - return; - } - char esc_c[CBM_SZ_256]; - char esc_k[CBM_SZ_256]; - cbm_json_escape(esc_c, sizeof(esc_c), call->callee_name); - cbm_json_escape(esc_k, sizeof(esc_k), arg ? arg : ""); - char props[CBM_SZ_2K]; - int n = snprintf(props, sizeof(props), "{\"callee\":\"%s\",\"key\":\"%s\",\"confidence\":%.2f", - esc_c, esc_k, res->confidence); - finalize_and_emit(gbuf, source->id, target->id, "CONFIGURES", props, n, call); -} - -/* Emit normal CALLS edge. */ -static void emit_normal_calls_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, - const cbm_gbuf_node_t *target, const CBMCall *call, - const cbm_resolution_t *res) { - /* A CALLS edge needs a real target; the HTTP/ASYNC external-client bypass - * (#523) can reach emit_service_edge with target==NULL, so guard the deref. */ - if (!target) { - return; - } - char esc_c[CBM_SZ_256]; - cbm_json_escape(esc_c, sizeof(esc_c), call->callee_name); - char props[CBM_SZ_2K]; - int n = snprintf(props, sizeof(props), - "{\"callee\":\"%s\",\"confidence\":%.2f,\"strategy\":\"%s\",\"candidates\":%d", - esc_c, res->confidence, res->strategy ? res->strategy : "unknown", - res->candidate_count); - finalize_and_emit(gbuf, source->id, target->id, "CALLS", props, n, call); -} - -/* Classify a resolved call by library identity and emit the appropriate edge. */ -/* Create Route node + CALLS + HANDLES edges for a route registration call. - * route_mount is the framework mount of the registering file ("/api" for a - * Laravel 11+ `withRouting(api: ...)` file, #1146) or "". */ -static void emit_route_registration(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, - const CBMCall *call, const char *route_path, - const char *handler_ref, const char *module_qn, - const cbm_registry_t *registry, const cbm_gbuf_t *main_gbuf, - const char **ik, const char **iv, int ic, - const char *route_mount) { - const char *method = cbm_service_pattern_route_method(call->callee_name); - char mounted[CBM_SZ_256]; - route_path = cbm_laravel_mount_route(route_mount, route_path, mounted, sizeof(mounted)); - char rqn[CBM_ROUTE_QN_SIZE]; - char cpath[CBM_SZ_256]; - snprintf(rqn, sizeof(rqn), "__route__%s__%s", method ? method : "ANY", - cbm_route_canon_path(route_path, cpath, sizeof(cpath))); - char rp[CBM_SZ_256]; - snprintf(rp, sizeof(rp), "{\"method\":\"%s\"}", method ? method : "ANY"); - int64_t rid = cbm_gbuf_upsert_node(gbuf, "Route", route_path, rqn, "", 0, 0, rp); - char esc_cn[CBM_SZ_256]; /* sliced source text: escape quotes/newlines */ - char esc_rp[CBM_SZ_512]; - cbm_json_escape(esc_cn, sizeof(esc_cn), call->callee_name); - cbm_json_escape(esc_rp, sizeof(esc_rp), route_path); - char props[CBM_SZ_1K]; - snprintf(props, sizeof(props), - "{\"callee\":\"%s\",\"url_path\":\"%s\",\"via\":\"route_registration\"}", esc_cn, - esc_rp); - cbm_gbuf_insert_edge(gbuf, source->id, rid, "CALLS", props); - if (handler_ref && handler_ref[0] != '\0') { - cbm_resolution_t hres = - cbm_registry_resolve_handler(registry, handler_ref, module_qn, ik, iv, ic, main_gbuf); - if (hres.qualified_name && hres.qualified_name[0] != '\0') { - const cbm_gbuf_node_t *h = cbm_gbuf_find_by_qn(main_gbuf, hres.qualified_name); - if (h) { - char hp[CBM_SZ_1K]; /* must exceed escaped value + wrapper or snprintf cuts the - closing brace */ - char esc_h2[CBM_SZ_512]; - cbm_json_escape(esc_h2, sizeof(esc_h2), hres.qualified_name); - snprintf(hp, sizeof(hp), "{\"handler\":\"%s\"}", esc_h2); - cbm_gbuf_insert_edge(gbuf, h->id, rid, "HANDLES", hp); - } - } - } -} - -/* Reject regex metacharacters, spaces, double-slashes in URL candidates. */ -static bool is_junk_url(const char *s) { - for (int i = 0; s[i]; i++) { - char ch = s[i]; - if (ch == '\\' || ch == '^' || ch == '$' || ch == '*' || ch == '+' || ch == '(' || - ch == ')' || ch == '[' || ch == ']' || ch == '|' || ch == ' ') { - return true; - } - if (ch == '/' && i > 0 && s[i - SKIP_ONE] == '/') { - return true; - } - } - return false; -} - -/* Normalize a template literal URL and reject junk patterns. - * Returns true if norm contains a valid API path. */ -static bool normalize_url_arg(const char *url, char *norm, int norm_sz) { - int ni = 0; - const char *p = url; - if (*p == '`' || *p == '"' || *p == '\'') { - p++; - } - if (*p != '/') { - return false; - } - while (*p && ni < norm_sz - PAIR_LEN) { - if (*p == '$' && *(p + SKIP_ONE) == '{') { - norm[ni++] = ':'; - p += PAIR_LEN; - while (*p && *p != '}' && ni < norm_sz - PAIR_LEN) { - norm[ni++] = *p++; - } - if (*p == '}') { - p++; - } - } else if (*p == '`' || *p == '"' || *p == '\'' || *p == '?') { - break; - } else { - norm[ni++] = *p++; - } - } - norm[ni] = '\0'; - enum { MIN_URL_LEN = 4 }; - if (ni < MIN_URL_LEN || !strchr(norm + SKIP_ONE, '/')) { - return false; - } - return !is_junk_url(norm); -} - -static bool arg_is_topic(const CBMCallArg *arg) { - static const char *keywords[] = {"topic", "topic_id", "topic_name", "queue", "queue_name", - "queue_id", "subject", "channel", NULL}; - for (int i = 0; arg->keyword && keywords[i]; i++) { - if (strcmp(arg->keyword, keywords[i]) == 0) { - return true; - } - } - return false; -} - -/* File-language scope of the client-navigation heuristic (#1250): the JS/TS - * family and the hosts that embed it. Kept explicit so changes to the - * weak-member guard cannot broaden it. */ -static bool pp_client_nav_lang(CBMLanguage lang) { - return lang == CBM_LANG_TYPESCRIPT || lang == CBM_LANG_TSX || lang == CBM_LANG_JAVASCRIPT || - lang == CBM_LANG_VUE || lang == CBM_LANG_SVELTE || lang == CBM_LANG_HTML || - lang == CBM_LANG_ASTRO; -} - -/* Detect API paths in call arguments and create HTTP_CALLS edges. */ -static void detect_url_in_args(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, - const CBMCall *call, CBMLanguage lang, cbm_svc_kind_t svc) { - /* A resolved alias may be ASYNC even when its source spelling is unknown. - * Neither raw slash topics nor URL projections are HTTP evidence there. */ - if (svc == CBM_SVC_ASYNC || cbm_service_pattern_match(call->callee_name) == CBM_SVC_ASYNC) { - return; - } - if (cbm_service_pattern_is_client_navigation(pp_client_nav_lang(lang), call->callee_name)) { - return; - } - /* A bare `fetch` reaching here resolved to the project's own `fetch`, - * which shadows the global API (#856): a plain call, not HTTP. */ - if (cbm_service_pattern_is_global_fetch(call->callee_name)) { - return; - } - for (int ai = 0; ai < call->arg_count; ai++) { - const CBMCallArg *ca = &call->args[ai]; - if (arg_is_topic(ca)) { - continue; - } - /* A slash-prefixed raw expression is not a URL string. In JS/TS this - * is notably a regex literal (`/value && !ca->url_value && ca->expr && ca->expr[0] == '/') { - continue; - } - const char *url = ca->url_value ? ca->url_value : (ca->value ? ca->value : ca->expr); - if (!url || (url[0] != '/' && url[0] != '`')) { - continue; - } - char norm[CBM_SZ_256]; - if (!normalize_url_arg(url, norm, (int)sizeof(norm))) { - continue; - } - if (!cbm_service_pattern_is_http_route_literal(norm, call->callee_name)) { - continue; - } - char route_qn[CBM_ROUTE_QN_SIZE]; - char cpath[CBM_SZ_256]; - snprintf(route_qn, sizeof(route_qn), "__route__ANY__%s", - cbm_route_canon_path(norm, cpath, sizeof(cpath))); - int64_t route_id = cbm_gbuf_upsert_node(gbuf, "Route", norm, route_qn, "", 0, 0, - "{\"source\":\"arg_url\"}"); - char esc_c[CBM_SZ_256]; - char esc_n[CBM_SZ_256]; - cbm_json_escape(esc_c, sizeof(esc_c), call->callee_name); - cbm_json_escape(esc_n, sizeof(esc_n), norm); - char eprops[CBM_SZ_512]; - snprintf(eprops, sizeof(eprops), - "{\"callee\":\"%s\",\"url_path\":\"%s\",\"via\":\"arg_url\"}", esc_c, esc_n); - cbm_gbuf_insert_edge(gbuf, source->id, route_id, "HTTP_CALLS", eprops); - break; - } -} - -/* Extract gRPC service and method from a callee name. - * Handles patterns like: pb.NewFooServiceClient(conn).GetBar → Foo/GetBar - * Also: FooServiceGrpc.newBlockingStub(ch).getBar → FooService/getBar */ -bool extract_grpc_service_method(const char *callee, char *service, size_t srv_sz, char *method, - size_t meth_sz) { - service[0] = '\0'; - method[0] = '\0'; - if (!callee) { - return false; - } - /* Find last dot to split service.Method */ - const char *last_dot = strrchr(callee, '.'); - if (!last_dot || !last_dot[SKIP_ONE]) { - return false; - } - snprintf(method, meth_sz, "%s", last_dot + SKIP_ONE); - - /* Extract service name: everything before the last dot, stripped of prefixes/suffixes */ - size_t prefix_len = (size_t)(last_dot - callee); - char raw[CBM_SZ_256]; - if (prefix_len >= sizeof(raw)) { - prefix_len = sizeof(raw) - SKIP_ONE; - } - memcpy(raw, callee, prefix_len); - raw[prefix_len] = '\0'; - - /* Strip common prefixes: pb.New, New, pb. */ - const char *s = raw; - if (strncmp(s, "pb.New", CBM_SZ_6) == 0) { - s += CBM_SZ_6; - } else if (strncmp(s, "pb.", CBM_SZ_3) == 0 || strncmp(s, "New", CBM_SZ_3) == 0) { - s += CBM_SZ_3; - } - - /* Strip the generated-stub/client suffix, preserving the canonical - * proto-declared service name. The proto service is `Service`; the - * generated client is `ServiceClient` / `ServiceGrpc`, so we strip - * only the trailing stub/client token (Client/Stub/Grpc/…), NOT "Service" - * itself — stripping "ServiceClient" yielded `` and broke cross-repo - * matching against the `Service` declared name (#294). Longest tokens - * first so e.g. BlockingStub wins over Stub. - * - * A match also serves as the gRPC stub-type signal: we ONLY emit a Route - * when a recognized suffix is actually present. Without this gate the - * fallback turned ordinary receiver vars (`_provider.GetGroup`, - * `_builder.AddSomeService`) into phantom `__grpc__provider/...` Routes - * that correspond to no .proto anywhere (#294). */ - snprintf(service, srv_sz, "%s", s); - size_t slen = strlen(service); - static const char *const suffixes[] = {"BlockingStub", "FutureStub", "AsyncStub", - "AsyncClient", "Servicer", "Client", - "Stub", "Grpc", NULL}; - bool stripped = false; - for (const char *const *sfx = suffixes; *sfx; sfx++) { - size_t flen = strlen(*sfx); - if (slen > flen && strcmp(service + slen - flen, *sfx) == 0) { - service[slen - flen] = '\0'; - stripped = true; - break; - } - } - - return stripped && service[0] && method[0]; -} - -/* Emit GRPC_CALLS edge via gRPC Route node. */ -static void emit_grpc_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, const CBMCall *call, - const cbm_resolution_t *res) { - char service[CBM_SZ_256]; - char method[CBM_SZ_256]; - /* Try callee_name first (e.g., "pb.NewCartServiceClient.GetCart") */ - if (!extract_grpc_service_method(call->callee_name, service, sizeof(service), method, - sizeof(method))) { - /* Fallback: try the resolved QN for Go chained calls. - * Go pattern: pb.NewCartServiceClient(conn).GetCart(ctx, req) - * callee_name = "GetCart", QN = "...CartServiceClient.GetCart" - * The QN contains the full ServiceClient.Method pattern. */ - if (!res->qualified_name || - !extract_grpc_service_method(res->qualified_name, service, sizeof(service), method, - sizeof(method))) { - return; - } - } - - char route_qn[CBM_SZ_512]; - snprintf(route_qn, sizeof(route_qn), "__grpc__%s/%s", service, method); - - char route_name[CBM_SZ_256]; - snprintf(route_name, sizeof(route_name), "%s/%s", service, method); - - int64_t route_id = cbm_gbuf_upsert_node(gbuf, "Route", route_name, route_qn, "", 0, 0, - "{\"source\":\"grpc\"}"); - - char esc_c[CBM_SZ_256]; - cbm_json_escape(esc_c, sizeof(esc_c), call->callee_name); - char props[CBM_SZ_1K]; - snprintf(props, sizeof(props), - "{\"callee\":\"%s\",\"service\":\"%s\",\"method\":\"%s\",\"confidence\":%.2f}", esc_c, - service, method, res->confidence); - cbm_gbuf_insert_edge(gbuf, source->id, route_id, "GRPC_CALLS", props); -} - -/* Emit GRAPHQL_CALLS edge. The Route is keyed by the operation NAME only - * (#598), never by the operation text: - * named operation -> __graphql__ name "" - * anonymous query/mutation/ -> __graphql____anonymous name "(anonymous )" - * subscription (incl. the `{ ... }` query shorthand) - * no GraphQL document -> __graphql__operation__anonymous name "(anonymous operation)" - * (URL, variable, missing string argument) - * Keys are bounded by the GraphQL Name grammar and deterministic. */ -static void emit_graphql_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, const CBMCall *call, - const cbm_resolution_t *res) { - const char *op_type = "operation"; - char op_name[CBM_SZ_128]; - bool named = cbm_service_pattern_graphql_operation(call->first_string_arg, &op_type, op_name, - sizeof(op_name)); - - char route_name[CBM_SZ_256]; - char route_qn[CBM_SZ_256]; - if (named) { - snprintf(route_name, sizeof(route_name), "%s", op_name); - snprintf(route_qn, sizeof(route_qn), "__graphql__%s", op_name); - } else { - snprintf(route_name, sizeof(route_name), "(anonymous %s)", op_type); - snprintf(route_qn, sizeof(route_qn), "__graphql__%s__anonymous", op_type); - } - - int64_t route_id = cbm_gbuf_upsert_node(gbuf, "Route", route_name, route_qn, "", 0, 0, - "{\"source\":\"graphql\"}"); - - char esc_c[CBM_SZ_256]; - char esc_op[CBM_SZ_512]; - cbm_json_escape(esc_c, sizeof(esc_c), call->callee_name); - cbm_json_escape(esc_op, sizeof(esc_op), route_name); - char props[CBM_SZ_1K]; - snprintf(props, sizeof(props), - "{\"callee\":\"%s\",\"operation\":\"%s\",\"operation_type\":\"%s\"," - "\"confidence\":%.2f}", - esc_c, esc_op, op_type, res->confidence); - cbm_gbuf_insert_edge(gbuf, source->id, route_id, "GRAPHQL_CALLS", props); -} - -/* Emit TRPC_CALLS edge. Extract procedure path from callee chain. */ -static void emit_trpc_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, const CBMCall *call, - const cbm_resolution_t *res) { - /* tRPC calls: trpc.user.getById.query() → extract "user.getById" */ - const char *callee = call->callee_name; - if (!callee) { - return; - } - /* Strip trailing .query/.mutate/.subscribe */ - char proc[CBM_SZ_256]; - snprintf(proc, sizeof(proc), "%s", callee); - char *last_dot = strrchr(proc, '.'); - if (last_dot && (strcmp(last_dot, ".query") == 0 || strcmp(last_dot, ".mutate") == 0 || - strcmp(last_dot, ".subscribe") == 0 || strcmp(last_dot, ".useQuery") == 0 || - strcmp(last_dot, ".useMutation") == 0)) { - *last_dot = '\0'; - } - /* Strip leading trpc. */ - const char *p = proc; - if (strncmp(p, "trpc.", CBM_SZ_5) == 0) { - p += CBM_SZ_5; - } - - char route_qn[CBM_SZ_512]; - snprintf(route_qn, sizeof(route_qn), "__trpc__%s", p); - - int64_t route_id = - cbm_gbuf_upsert_node(gbuf, "Route", p, route_qn, "", 0, 0, "{\"source\":\"trpc\"}"); - - char esc_c[CBM_SZ_256]; - cbm_json_escape(esc_c, sizeof(esc_c), call->callee_name); - char props[CBM_SZ_1K]; - snprintf(props, sizeof(props), "{\"callee\":\"%s\",\"procedure\":\"%s\",\"confidence\":%.2f}", - esc_c, p, res->confidence); - cbm_gbuf_insert_edge(gbuf, source->id, route_id, "TRPC_CALLS", props); -} - -/* When suppress_plain_calls is true (a TS/JS/TSX weak short-name member-call - * match, #592/#606), every service classification below still runs — only the - * plain CALLS fall-through (emit_normal_calls_edge) is skipped. detect_url_in_args - * and the HTTP/ASYNC/gRPC/GraphQL/tRPC/CONFIG/route branches are unaffected, so - * a verb-suffix HTTP client (api.patch('/x')), broker, or route registration - * keeps its edge; only the fabricated project CALLS edge is dropped. */ -static void emit_service_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, - const cbm_gbuf_node_t *target, const CBMCall *call, - const cbm_resolution_t *res, const char *module_qn, - const cbm_registry_t *registry, const cbm_gbuf_t *main_gbuf, - const char **imp_keys, const char **imp_vals, int imp_count, - bool suppress_plain_calls, CBMLanguage lang, - const char *route_mount) { - cbm_svc_kind_t svc = cbm_service_pattern_match(res->qualified_name); - const char *arg = call->first_string_arg; - - /* Also detect route registration by callee name suffix alone (handles unresolved - * local variables like app.include_router where QN resolution fails). */ - if (svc == CBM_SVC_NONE && cbm_service_pattern_route_method(call->callee_name) != NULL) { - svc = CBM_SVC_ROUTE_REG; - } - - /* Detect gRPC stub method calls by resolved QN. - * Go pattern: pb.NewCartServiceClient(conn).GetCart(ctx, req) - * Tree-sitter extracts GetCart as the callee, which resolves to the - * generated pb interface method (QN contains "ServiceClient"). */ - if (svc == CBM_SVC_NONE && res->qualified_name) { - if (strstr(res->qualified_name, "ServiceClient") != NULL || - strstr(res->qualified_name, "ServiceGrpc") != NULL || - strstr(res->qualified_name, "Servicer") != NULL) { - svc = CBM_SVC_GRPC; - } - } - - if (svc == CBM_SVC_ROUTE_REG) { - const char *handler_ref = NULL; - const char *route_path = find_route_path_in_args(call, &handler_ref); - if (route_path) { - emit_route_registration(gbuf, source, call, route_path, handler_ref, module_qn, - registry, main_gbuf, imp_keys, imp_vals, imp_count, - route_mount); - return; - } - /* No path found — fall through to normal CALLS edge */ - } - - bool has_url = (arg && arg[0] != '\0' && (arg[0] == '/' || strstr(arg, "://") != NULL)); - bool has_topic = (arg && arg[0] != '\0' && svc == CBM_SVC_ASYNC && strlen(arg) > PP_ESC_SPACE); - /* Set when this call already has its HTTP_CALLS edge from the service - * patterns (typed with the verb its callee names). */ - bool http_edge_emitted = false; - - if ((svc == CBM_SVC_HTTP || svc == CBM_SVC_ASYNC) && (has_url || has_topic)) { - emit_http_async_service_edge(gbuf, source, call, res, svc, arg); - http_edge_emitted = svc == CBM_SVC_HTTP; - } else if (svc == CBM_SVC_GRPC) { - emit_grpc_edge(gbuf, source, call, res); - } else if (svc == CBM_SVC_GRAPHQL) { - emit_graphql_edge(gbuf, source, call, res); - } else if (svc == CBM_SVC_TRPC) { - emit_trpc_edge(gbuf, source, call, res); - } else if (svc == CBM_SVC_CONFIG) { - emit_config_edge(gbuf, source, target, call, res, arg); - } else if (!suppress_plain_calls) { - emit_normal_calls_edge(gbuf, source, target, call, res); - } - - /* The arg-URL heuristic is for calls no service pattern knows (a local - * fetch wrapper). On a classified HTTP call that already has its typed - * edge, skip it so we do not mint a second, method-less ANY edge. */ - if (!http_edge_emitted) { - detect_url_in_args(gbuf, source, call, lang, svc); - } -} - -/* The #725 guard refuses a suffix_match binding across a language boundary, - * which leaves the callee unresolved. An unresolved callee the route - * classifier recognises is still a registration: `Route::get('/x', ...)` - * beside a JS `get`, `app.get('/x', h)` beside a Python `get`. Emit exactly - * the Route + CALLS + HANDLES that the unresolved-callee (callee_suffix) - * fallback in resolve_file_calls emits for it, classified by callee name the - * same way emit_service_edge classifies that fallback, and nothing else: no - * CALLS edge to the refused target and no URL-argument scan. Dropping the - * whole call lost every GET registration in a mixed-language repo while POST - * (no `post` to collide with) survived. */ -static void emit_xlang_refused_route(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, - const CBMCall *call, const char *module_qn, - const cbm_registry_t *registry, const cbm_gbuf_t *main_gbuf, - const char **imp_keys, const char **imp_vals, int imp_count, - const char *route_mount) { - if (cbm_service_pattern_route_method(call->callee_name) == NULL) { - return; - } - cbm_svc_kind_t svc = cbm_service_pattern_match(call->callee_name); - if (svc != CBM_SVC_NONE && svc != CBM_SVC_ROUTE_REG) { - return; - } - const char *handler_ref = NULL; - const char *route_path = find_route_path_in_args(call, &handler_ref); - if (route_path) { - emit_route_registration(gbuf, source, call, route_path, handler_ref, module_qn, registry, - main_gbuf, imp_keys, imp_vals, imp_count, route_mount); - } -} - -/* Find the source node for an edge: enclosing function or file node. */ -static const cbm_gbuf_node_t *find_source_node(const cbm_gbuf_t *gbuf, const char *project, - const char *rel, const char *enclosing_qn) { - const cbm_gbuf_node_t *src = NULL; - if (enclosing_qn) { - src = cbm_gbuf_find_by_qn(gbuf, enclosing_qn); - /* A class-level reference in a directory-module language carries the - * DIRECTORY module QN, which hits the shared Folder/Project node — - * attribute to this file's File node instead (#787). */ - if (cbm_pipeline_node_is_dir_container(src)) { - src = NULL; - } - } - if (!src) { - char *file_qn = cbm_pipeline_fqn_compute(project, rel, "__file__"); - src = cbm_gbuf_find_by_qn(gbuf, file_qn); - free(file_qn); - } - return src; -} - -/* Field type hint resolution for obj.Method() with multiple candidates. - * Strips C# field prefixes (_ / m_), capitalizes to get type name, and - * checks if TypeName.Method or ITypeName.Method exists among candidates. */ -static void try_field_type_hint(resolve_ctx_t *rc, cbm_resolution_t *res, const char *callee_name, - int64_t source_id) { - if (!res->qualified_name || res->candidate_count <= SKIP_ONE) { - return; - } - const char *dot = strchr(callee_name, '.'); - if (!dot) { - return; - } - size_t plen = (size_t)(dot - callee_name); - char obj_name[CBM_SZ_256]; - if (plen >= sizeof(obj_name)) { - return; - } - memcpy(obj_name, callee_name, plen); - obj_name[plen] = '\0'; - - const char *type_hint = obj_name; - if (type_hint[0] == '_') { - type_hint++; - } - if (type_hint[0] == 'm' && type_hint[SKIP_ONE] == '_') { - type_hint += PP_CSHARP_M_PREFIX_LEN; - } - - char type_name[CBM_SZ_256]; - snprintf(type_name, sizeof(type_name), "%s", type_hint); - if (type_name[0] >= 'a' && type_name[0] <= 'z') { - type_name[0] -= ('a' - 'A'); - } - - char iface_name[CBM_SZ_256]; - snprintf(iface_name, sizeof(iface_name), "I%s", type_name); - - const char *method = dot + SKIP_ONE; - const char **cands = NULL; - int cand_count = 0; - cbm_registry_find_by_name(rc->registry, method, &cands, &cand_count); - for (int ci = 0; ci < cand_count; ci++) { - if (strstr(cands[ci], type_name) || strstr(cands[ci], iface_name)) { - const cbm_gbuf_node_t *better = cbm_gbuf_find_by_qn(rc->main_gbuf, cands[ci]); - if (better && better->id != source_id) { - res->qualified_name = cands[ci]; - res->confidence = PP_FIELD_HINT_CONF; - res->strategy = "field_type_hint"; - return; - } - } - } -} - -/* Free a strdup'd key stored in the per-file lsp_idx hash table. */ -static void lsp_idx_free_key(const char *key, void *value, void *ud) { - (void)value; - (void)ud; - free((char *)key); -} - -/* Resolve calls for one file and emit CALLS/HTTP_CALLS/ASYNC_CALLS edges. */ -static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CBMFileResult *result, - const char *rel, const char *module_qn, const char **imp_keys, - const char **imp_vals, int imp_count, CBMLanguage lang) { - /* Framework mount of this file's routes (Laravel 11+ withRouting, #1146). */ - char route_mount[CBM_SZ_128]; - cbm_laravel_file_route_mount(rc->repo_path, rel, lang, result, route_mount, - sizeof(route_mount)); - - /* Build a per-file hash index of resolved_calls keyed by - * "caller_qn|callee_short" for O(1) lookup. cbm_pipeline_find_lsp_ - * resolution would otherwise do an O(N) linear scan over - * resolved_calls for EACH of result->calls.count calls — the - * dominant cost in parallel_resolve on kubernetes (~50s of pure - * scanning). On insert, keep the highest-confidence entry per key - * (matches the original "best" tie-break). Skip the build entirely - * when there are no calls (nothing to look up) or no resolved - * entries (lookups would all miss). */ - CBMHashTable *lsp_idx = NULL; - if (result->calls.count > 0 && result->resolved_calls.count > 0) { - lsp_idx = cbm_ht_create((uint32_t)result->resolved_calls.count * 2u + 16u); - if (lsp_idx) { - for (int i = 0; i < result->resolved_calls.count; i++) { - CBMResolvedCall *rc_e = &result->resolved_calls.items[i]; - if (!rc_e->caller_qn || !rc_e->callee_qn || - rc_e->kind != CBM_RESOLVED_INVOCATION || - rc_e->confidence < CBM_LSP_CONFIDENCE_FLOOR) { - continue; - } - const char *short_name = strrchr(rc_e->callee_qn, '.'); - short_name = short_name ? short_name + 1 : rc_e->callee_qn; - char key[1024]; - /* A row with a site span proves one call site: key it by that - * site so it never answers a same-named call elsewhere in the - * caller (cbm_pipeline_lsp_site_admits). */ - int kn = rc_e->site_end_byte > rc_e->site_start_byte - ? snprintf(key, sizeof(key), "%s|%s@%u", rc_e->caller_qn, short_name, - rc_e->site_start_byte) - : snprintf(key, sizeof(key), "%s|%s", rc_e->caller_qn, short_name); - if (kn <= 0 || kn >= (int)sizeof(key)) - continue; - CBMResolvedCall *existing = (CBMResolvedCall *)cbm_ht_get(lsp_idx, key); - if (!existing) { - /* New entry — strdup so the key outlives the loop body. */ - char *kdup = strdup(key); - if (kdup) - cbm_ht_set(lsp_idx, kdup, rc_e); - } else if (rc_e->confidence > existing->confidence) { - /* Update value; reuse stored key pointer to avoid leak. */ - const char *skey = cbm_ht_get_key(lsp_idx, key); - if (skey) - cbm_ht_set(lsp_idx, skey, rc_e); - } - } - } - } - - for (int c = 0; c < result->calls.count; c++) { - CBMCall *call = &result->calls.items[c]; - if (!call->callee_name) { - continue; - } - uint64_t _rc_t0 = extract_now_ns(); - const cbm_gbuf_node_t *source_node = - find_source_node(rc->main_gbuf, rc->project_name, rel, call->enclosing_func_qn); - atomic_fetch_add_explicit(&rc->time_ns_rc_source, extract_now_ns() - _rc_t0, - memory_order_relaxed); - if (!source_node) { - continue; - } - - /* LSP-resolved calls take precedence over registry textual matching. - * The shared helper and CBM_LSP_CONFIDENCE_FLOOR (lsp_resolve.h) - * decide which LSP overrides are admitted. Unique-tail fallbacks - * are JVM-only (see cbm_pipeline_lsp_allow_tail_match). */ - bool allow_tail = cbm_pipeline_lsp_allow_tail_match(lang); - cbm_resolution_t res = {0}; - const CBMResolvedCall *lsp = NULL; - _rc_t0 = extract_now_ns(); - if (lsp_idx && call->enclosing_func_qn) { - const char *call_leaf = cbm_pipeline_call_callee_leaf(call->callee_name); - char key[1024]; - int kn = call_leaf ? snprintf(key, sizeof(key), "%s|%s@%u", call->enclosing_func_qn, - call_leaf, call->site_start_byte) - : -1; - if (kn > 0 && kn < (int)sizeof(key)) { - lsp = (const CBMResolvedCall *)cbm_ht_get(lsp_idx, key); - } - kn = call_leaf && !lsp - ? snprintf(key, sizeof(key), "%s|%s", call->enclosing_func_qn, call_leaf) - : -1; - if (kn > 0 && kn < (int)sizeof(key)) { - lsp = (const CBMResolvedCall *)cbm_ht_get(lsp_idx, key); - } - } - if (!lsp) { - /* Fallback to the linear scan for edge cases the index may - * miss (e.g. callee_name that wasn't the registered short - * name). Keeps semantics identical. */ - lsp = cbm_pipeline_find_lsp_resolution(&result->resolved_calls, call, allow_tail); - } - atomic_fetch_add_explicit(&rc->time_ns_rc_lsp_lookup, extract_now_ns() - _rc_t0, - memory_order_relaxed); - _rc_t0 = extract_now_ns(); - const cbm_gbuf_node_t *lsp_target = NULL; - if (lsp) { - /* Canonicalise to the gbuf node's QN so res.qualified_name matches - * the gbuf even when the cross-file fallback had to prefix the - * project name. */ - lsp_target = cbm_pipeline_lsp_target_node(rc->main_gbuf, rc->project_name, - lsp->callee_qn, allow_tail); - if (lsp_target) { - res.qualified_name = lsp_target->qualified_name; - res.strategy = lsp->strategy ? lsp->strategy : "lsp_override"; - res.confidence = (double)lsp->confidence; - res.candidate_count = 1; - ws->lsp_overrides++; - } - } - /* #1085: fall back to the registry resolver whenever the LSP did not - * yield a gbuf-resolvable target — whether no LSP resolution existed, - * OR the LSP was confident but its callee_qn isn't a node in the gbuf - * (the JSX-via-tsconfig-alias case: the TS LSP resolves the element - * ref to an alias-path QN that never matches a def node, so lsp_target - * is NULL). The old `else` ran the registry ONLY when lsp was null, so - * an LSP-with-unresolvable-target dropped the edge outright — silently - * losing every alias-imported JSX component edge (~21% of a Next.js - * call graph). Falling THROUGH to the registry keeps them via the - * import_map / unique_name resolution. */ - if (!res.qualified_name || !res.qualified_name[0]) { - res = cbm_registry_resolve(rc->registry, call->callee_name, module_qn, imp_keys, - imp_vals, imp_count); - } - atomic_fetch_add_explicit(&rc->time_ns_rc_resolve, extract_now_ns() - _rc_t0, - memory_order_relaxed); - - _rc_t0 = extract_now_ns(); - try_field_type_hint(rc, &res, call->callee_name, source_node->id); - atomic_fetch_add_explicit(&rc->time_ns_rc_hint, extract_now_ns() - _rc_t0, - memory_order_relaxed); - - /* Perl call-graph noise guard (#476). Perl has no LSP resolver; for - * builtins (push/shift/ - * keys/...) and method calls ($obj->m, unresolved receiver), suppress - * only WEAK cross-file short-name matches and keep the high-confidence - * same_module / import_map strategies so a genuine same-file or - * imported call to a builtin-named sub still resolves. Placed after the - * field-type hint so a hint cannot re-introduce a suppressed edge. - * Gated to Perl — other languages are unaffected. */ - if (cbm_perl_suppress_generic_match(lang == CBM_LANG_PERL, call->is_method, - call->callee_name, res.strategy)) { - continue; - } - - /* Dynamic-language weak-member suppression (#592/#606/#1276). The - * receiver-aware guard must NOT drop this call here: doing so would also - * skip the #523 callee-name service bypass below, emit_service_edge's - * route/gRPC/config branches, and its unconditional detect_url_in_args - * (which classifies verb-suffix HTTP clients like api.patch('/x')). - * Instead, defer to the emit path and suppress ONLY the plain-CALLS - * fall-through (emit_normal_calls_edge), so every service edge stays - * main-identical by construction. res.strategy may carry an lsp_* value - * here (LSP-resolved calls keep res through this point); the helper's - * EXPLICIT drop-list leaves lsp_ts_method / lsp_cross untouched. See - * #606 direction. */ - bool suppress_weak_member = lang == CBM_LANG_PYTHON || lang == CBM_LANG_JAVASCRIPT || - lang == CBM_LANG_TYPESCRIPT || lang == CBM_LANG_TSX || - lang == CBM_LANG_ZIG; - /* Bare-call local-binding suppression. A member call has a receiver the - * guard above can reason about; a bare `run()` has none, so that guard - * cannot see this class at all. Python-only today because the - * extraction flag is set only for Python. */ - bool suppress_weak_local_binding = lang == CBM_LANG_PYTHON; - bool drop_plain_call = - cbm_suppress_weak_member_match(suppress_weak_member, call->is_method, res.strategy) || - cbm_suppress_zig_qualified_member(lang == CBM_LANG_ZIG, call->is_method, res.strategy) || - cbm_suppress_weak_local_binding_call(suppress_weak_local_binding, - call->callee_is_locally_bound, res.strategy); - - /* First: a call on an axios.create() instance (#1916) — decided by the - * receiver's binding, before the #523 spelling check and the - * route-registration suffix fallback. */ - char client_url[CBM_SZ_512]; - if (cbm_pipeline_http_client_call_url(rc->main_gbuf, rc->project_name, rel, result, - imp_keys, imp_vals, imp_count, call, client_url, - sizeof(client_url))) { - cbm_resolution_t svc_res = {.qualified_name = call->callee_name, - .confidence = PP_HALF_CONF, - .strategy = "http_client_instance"}; - emit_http_async_service_edge(ws->local_edge_buf, source_node, call, &svc_res, - CBM_SVC_HTTP, client_url); - continue; - } - /* Service-pattern HTTP/ASYNC client call (`requests.get(url)`): the - * service signal lives in the callee_name. The registry can mis-resolve - * it to a spurious builtin short-name match (`requests.get` -> - * `builtins.dict.get` via "get"), which is non-empty and not an HTTP - * pattern, so the resolved-QN service checks below miss it and the call - * is dropped. Detect it on the callee_name FIRST so the HTTP_CALLS/ - * ASYNC_CALLS edge is emitted regardless (target is a synthesized route - * node, not the unindexed library). (#523) */ - cbm_svc_kind_t csvc = cbm_service_pattern_match(call->callee_name); - if (csvc == CBM_SVC_HTTP || csvc == CBM_SVC_ASYNC) { - const char *cu = call->first_string_arg; - bool chas_url = cu && cu[0] != '\0' && - (cu[0] == '/' || strstr(cu, "://") != NULL || - (csvc == CBM_SVC_ASYNC && strlen(cu) > PP_ESC_SPACE)); - if (chas_url) { - cbm_resolution_t svc_res = {.qualified_name = call->callee_name, - .confidence = PP_HALF_CONF, - .strategy = "service_pattern"}; - emit_service_edge(ws->local_edge_buf, source_node, source_node, call, &svc_res, - module_qn, rc->registry, rc->main_gbuf, imp_keys, imp_vals, - imp_count, false, lang, route_mount); - continue; - } - } - - bool unresolved = !res.qualified_name || res.qualified_name[0] == '\0'; - if (unresolved || cbm_registry_strategy_is_weak(res.strategy)) { - /* A callee an import binds to an external client library - * (`send` under `from paho.mqtt import publish as send`): - * spelled through its import, then classified. The binding - * outranks a weak name match. */ - char spelled[CBM_SZ_512]; - cbm_svc_kind_t isvc = cbm_pipeline_external_import_service( - lang, &result->imports, imp_keys, imp_count, call, spelled, sizeof(spelled)); - if (isvc != CBM_SVC_NONE) { - CBMCall through = *call; - through.callee_name = spelled; - cbm_resolution_t imp_res = {.qualified_name = spelled, - .confidence = PP_HALF_CONF, - .strategy = "import_binding"}; - emit_http_async_service_edge(ws->local_edge_buf, source_node, &through, &imp_res, - isvc, through.first_string_arg); - ws->calls_resolved++; - continue; - } - } - if (unresolved) { - if (cbm_service_pattern_route_method(call->callee_name) != NULL) { - cbm_resolution_t fake_res = {.qualified_name = call->callee_name, - .confidence = PP_HALF_CONF, - .strategy = "callee_suffix"}; - /* The caller stands in for the unknown target, so a plain - * CALLS fall-through would be a self-loop. */ - emit_service_edge(ws->local_edge_buf, source_node, source_node, call, &fake_res, - module_qn, rc->registry, rc->main_gbuf, imp_keys, imp_vals, - imp_count, true, lang, route_mount); - } else if (cbm_service_pattern_is_global_fetch(call->callee_name)) { - /* Native `fetch()` (#856): only the global API once resolution - * has failed to find a local/imported `fetch`. Call the low-level - * emitter directly — emit_service_edge re-derives its own kind - * from res->qualified_name via cbm_service_pattern_match, which - * "fetch" deliberately never matches. */ - const char *u = call->first_string_arg; - if (u && u[0] != '\0' && (u[0] == '/' || strstr(u, "://") != NULL)) { - cbm_resolution_t fake_res = {.qualified_name = call->callee_name, - .confidence = PP_HALF_CONF, - .strategy = "service_pattern"}; - emit_http_async_service_edge(ws->local_edge_buf, source_node, call, &fake_res, - CBM_SVC_HTTP, u); - } - } - continue; - } - /* Reuse lsp_target as target_node when LSP resolved — avoids a - * second cbm_gbuf_find_by_qn lookup. try_field_type_hint may have - * upgraded res.qualified_name to a different candidate, in which - * case we must re-resolve. */ - _rc_t0 = extract_now_ns(); - const cbm_gbuf_node_t *target_node; - if (lsp_target && res.qualified_name == lsp_target->qualified_name) { - target_node = lsp_target; - } else { - target_node = cbm_gbuf_find_by_qn(rc->main_gbuf, res.qualified_name); - } - atomic_fetch_add_explicit(&rc->time_ns_rc_target, extract_now_ns() - _rc_t0, - memory_order_relaxed); - if (target_node && source_node->id != target_node->id && - cbm_suppress_cross_language_suffix_match(lang, target_node->file_path, res.strategy)) { - /* #725: do not emit a suffix_match - * CALLS edge across a language boundary. A route registration - * behind the refused binding still gets its Route. */ - emit_xlang_refused_route(ws->local_edge_buf, source_node, call, module_qn, rc->registry, - rc->main_gbuf, imp_keys, imp_vals, imp_count, route_mount); - continue; - } - if (!target_node || source_node->id == target_node->id) { - /* HTTP/ASYNC calls to an EXTERNAL client library (`requests.get(url)`) - * resolve to an unindexed QN (target_node == NULL), but their edge - * target is a synthesized route node, not the library — emit them - * anyway so cross-repo matching has an HTTP_CALLS edge to work with - * (#523). */ - cbm_svc_kind_t psvc = cbm_service_pattern_match(res.qualified_name); - if ((psvc == CBM_SVC_HTTP || psvc == CBM_SVC_ASYNC) && !target_node) { - const char *u = call->first_string_arg; - bool url_or_topic = u && u[0] != '\0' && - (u[0] == '/' || strstr(u, "://") != NULL || - (psvc == CBM_SVC_ASYNC && strlen(u) > PP_ESC_SPACE)); - if (url_or_topic) { - emit_service_edge(ws->local_edge_buf, source_node, NULL, call, &res, module_qn, - rc->registry, rc->main_gbuf, imp_keys, imp_vals, imp_count, - false, lang, route_mount); - ws->calls_resolved++; - } - } - continue; - } - _rc_t0 = extract_now_ns(); - emit_service_edge(ws->local_edge_buf, source_node, target_node, call, &res, module_qn, - rc->registry, rc->main_gbuf, imp_keys, imp_vals, imp_count, - drop_plain_call, lang, route_mount); - atomic_fetch_add_explicit(&rc->time_ns_rc_emit, extract_now_ns() - _rc_t0, - memory_order_relaxed); - ws->calls_resolved++; - } - if (lsp_idx) { - cbm_ht_foreach(lsp_idx, lsp_idx_free_key, NULL); - cbm_ht_free(lsp_idx); - } -} - -/* Resolve USAGE and CALL_REFERENCE edges for one file. CALL_REFERENCE only - * through the exact LSP join, never from the textual registry. */ -static void resolve_file_usages(resolve_ctx_t *rc, resolve_worker_state_t *ws, - CBMFileResult *result, const char *rel, const char *module_qn, - const char **imp_keys, const char **imp_vals, int imp_count, - CBMLanguage lang) { - cbm_pipeline_ref_index_t ref_index; - (void)cbm_pipeline_ref_index_build(&result->resolved_calls, &ref_index); - bool allow_tail = cbm_pipeline_lsp_allow_tail_match(lang); - for (int u = 0; u < result->usages.count; u++) { - CBMUsage *usage = &result->usages.items[u]; - if (!usage->ref_name) { - continue; - } - const cbm_gbuf_node_t *src = - find_source_node(rc->main_gbuf, rc->project_name, rel, usage->enclosing_func_qn); - if (!src) { - continue; - } - const cbm_gbuf_node_t *tgt = NULL; - const CBMResolvedCall *row = NULL; - cbm_ref_join_t join = cbm_pipeline_join_lsp_reference( - rc->main_gbuf, rc->project_name, &ref_index, usage, allow_tail, &tgt, &row); - if (join == CBM_REF_JOIN_SKIP || (join == CBM_REF_JOIN_VALUE && usage->reference_only)) { - continue; - } - if (join == CBM_REF_JOIN_NONE) { - cbm_resolution_t res = cbm_registry_resolve(rc->registry, usage->ref_name, module_qn, - imp_keys, imp_vals, imp_count); - if (!res.qualified_name || res.qualified_name[0] == '\0') { - continue; - } - if (!cbm_pipeline_reference_fallback_allowed(lang, usage, res.strategy, imp_keys, - imp_count)) { - continue; - } - tgt = cbm_gbuf_find_by_qn(rc->main_gbuf, res.qualified_name); - /* #1928: the registry fallback is a bare-name guess — never let it - * bind a reference across a language boundary. */ - if (tgt && cbm_suppress_cross_language_ref(lang, tgt->file_path)) { - continue; - } - /* #1942/#1962: a bare Go reference can never denote a struct - * field; the member half of a selector may. */ - if (tgt && cbm_go_suppress_bare_field_ref(lang == CBM_LANG_GO, usage->is_member_access, - tgt->label)) { - continue; - } - } - if (!tgt || src->id == tgt->id) { - continue; - } - char uprops[CBM_SZ_512]; - char esc_ref[CBM_SZ_256]; /* sliced source text: escape quotes/newlines */ - cbm_json_escape(esc_ref, sizeof(esc_ref), usage->ref_name); - cbm_pipeline_reference_props(uprops, sizeof(uprops), esc_ref, - join == CBM_REF_JOIN_CALLABLE ? row : NULL); - cbm_gbuf_insert_edge(ws->local_edge_buf, src->id, tgt->id, - join == CBM_REF_JOIN_CALLABLE ? "CALL_REFERENCE" : "USAGE", uprops); - ws->usages_resolved++; - } - cbm_pipeline_ref_index_free(&ref_index); -} - -/* Resolve throws/raises for one file. */ -static void resolve_file_throws(resolve_ctx_t *rc, resolve_worker_state_t *ws, - CBMFileResult *result, const char *module_qn, const char **imp_keys, - const char **imp_vals, int imp_count) { - for (int t = 0; t < result->throws.count; t++) { - CBMThrow *thr = &result->throws.items[t]; - if (!thr->exception_name || !thr->enclosing_func_qn) { - continue; - } - const cbm_gbuf_node_t *src = cbm_gbuf_find_by_qn(rc->main_gbuf, thr->enclosing_func_qn); - if (!src) { - continue; - } - const char *edge_type = is_checked_exception(thr->exception_name) ? "THROWS" : "RAISES"; - cbm_resolution_t res = cbm_registry_resolve(rc->registry, thr->exception_name, module_qn, - imp_keys, imp_vals, imp_count); - if (!res.qualified_name || res.qualified_name[0] == '\0') { - continue; - } - const cbm_gbuf_node_t *tgt = cbm_gbuf_find_by_qn(rc->main_gbuf, res.qualified_name); - if (!tgt || src->id == tgt->id) { - continue; - } - cbm_gbuf_insert_edge(ws->local_edge_buf, src->id, tgt->id, edge_type, "{}"); - } -} - -/* Resolve reads/writes for one file. */ -static void resolve_file_rw(resolve_ctx_t *rc, resolve_worker_state_t *ws, CBMFileResult *result, - const char *rel, const char *module_qn, const char **imp_keys, - const char **imp_vals, int imp_count, CBMLanguage lang) { - for (int r = 0; r < result->rw.count; r++) { - CBMReadWrite *rw = &result->rw.items[r]; - if (!rw->var_name) { - continue; - } - const cbm_gbuf_node_t *src = - find_source_node(rc->main_gbuf, rc->project_name, rel, rw->enclosing_func_qn); - if (!src) { - continue; - } - cbm_resolution_t res = cbm_registry_resolve(rc->registry, rw->var_name, module_qn, imp_keys, - imp_vals, imp_count); - if (!res.qualified_name || res.qualified_name[0] == '\0') { - continue; - } - const cbm_gbuf_node_t *tgt = cbm_gbuf_find_by_qn(rc->main_gbuf, res.qualified_name); - if (!tgt || src->id == tgt->id) { - continue; - } - /* #1928: every resolution here is a bare-name registry guess — never - * let it bind a read/write across a language boundary. */ - if (cbm_suppress_cross_language_ref(lang, tgt->file_path)) { - continue; - } - /* #1942/#1962: a bare Go reference can never denote a struct field; - * a selector-LHS write (`t.err = x`) may bind it. */ - if (cbm_go_suppress_bare_field_ref(lang == CBM_LANG_GO, rw->is_member_access, tgt->label)) { - continue; - } - const char *etype = rw->is_write ? "WRITES" : "READS"; - cbm_gbuf_insert_edge(ws->local_edge_buf, src->id, tgt->id, etype, "{}"); - } -} - -/* Resolve base_classes → INHERITS / IMPLEMENTS edges for one definition. */ -static void resolve_def_inherits(resolve_ctx_t *rc, resolve_worker_state_t *ws, - const CBMDefinition *def, const cbm_gbuf_node_t *node, - const char *mq, const char **ik, const char **iv, int ic) { - if (!def->base_classes) { - return; - } - for (int b = 0; def->base_classes[b]; b++) { - const char *bqn = resolve_as_class(rc->registry, def->base_classes[b], mq, ik, iv, ic); - if (!bqn) { - continue; - } - const cbm_gbuf_node_t *bn = cbm_gbuf_find_by_qn(rc->main_gbuf, bqn); - if (bn && node->id != bn->id) { - /* A base that resolves to an Interface is an IMPLEMENTS relation - * (Java `implements`, C# `: IFace`, TS `implements`); a Class/ - * Type/Enum base is plain INHERITS. */ - const char *base_label = cbm_registry_label_of(rc->registry, bqn); - const char *edge_type = - (base_label && strcmp(base_label, "Interface") == 0) ? "IMPLEMENTS" : "INHERITS"; - cbm_gbuf_insert_edge(ws->local_edge_buf, node->id, bn->id, edge_type, "{}"); - ws->semantic_resolved++; - } - } -} - -/* Resolve decorators → DECORATES edges for one definition. */ -static void resolve_def_decorators(resolve_ctx_t *rc, resolve_worker_state_t *ws, - const CBMDefinition *def, const cbm_gbuf_node_t *node, - const char *mq, const char **ik, const char **iv, int ic) { - if (!def->decorators) { - return; - } - for (int dc = 0; def->decorators[dc]; dc++) { - char fn[CBM_SZ_256]; - extract_decorator_func(def->decorators[dc], fn, sizeof(fn)); - if (fn[0] == '\0') { - continue; - } - cbm_resolution_t res = cbm_registry_resolve(rc->registry, fn, mq, ik, iv, ic); - if ((!res.qualified_name || res.qualified_name[0] == '\0') && !strchr(fn, '.')) { - /* C# attributes are referenced by their short name (`[Log]`) but - * declared with an `Attribute` suffix (`class LogAttribute`). */ - char with_suffix[CBM_SZ_256]; - int wn = snprintf(with_suffix, sizeof(with_suffix), "%sAttribute", fn); - if (wn > 0 && (size_t)wn < sizeof(with_suffix)) { - res = cbm_registry_resolve(rc->registry, with_suffix, mq, ik, iv, ic); - } - } - const cbm_gbuf_node_t *dn = NULL; - if (res.qualified_name && res.qualified_name[0] != '\0') { - dn = cbm_gbuf_find_by_qn(rc->main_gbuf, res.qualified_name); - } - int64_t dn_id = 0; - if (dn) { - dn_id = dn->id; - } else { - /* External/stdlib decorator (Rust `#[derive(Debug)]`, Swift - * `@discardableResult`, Scala `@deprecated`, Python `@cache`, - * Java `@Override`, ...): no local symbol resolves. Materialise a - * synthetic "Decorator" node so the DECORATES relation is recorded. - * The node is created in the per-worker local_edge_buf (shared-ID - * gbuf); the sequential merge dedupes by QN across workers, so all - * uses of the same decorator name collapse to one node project-wide - * and the edge target IDs are remapped consistently. */ - char syn_qn[CBM_SZ_512]; - snprintf(syn_qn, sizeof(syn_qn), "", fn); - dn_id = - cbm_gbuf_upsert_node(ws->local_edge_buf, "Decorator", fn, syn_qn, "", 0, 0, "{}"); - } - if (dn_id != 0 && node->id != dn_id) { - /* Decorator SOURCE TEXT can contain quotes and raw newlines - * (e.g. @register.tag("block"), multi-line @override_settings) — - * interpolating it raw produced malformed properties JSON that - * aborts every json_extract consumer (django: 3826 such edges). */ - char esc_dec[CBM_SZ_256]; - cbm_json_escape(esc_dec, sizeof(esc_dec), def->decorators[dc]); - char dp[CBM_SZ_512]; - snprintf(dp, sizeof(dp), "{\"decorator\":\"%s\"}", esc_dec); - cbm_gbuf_insert_edge(ws->local_edge_buf, node->id, dn_id, "DECORATES", dp); - /* Ensure a reference-style edge exists so the decorator appears in queries - * without being misclassified as a real call by downstream passes. */ - cbm_gbuf_insert_edge(ws->local_edge_buf, node->id, dn_id, "USAGE", "{}"); - ws->semantic_resolved++; - } - } -} - -/* Resolve INHERITS + DECORATES + IMPLEMENTS for one file. */ -static void resolve_file_semantic(resolve_ctx_t *rc, resolve_worker_state_t *ws, - CBMFileResult *result, const char *module_qn, - const char **imp_keys, const char **imp_vals, int imp_count) { - for (int d = 0; d < result->defs.count; d++) { - CBMDefinition *def = &result->defs.items[d]; - if (!def->qualified_name) { - continue; - } - const cbm_gbuf_node_t *node = cbm_gbuf_find_by_qn(rc->main_gbuf, def->qualified_name); - if (!node) { - continue; - } - resolve_def_inherits(rc, ws, def, node, module_qn, imp_keys, imp_vals, imp_count); - resolve_def_decorators(rc, ws, def, node, module_qn, imp_keys, imp_vals, imp_count); - } - for (int t = 0; t < result->impl_traits.count; t++) { - CBMImplTrait *it = &result->impl_traits.items[t]; - if (!it->trait_name || !it->struct_name) { - continue; - } - const char *tqn = resolve_as_class(rc->registry, it->trait_name, module_qn, imp_keys, - imp_vals, imp_count); - const char *sqn = resolve_as_class(rc->registry, it->struct_name, module_qn, imp_keys, - imp_vals, imp_count); - if (!tqn || !sqn) { - continue; - } - const cbm_gbuf_node_t *tn = cbm_gbuf_find_by_qn(rc->main_gbuf, tqn); - const cbm_gbuf_node_t *sn = cbm_gbuf_find_by_qn(rc->main_gbuf, sqn); - if (tn && sn && tn->id != sn->id) { - cbm_gbuf_insert_edge(ws->local_edge_buf, sn->id, tn->id, "IMPLEMENTS", "{}"); - ws->semantic_resolved++; - } - } -} - -/* ── FastAPI Depends() tracking ──────────────────────────────────── */ -/* Scans Python function signatures for Depends(func_ref) and creates CALLS - * edges from the endpoint to the dependency function. Without this, FastAPI - * auth/DI functions appear as dead code (in_degree=0). */ - -enum { PP_SIG_SCAN_LINES = 15, PP_REGEX_GROUPS = 2 }; - -/* Python signature text of the def starting at start_line. Caller frees. */ -static char *extract_py_signature(const char *source, int start_line, int end_line) { - int sig_end = start_line + PP_SIG_SCAN_LINES; - if (end_line > 0 && sig_end > end_line) { - sig_end = end_line; - } - const char *p = source; - int line = SKIP_ONE; - while (*p && line < start_line) { - if (*p == '\n') { - line++; - } - p++; - } - const char *sig_start = p; - while (*p && line < sig_end) { - if (*p == '\n') { - line++; - } - p++; - if (p > sig_start + SKIP_ONE && p[-SKIP_ONE] == ':' && p[-PAIR_LEN] == ')') { - break; - } - } - size_t sig_len = (size_t)(p - sig_start); - char *sig = malloc(sig_len + SKIP_ONE); - if (!sig) { - return NULL; - } - memcpy(sig, sig_start, sig_len); - sig[sig_len] = '\0'; - return sig; -} - -/* CALLS edges for each Depends(func_ref) in one def's signature. */ -static void scan_depends_in_sig(resolve_ctx_t *rc, resolve_worker_state_t *ws, - const cbm_regex_t *re, const char *sig, - const cbm_gbuf_node_t *source_node, const char *module_qn, - const char **imp_keys, const char **imp_vals, int imp_count) { - cbm_regmatch_t match[PP_REGEX_GROUPS]; - const char *scan = sig; - while (cbm_regexec(re, scan, PP_REGEX_GROUPS, match, 0) == 0) { - int ref_len = match[SKIP_ONE].rm_eo - match[SKIP_ONE].rm_so; - char func_ref[CBM_SZ_256]; - if (ref_len >= (int)sizeof(func_ref)) { - ref_len = (int)sizeof(func_ref) - SKIP_ONE; - } - memcpy(func_ref, scan + match[SKIP_ONE].rm_so, (size_t)ref_len); - func_ref[ref_len] = '\0'; - cbm_resolution_t res = - cbm_registry_resolve(rc->registry, func_ref, module_qn, imp_keys, imp_vals, imp_count); - if (res.qualified_name && res.qualified_name[0] != '\0') { - const cbm_gbuf_node_t *tn = cbm_gbuf_find_by_qn(rc->main_gbuf, res.qualified_name); - if (tn && source_node->id != tn->id) { - cbm_gbuf_insert_edge(ws->local_edge_buf, source_node->id, tn->id, "CALLS", - "{\"confidence\":0.95,\"strategy\":\"fastapi_depends\"}"); - ws->calls_resolved++; - } - } - scan += match[0].rm_eo; - } -} - -static void resolve_file_fastapi_depends(resolve_ctx_t *rc, resolve_worker_state_t *ws, - const CBMFileResult *result, const char *path, - const char *module_qn, const char **imp_keys, - const char **imp_vals, int imp_count) { - bool has_depends = false; - for (int c = 0; c < result->calls.count && !has_depends; c++) { - const char *callee = result->calls.items[c].callee_name; - has_depends = callee && strcmp(callee, "Depends") == 0; - } - if (!has_depends) { - return; - } - cbm_regex_t depends_re; - if (cbm_regcomp(&depends_re, "Depends\\(([A-Za-z_][A-Za-z0-9_.]*)", CBM_REG_EXTENDED) != 0) { - return; - } - char *source_owned = NULL; - const char *source = result->source; - if (!source || result->source_len <= 0) { - int source_len = 0; - source_owned = path ? read_file(path, &source_len, NULL, NULL) : NULL; - source = source_owned; - } - for (int d = 0; source && d < result->defs.count; d++) { - const CBMDefinition *def = &result->defs.items[d]; - if (!def->qualified_name || def->start_line <= 0 || !def->label || - (strcmp(def->label, "Function") != 0 && strcmp(def->label, "Method") != 0)) { - continue; - } - const cbm_gbuf_node_t *source_node = - cbm_gbuf_find_by_qn(rc->main_gbuf, def->qualified_name); - if (!source_node) { - continue; - } - char *sig = extract_py_signature(source, (int)def->start_line, (int)def->end_line); - if (!sig) { - continue; - } - scan_depends_in_sig(rc, ws, &depends_re, sig, source_node, module_qn, imp_keys, imp_vals, - imp_count); - free(sig); - } - free_source(source_owned); - cbm_regfree(&depends_re); -} - -/* F4: get (or lazily build ONCE) the shared all_defs Rust registry. Called by the - * first worker that hits a NULL-filter rust file; later null-files reuse it. Fast - * path is a lock-free atomic load; the build happens under rust_shared_mu into the - * dedicated rust_shared_arena (never a shared pipeline arena from a worker thread). - * Returns NULL if there are no defs (caller falls back to the per-file build). */ -static CBMTypeRegistry *pp_rust_shared_registry(resolve_ctx_t *rc) { - CBMTypeRegistry *p = atomic_load_explicit(&rc->rust_shared_reg, memory_order_acquire); - if (p) - return p; - if (!rc->all_defs || rc->def_count <= 0) - return NULL; - cbm_mutex_lock(&rc->rust_shared_mu); - p = atomic_load_explicit(&rc->rust_shared_reg, memory_order_relaxed); - if (!p) { - cbm_arena_init(&rc->rust_shared_arena); - rc->rust_shared_arena_live = true; - p = cbm_rust_build_cross_registry(&rc->rust_shared_arena, rc->all_defs, rc->def_count); - if (p) { - char sb[96]; - snprintf(sb, sizeof(sb), "types=%d funcs=%d", p->type_count, p->func_count); - cbm_log_info("cross_lsp.rust_registry", "scale", sb); - } - atomic_store_explicit(&rc->rust_shared_reg, p, memory_order_release); - } - cbm_mutex_unlock(&rc->rust_shared_mu); - return p; -} - -/* void*-typed adapter so the shared dispatch helper (pass_lsp_cross.c) can - * borrow the lazily-built shared Rust registry without knowing resolve_ctx_t. */ -static CBMTypeRegistry *pp_rust_shared_registry_get(void *ctx) { - return pp_rust_shared_registry((resolve_ctx_t *)ctx); -} - -static void resolve_worker(int worker_id, void *ctx_ptr) { - resolve_ctx_t *rc = ctx_ptr; - resolve_worker_state_t *ws = &rc->workers[worker_id]; - - cbm_pxc_set_rust_manifest(rc->rust_manifest); - - if (!ws->local_edge_buf) { - ws->local_edge_buf = - cbm_gbuf_new_shared_ids(rc->project_name, rc->repo_path, rc->shared_ids); - } - - /* Per-worker service-pattern result cache. The same resolved QN - * (e.g. "fmt.Errorf", "context.Context.Done") appears in many - * call edges across many files within a project — caching turns - * cbm_service_pattern_match's 6 × 30 × strstr scan into one hash - * lookup after the first miss for each QN. Scoped to the worker's - * lifetime in the parallel_resolve phase. */ - cbm_service_pattern_cache_begin(); - - /* A result loaded from the spill store lives for one iteration; it is - * released at the top of the next one (every `continue` passes there) - * and after the loop. */ - CBMFileResult *loaded = NULL; - int loaded_idx = -1; - while (SKIP_ONE) { - cbm_result_cache_release(rc->result_cache, loaded_idx, loaded); - loaded = NULL; - loaded_idx = -1; - int file_idx = - atomic_fetch_add_explicit(&rc->next_file_idx, SKIP_ONE, memory_order_relaxed); - if (file_idx >= rc->file_count) { - break; - } - if (atomic_load_explicit(rc->cancelled, memory_order_relaxed)) { - break; - } - - uint64_t _loop_t0 = extract_now_ns(); - - CBMFileResult *result = cbm_result_cache_acquire(rc->result_cache, rc->spill, file_idx); - if (result && result != rc->result_cache[file_idx]) { - loaded = result; - loaded_idx = file_idx; - /* cbm_pxc_collect_all_defs stored the inferred JVM namespace on - * the in-memory result; a loaded copy needs it again. */ - if (rc->all_defs) { - cbm_pxc_apply_inferred_namespace(result, &rc->files[file_idx]); - } - } - if (!result) { - atomic_fetch_add_explicit(&rc->time_ns_total_loop, extract_now_ns() - _loop_t0, - memory_order_relaxed); - continue; - } - atomic_fetch_add_explicit(&rc->total_files_visited, 1, memory_order_relaxed); - - CBMLanguage lang = rc->files[file_idx].language; - const char *rel = rc->files[file_idx].rel_path; - - /* Skip cross-LSP for machine-generated files — they're huge (10k- - * 70k lines for k8s protobuf/openapi), have low semantic value for - * graph navigation (boilerplate getters/setters/marshal), and - * dominate the cross-LSP wall time when they have even one - * unresolved call (tree-sitter parse on a 70k-line file is ~1-2s). - * The per-file LSP during extract still indexes their defs/calls - * normally — only the cross-file resolution refinement is skipped. */ - bool is_generated = false; - if (rel) { - is_generated = - (strstr(rel, ".pb.go") != NULL) || (strstr(rel, "zz_generated") != NULL) || - (strstr(rel, "_generated.go") != NULL) || (strstr(rel, ".gen.go") != NULL) || - (strstr(rel, "/applyconfigurations/") != NULL) || - (strstr(rel, "_pb2.py") != NULL) || (strstr(rel, "_pb2_grpc.py") != NULL) || - (strstr(rel, ".pb.cc") != NULL) || (strstr(rel, ".pb.h") != NULL) || - (strstr(rel, ".pb-c.c") != NULL) || (strstr(rel, ".pb-c.h") != NULL); - } - - /* Cross-file LSP is a per-file tree-sitter re-parse + AST walk + - * registry lookups — ~50-150ms per file. It can ONLY find calls - * that exist in the AST. If the per-file extract found zero calls, - * cross-LSP will too: the AST is the same. For non-JVM languages, - * skip when per-file LSP already produced at least as many resolved - * entries as textual calls. Java/Kotlin per-file LSP can fill the - * count with constructors or same-file calls while a mixed-source-root - * Java↔Kotlin call remains unresolved, so JVM callers run whenever - * calls exist. Callable-reference candidates (`accept(handler)` with - * an imported `handler`) count the same way against reference rows. */ - bool jvm_cross_lsp = (lang == CBM_LANG_JAVA || lang == CBM_LANG_KOTLIN); - bool rust_workspace_cross_lsp = - (lang == CBM_LANG_RUST && rc->rust_manifest && rc->rust_manifest->member_count > 0); - int invocation_rows = cbm_pipeline_invocation_row_count(&result->resolved_calls); - int ref_candidates = 0; - for (int u = 0; u < result->usages.count; u++) { - ref_candidates += result->usages.items[u].may_be_call_reference ? 1 : 0; - } - bool cross_lsp_eligible = - (rc->all_defs && rc->def_count > 0 && cbm_pxc_has_cross_lsp(lang) && - (result->calls.count > 0 || ref_candidates > 0) && - (jvm_cross_lsp || rust_workspace_cross_lsp || invocation_rows < result->calls.count || - cbm_pipeline_reference_row_count(&result->resolved_calls) < ref_candidates || - (lang == CBM_LANG_ZIG && rc->def_modules && - cbm_pxc_zig_args_in_module(rc->cross_registries, rc->def_modules[file_idx]))) && - !is_generated); - - /* Skip files with nothing else to resolve and no cross-LSP work. */ - if (result->calls.count == 0 && result->usages.count == 0 && result->throws.count == 0 && - result->rw.count == 0 && result->defs.count == 0 && result->impl_traits.count == 0 && - !cross_lsp_eligible) { - continue; - } - - /* Build import map ONCE (read-only access to main_gbuf). The - * same imp_keys/imp_vals feed both the fused cross-file LSP - * step below AND the resolve_file_* chain — no duplicate build. */ - const char **imp_keys = NULL; - const char **imp_vals = NULL; - int imp_count = 0; - uint64_t _imp_t0 = extract_now_ns(); - build_import_map(rc->main_gbuf, rc->project_name, rel, &imp_keys, &imp_vals, &imp_count); - atomic_fetch_add_explicit(&rc->time_ns_import_map, extract_now_ns() - _imp_t0, - memory_order_relaxed); - - /* Per-file is_import_reachable memoization. Spans all 5 resolve - * sub-passes (calls/usages/throws/rw/semantic) which all flow - * through cbm_registry_resolve. Same callee_name appears in - * many call sites — first eval pays the strstr cost, repeats - * are O(1) hash. Imports are constant within a file so the - * cache is sound; invalidated at file exit. */ - cbm_registry_reach_cache_begin(result->calls.count + result->usages.count + 64); - - /* Per-file import-map prefix → module-qn hash. resolve_import_map - * was doing O(imports) linear strcmp per call; with this it - * becomes O(1). Keys/values borrowed from imp_keys/imp_vals - * which outlive this scope. */ - cbm_registry_import_map_cache_begin(imp_keys, imp_vals, imp_count); - - /* THE BIG ONE: per-file cache of cbm_registry_resolve results. - * Same callee_name in multiple call sites resolves identically - * within a file (module_qn is fixed) — first call walks the - * strategy chain, repeats are O(1). On K8s this targets the - * 98.7% hot spot in resolve_file_calls (881 of 893s CPU). */ - cbm_registry_resolve_cache_begin(result->calls.count + result->usages.count + 64); - - char *module_qn = - cbm_pipeline_fqn_module_dir(rc->project_name, rel, pp_module_is_dir(lang)); - - /* ── Cross-file LSP (FUSED) ───────────────────────────── - * Runs BEFORE resolve_file_calls so its additions to - * result->resolved_calls are picked up by - * cbm_pipeline_find_lsp_resolution when calls become CALLS - * edges. Prefers source bytes retained in result->arena during - * extract; when the low-RAM retention cap dropped this file - * (result->source==NULL) it FALLS BACK to a bounded per-file - * read from disk, freed immediately after the LSP call. This is - * the correctness guarantee: lowering the retention cap trades - * retained RAM for a bounded re-read, it NEVER drops a cross-file - * edge. Only a genuine read failure (deleted/unreadable/oversized) - * leaves source NULL and is counted as skipped_no_source; defs/calls - * already in the extract are unaffected either way. - * - * Slab reclaim afterward: the LSP re-parses via tree-sitter, - * which allocates through this worker's TLS slab. Reclaiming - * here keeps the slab high-water bounded as the resolve phase - * walks across thousands of files in a single worker thread. */ - if (cross_lsp_eligible) { - char *lsp_source_owned = NULL; - const char *lsp_source = result->source; - int lsp_source_len = result->source_len; - if ((!lsp_source || lsp_source_len <= 0) && rc->files[file_idx].path) { - /* Retention cap skipped this file — re-read on demand (bounded - * by read_file's cbm_max_file_bytes cap), freed below. */ - lsp_source_owned = read_file(rc->files[file_idx].path, &lsp_source_len, NULL, NULL); - lsp_source = lsp_source_owned; - } - if (lsp_source && lsp_source_len > 0) { - const char *def_module = rc->def_modules ? rc->def_modules[file_idx] : module_qn; - - uint64_t lsp_t0 = extract_now_ns(); - - /* Shared per-file dispatch (pass_lsp_cross.c): module-def - * filter → shared prebuilt registry (overlay pattern) → - * filtered per-file fallback. Journal the - * file around the resolve so a hang HERE is attributed to - * this file, not to a stale extraction marker. */ - cbm_index_mark_start(rel); - bool had_error = result->has_error; - cbm_pxc_dispatch_file(lang, result, lsp_source, lsp_source_len, rel, def_module, - rc->cross_registries, rc->module_def_index, rc->all_defs, - rc->def_count, imp_keys, imp_vals, imp_count, - pp_rust_shared_registry_get, rc); - cbm_index_mark_done(rel); - if (!had_error && result->has_error) { - rc->lsp_failures[file_idx] = result->error_msg; - } - /* Free the on-demand re-read (no-op when source was retained). */ - free_source(lsp_source_owned); - /* Contract: cbm_slab_reclaim() requires the thread parser to be - * destroyed first; otherwise its lexer holds slab pointers - * (lexer.included_ranges) that get freed underneath it, causing - * a heap-use-after-free on the next ts_lexer_goto. The next - * cbm_extract_file on this thread will recreate the parser. */ - cbm_destroy_thread_parser(); - cbm_slab_reclaim(); - uint64_t lsp_elapsed_ns = extract_now_ns() - lsp_t0; - atomic_fetch_add_explicit(&rc->time_ns_cross_lsp, lsp_elapsed_ns, - memory_order_relaxed); - uint64_t lsp_elapsed_ms = lsp_elapsed_ns / PP_USEC_PER_MS; - if (lsp_elapsed_ms > PP_TIMER_THRESH) { - cbm_log_info("parallel.resolve.lsp_cross.slow", "elapsed_ms", - cbm_log_itoa((int)lsp_elapsed_ms), "path", rel); - } - atomic_fetch_add_explicit(&rc->lsp_cross_processed, SKIP_ONE, memory_order_relaxed); - } else { - /* Source unavailable even after the re-read fallback (file - * deleted / unreadable / oversized) → the cross-file LSP - * refinement no-ops for this file. This is a bounded skip, NOT - * a file failure: defs/calls were already extracted and are - * unaffected. Deliberately NOT recorded as a cbm_file_error — - * doing so would flood skipped[] with false positives (itself a - * false-guard bug). The "cross_lsp" phase string is reserved for - * Track C's real crash-attribution signal; leave it unwired. */ - atomic_fetch_add_explicit(&rc->lsp_cross_skipped_no_source, SKIP_ONE, - memory_order_relaxed); - } - } - - /* Per-sub-phase wall-clock so we can attribute the dominant cost. */ - uint64_t _ph_t0; - - /* ── CALLS resolution ──────────────────────────────────── */ - _ph_t0 = extract_now_ns(); - resolve_file_calls(rc, ws, result, rel, module_qn, imp_keys, imp_vals, imp_count, lang); - if (lang == CBM_LANG_PYTHON) { - resolve_file_fastapi_depends(rc, ws, result, rc->files[file_idx].path, module_qn, - imp_keys, imp_vals, imp_count); - } - atomic_fetch_add_explicit(&rc->time_ns_calls, extract_now_ns() - _ph_t0, - memory_order_relaxed); - - /* ── USAGE resolution ──────────────────────────────────── */ - _ph_t0 = extract_now_ns(); - resolve_file_usages(rc, ws, result, rel, module_qn, imp_keys, imp_vals, imp_count, - lang); - atomic_fetch_add_explicit(&rc->time_ns_usages, extract_now_ns() - _ph_t0, - memory_order_relaxed); - - /* ── THROWS / RAISES ───────────────────────────────────── */ - _ph_t0 = extract_now_ns(); - resolve_file_throws(rc, ws, result, module_qn, imp_keys, imp_vals, imp_count); - atomic_fetch_add_explicit(&rc->time_ns_throws, extract_now_ns() - _ph_t0, - memory_order_relaxed); - - /* ── READS / WRITES ────────────────────────────────────── */ - _ph_t0 = extract_now_ns(); - resolve_file_rw(rc, ws, result, rel, module_qn, imp_keys, imp_vals, imp_count, lang); - atomic_fetch_add_explicit(&rc->time_ns_rw, extract_now_ns() - _ph_t0, memory_order_relaxed); - - /* ── INHERITS + DECORATES + IMPLEMENTS ──────────────────── */ - _ph_t0 = extract_now_ns(); - resolve_file_semantic(rc, ws, result, module_qn, imp_keys, imp_vals, imp_count); - atomic_fetch_add_explicit(&rc->time_ns_semantic, extract_now_ns() - _ph_t0, - memory_order_relaxed); - - cbm_registry_reach_cache_end(); - cbm_registry_import_map_cache_end(); - cbm_registry_resolve_cache_end(); - - free(module_qn); - free_import_map(imp_keys, imp_vals, imp_count); - - atomic_fetch_add_explicit(&rc->time_ns_total_loop, extract_now_ns() - _loop_t0, - memory_order_relaxed); - } - cbm_result_cache_release(rc->result_cache, loaded_idx, loaded); - - /* Tear down this worker's thread-local parser + slab state, mirroring - * extract_worker. Without this, resolve-worker pages keep an owner pointer - * into dead TLS and are never retired, so a later cross-thread free can - * never bring their refcount to zero (leak). Retiring them here releases - * each page as its final chunk returns. */ - cbm_pxc_set_rust_manifest(NULL); - cbm_destroy_thread_parser(); - cbm_slab_destroy_thread(); - cbm_service_pattern_cache_end(); -} - -int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count, - CBMFileResult **result_cache, _Atomic int64_t *shared_ids, - int worker_count, CBMLSPDef *all_defs, int def_count, - char *const *def_modules, struct CBMModuleDefIndex *module_def_index, - void *cross_registries_v) { - /* See header: typed as void* across the TU boundary; cast back here. */ - CBMCrossLspRegistries *cross_registries = (CBMCrossLspRegistries *)cross_registries_v; - if (file_count == 0) { - return 0; - } - - cbm_log_info("parallel.resolve.start", "files", cbm_log_itoa(file_count), "workers", - cbm_log_itoa(worker_count)); - - resolve_worker_state_t *workers = NULL; - if (cbm_aligned_alloc((void **)&workers, CBM_CACHE_LINE, - (size_t)worker_count * sizeof(resolve_worker_state_t)) != 0) { - return CBM_NOT_FOUND; - } - memset(workers, 0, (size_t)worker_count * sizeof(resolve_worker_state_t)); - const char **lsp_failures = calloc((size_t)file_count, sizeof(*lsp_failures)); - if (!lsp_failures) { - cbm_aligned_free(workers); - return CBM_NOT_FOUND; - } - - bool have_rust = false; - for (int i = 0; i < file_count; i++) { - if (files[i].language == CBM_LANG_RUST && - cbm_result_cache_present(result_cache, ctx->spill, i)) { - have_rust = true; - break; - } - } - CBMArena rust_manifest_arena; - CBMCargoManifest rust_manifest; - bool rust_manifest_arena_live = false; - const CBMCargoManifest *rust_manifest_ptr = NULL; - if (have_rust) { - cbm_arena_init(&rust_manifest_arena); - rust_manifest_arena_live = true; - if (cbm_pxc_build_rust_manifest(ctx, &rust_manifest_arena, &rust_manifest)) { - rust_manifest_ptr = &rust_manifest; - } - } - - resolve_ctx_t rc = { - .files = files, - .file_count = file_count, - .project_name = ctx->project_name, - .repo_path = ctx->repo_path, - .workers = workers, - .max_workers = worker_count, - .result_cache = result_cache, - .spill = ctx->spill, - .main_gbuf = ctx->gbuf, - .registry = ctx->registry, - .shared_ids = shared_ids, - .cancelled = ctx->cancelled, - .all_defs = all_defs, - .def_count = def_count, - .def_modules = def_modules, - .module_def_index = module_def_index, - .cross_registries = cross_registries, - .rust_manifest = rust_manifest_ptr, - .lsp_failures = lsp_failures, - }; - atomic_init(&rc.next_file_idx, 0); - atomic_init(&rc.lsp_cross_processed, 0); - atomic_init(&rc.lsp_cross_skipped_no_source, 0); - atomic_store_explicit(&g_lsp_tail_lookups, 0, memory_order_relaxed); - atomic_store_explicit(&g_lsp_tail_candidates, 0, memory_order_relaxed); - cbm_pxc_filter_stats_reset(); - /* F4 lazy shared Rust registry: mutex up before workers spawn. */ - atomic_init(&rc.rust_shared_reg, NULL); - cbm_mutex_init(&rc.rust_shared_mu); - rc.rust_shared_arena_live = false; - - /* Sub-phase: Dispatch resolve workers (per-file call/usage resolution, PARALLEL) */ - CBM_PROF_START(t_resolve_dispatch); - cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false}; - cbm_parallel_for(worker_count, resolve_worker, &rc, opts); - CBM_PROF_END_N("parallel_resolve", "1_dispatch_workers_parallel", t_resolve_dispatch, - file_count); - for (int i = 0; i < file_count; i++) { - if (lsp_failures[i] && ctx->pipeline) { - cbm_pipeline_add_file_error(ctx->pipeline, files[i].rel_path, lsp_failures[i], - "lsp_incomplete"); - } - } - free(lsp_failures); - /* Workers joined: the shared Rust registry (if built) is no longer read. - * Free its dedicated arena + the lock (registry was self-contained: it strdup'd - * all QNs, so freeing all_defs afterward is safe). */ - if (rc.rust_shared_arena_live) { - cbm_arena_destroy(&rc.rust_shared_arena); - rc.rust_shared_arena_live = false; - } - cbm_mutex_destroy(&rc.rust_shared_mu); - if (rust_manifest_arena_live) { - cbm_arena_destroy(&rust_manifest_arena); - } - - /* Sub-phase: Merge all local edge bufs into main gbuf (SEQUENTIAL) */ - CBM_PROF_START(t_resolve_merge); - int total_calls = 0; - int total_usages = 0; - int total_semantic = 0; - int total_lsp_overrides = 0; - for (int i = 0; i < worker_count; i++) { - if (workers[i].local_edge_buf) { - cbm_gbuf_merge(ctx->gbuf, workers[i].local_edge_buf); - total_calls += workers[i].calls_resolved; - total_usages += workers[i].usages_resolved; - total_semantic += workers[i].semantic_resolved; - total_lsp_overrides += workers[i].lsp_overrides; - cbm_gbuf_free(workers[i].local_edge_buf); - } - } - CBM_PROF_END_N("parallel_resolve", "2_merge_edge_bufs_seq", t_resolve_merge, - total_calls + total_usages); - - cbm_aligned_free(workers); - - /* Go-style implicit interface satisfaction (needs full graph, serial) */ - int go_impl = cbm_pipeline_implements_go(ctx); - - if (atomic_load(ctx->cancelled)) { - return CBM_NOT_FOUND; - } - - /* Summary metric that replaces the removed `pass.timing pass=lsp_cross` - * log line — surfaces how many files the fused cross-file LSP step - * actually processed vs skipped (e.g. because their source bytes - * were not retained at extract time due to the per-file/total cap). */ - cbm_log_info( - "parallel.resolve.lsp_cross_done", "files_processed", - cbm_log_itoa(atomic_load_explicit(&rc.lsp_cross_processed, memory_order_relaxed)), - "files_skipped_no_source", - cbm_log_itoa(atomic_load_explicit(&rc.lsp_cross_skipped_no_source, memory_order_relaxed)), - "defs_total", cbm_log_itoa(def_count)); - - /* Cross-LSP cost normalised per file (upstream f6e3af43/cffb4e37). Wall - * time cannot tell "big repo" from "superlinear pass"; us_per_file can: - * for a pass linear in file count it stays roughly constant across repo - * sizes. defs_per_file is the lever behind it: if it tracks defs_total - * rather than the file's own module plus imports, the module filter is - * not containing the work and cross-file LSP is O(files x corpus_defs). */ - int cross_files = atomic_load_explicit(&rc.lsp_cross_processed, memory_order_relaxed); - uint64_t cross_us = - atomic_load_explicit(&rc.time_ns_cross_lsp, memory_order_relaxed) / 1000ULL; - if (cross_files > 0) { - char cf_buf[CBM_SZ_32]; - char nf_buf[CBM_SZ_32]; - char cu_buf[CBM_SZ_32]; - char pk_buf[CBM_SZ_32]; - snprintf(cf_buf, sizeof(cf_buf), "%llu", - (unsigned long long)(cross_us / (uint64_t)cross_files)); - snprintf(nf_buf, sizeof(nf_buf), "%d", cross_files); - snprintf(cu_buf, sizeof(cu_buf), "%llu", (unsigned long long)(cross_us / 1000ULL)); - snprintf(pk_buf, sizeof(pk_buf), "%llu", - (unsigned long long)(def_count > 0 ? (cross_us * 1000ULL) / - ((uint64_t)cross_files * - (uint64_t)def_count) - : 0ULL)); - cbm_log_info("parallel.resolve.cross_lsp_cost", "cross_lsp_ms", cu_buf, "files", nf_buf, - "us_per_file", cf_buf, "defs_total", cbm_log_itoa(def_count), - "us_per_file_per_kdef", pk_buf); - - uint64_t reg_defs = 0; - uint64_t reg_files = 0; - uint64_t flt_files = 0; - uint64_t flt_failed = 0; - cbm_pxc_filter_stats(®_defs, ®_files, &flt_files, &flt_failed); - if (reg_files > 0) { - char rd_buf[CBM_SZ_32]; - char ff_buf[CBM_SZ_32]; - char fp_buf[CBM_SZ_32]; - snprintf(rd_buf, sizeof(rd_buf), "%llu", (unsigned long long)(reg_defs / reg_files)); - snprintf(ff_buf, sizeof(ff_buf), "%llu", (unsigned long long)flt_failed); - snprintf(fp_buf, sizeof(fp_buf), "%llu", (unsigned long long)flt_files); - cbm_log_info("parallel.resolve.perfile_registry", "defs_per_file", rd_buf, "defs_total", - cbm_log_itoa(def_count), "filtered_files", fp_buf, "filter_failed", - ff_buf); - } - } - - cbm_log_info("parallel.resolve.done", "calls", cbm_log_itoa(total_calls), "usages", - cbm_log_itoa(total_usages), "semantic", cbm_log_itoa(total_semantic + go_impl), - "lsp_overrides", cbm_log_itoa(total_lsp_overrides)); - - /* Per-sub-phase breakdown so we stop guessing about hot paths. - * Numbers are summed across workers (total CPU-ms, not wall-time). */ - char loop_buf[32], visits_buf[32]; - snprintf( - loop_buf, sizeof(loop_buf), "%llu", - (unsigned long long)(atomic_load_explicit(&rc.time_ns_total_loop, memory_order_relaxed) / - 1000000ULL)); - snprintf(visits_buf, sizeof(visits_buf), "%d", - atomic_load_explicit(&rc.total_files_visited, memory_order_relaxed)); - cbm_log_info("parallel.resolve.phase_summary", "total_loop_cpu_ms", loop_buf, "files_visited", - visits_buf); - - char imp_buf[32], xls_buf[32], cal_buf[32]; - snprintf( - imp_buf, sizeof(imp_buf), "%llu", - (unsigned long long)(atomic_load_explicit(&rc.time_ns_import_map, memory_order_relaxed) / - 1000000ULL)); - snprintf( - xls_buf, sizeof(xls_buf), "%llu", - (unsigned long long)(atomic_load_explicit(&rc.time_ns_cross_lsp, memory_order_relaxed) / - 1000000ULL)); - snprintf(cal_buf, sizeof(cal_buf), "%llu", - (unsigned long long)(atomic_load_explicit(&rc.time_ns_calls, memory_order_relaxed) / - 1000000ULL)); - cbm_log_info("parallel.resolve.phase_ms_a", "import_map", imp_buf, "cross_lsp", xls_buf, - "resolve_calls", cal_buf); - - char use_buf[32], thr_buf[32], rw_buf[32], sem_buf[32]; - snprintf(use_buf, sizeof(use_buf), "%llu", - (unsigned long long)(atomic_load_explicit(&rc.time_ns_usages, memory_order_relaxed) / - 1000000ULL)); - snprintf(thr_buf, sizeof(thr_buf), "%llu", - (unsigned long long)(atomic_load_explicit(&rc.time_ns_throws, memory_order_relaxed) / - 1000000ULL)); - snprintf(rw_buf, sizeof(rw_buf), "%llu", - (unsigned long long)(atomic_load_explicit(&rc.time_ns_rw, memory_order_relaxed) / - 1000000ULL)); - snprintf(sem_buf, sizeof(sem_buf), "%llu", - (unsigned long long)(atomic_load_explicit(&rc.time_ns_semantic, memory_order_relaxed) / - 1000000ULL)); - cbm_log_info("parallel.resolve.phase_ms_b", "resolve_usages", use_buf, "resolve_throws", - thr_buf, "resolve_rw", rw_buf, "resolve_semantic", sem_buf); - - char src_buf[32], lsp_buf[32], rsv_buf[32], hnt_buf[32], tgt_buf[32], emt_buf[32]; - snprintf( - src_buf, sizeof(src_buf), "%llu", - (unsigned long long)(atomic_load_explicit(&rc.time_ns_rc_source, memory_order_relaxed) / - 1000000ULL)); - snprintf( - lsp_buf, sizeof(lsp_buf), "%llu", - (unsigned long long)(atomic_load_explicit(&rc.time_ns_rc_lsp_lookup, memory_order_relaxed) / - 1000000ULL)); - snprintf( - rsv_buf, sizeof(rsv_buf), "%llu", - (unsigned long long)(atomic_load_explicit(&rc.time_ns_rc_resolve, memory_order_relaxed) / - 1000000ULL)); - snprintf(hnt_buf, sizeof(hnt_buf), "%llu", - (unsigned long long)(atomic_load_explicit(&rc.time_ns_rc_hint, memory_order_relaxed) / - 1000000ULL)); - snprintf( - tgt_buf, sizeof(tgt_buf), "%llu", - (unsigned long long)(atomic_load_explicit(&rc.time_ns_rc_target, memory_order_relaxed) / - 1000000ULL)); - snprintf(emt_buf, sizeof(emt_buf), "%llu", - (unsigned long long)(atomic_load_explicit(&rc.time_ns_rc_emit, memory_order_relaxed) / - 1000000ULL)); - cbm_log_info("parallel.resolve.calls_breakdown", "find_source", src_buf, "lsp_lookup", lsp_buf, - "resolve", rsv_buf); - cbm_log_info("parallel.resolve.calls_breakdown2", "field_hint", hnt_buf, "find_target", tgt_buf, - "emit_edge", emt_buf); - - /* Candidate-scan cost (upstream f6e3af43). per_lookup is a property of - * the corpus, not the machine; if it grows with repo size the tail-match - * scan is turning resolve superlinear. */ - uint64_t tail_lookups = atomic_load_explicit(&g_lsp_tail_lookups, memory_order_relaxed); - uint64_t tail_cands = atomic_load_explicit(&g_lsp_tail_candidates, memory_order_relaxed); - char tl_buf[CBM_SZ_32]; - char tc_buf[CBM_SZ_32]; - char tp_buf[CBM_SZ_32]; - snprintf(tl_buf, sizeof(tl_buf), "%llu", (unsigned long long)tail_lookups); - snprintf(tc_buf, sizeof(tc_buf), "%llu", (unsigned long long)tail_cands); - snprintf(tp_buf, sizeof(tp_buf), "%llu", - (unsigned long long)(tail_lookups ? tail_cands / tail_lookups : 0ULL)); - cbm_log_info("parallel.resolve.scan_cost", "tail_lookups", tl_buf, "tail_candidates", tc_buf, - "per_lookup", tp_buf); - return 0; -} - -void cbm_parallel_resolve_calls(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, - int file_count) { - if (!ctx || !files || file_count <= 0) { - return; - } - _Atomic int64_t shared_ids; - atomic_init(&shared_ids, cbm_gbuf_next_id(ctx->gbuf)); - resolve_ctx_t rc = { - .files = files, - .file_count = file_count, - .project_name = ctx->project_name, - .repo_path = ctx->repo_path, - .main_gbuf = ctx->gbuf, - .registry = ctx->registry, - .shared_ids = &shared_ids, - .cancelled = ctx->cancelled, - }; - resolve_worker_state_t ws = { - .local_edge_buf = cbm_gbuf_new_shared_ids(ctx->project_name, ctx->repo_path, &shared_ids), - }; - if (!ws.local_edge_buf) { - return; - } - for (int i = 0; i < file_count && !cbm_pipeline_check_cancel(ctx); i++) { - const cbm_file_info_t *fi = &files[i]; - int source_len = 0; - char *source = read_file(fi->path, &source_len, NULL, NULL); - if (!source) { - continue; - } - CBMFileResult *result = - cbm_extract_file(source, source_len, fi->language, ctx->project_name, fi->rel_path, - CBM_EXTRACT_BUDGET, NULL, NULL); - free_source(source); - if (!result) { - continue; - } - if (result->calls.count > 0) { - const char **imp_keys = NULL; - const char **imp_vals = NULL; - int imp_count = 0; - build_import_map(ctx->gbuf, ctx->project_name, fi->rel_path, &imp_keys, &imp_vals, - &imp_count); - char *module_qn = cbm_pipeline_fqn_module_dir(ctx->project_name, fi->rel_path, - pp_module_is_dir(fi->language)); - resolve_file_calls(&rc, &ws, result, fi->rel_path, module_qn, imp_keys, imp_vals, - imp_count, fi->language); - if (fi->language == CBM_LANG_PYTHON) { - resolve_file_fastapi_depends(&rc, &ws, result, fi->path, module_qn, imp_keys, - imp_vals, imp_count); - } - free(module_qn); - free_import_map(imp_keys, imp_vals, imp_count); - } - cbm_free_result(result); - } - cbm_gbuf_merge(ctx->gbuf, ws.local_edge_buf); - cbm_gbuf_free(ws.local_edge_buf); - cbm_pipeline_sync_next_id(ctx->gbuf, &shared_ids); - cbm_log_info("parallel.resolve_calls.done", "files", cbm_log_itoa(file_count), "calls", - cbm_log_itoa(ws.calls_resolved)); -} diff --git a/src/pipeline/pass_parallel.zig b/src/pipeline/pass_parallel.zig new file mode 100644 index 00000000..87f7e931 --- /dev/null +++ b/src/pipeline/pass_parallel.zig @@ -0,0 +1,3662 @@ +//! pass_parallel.zig — Three-phase parallel pipeline. +//! +//! Phase 3A: Parallel extract + create definition nodes (per-worker gbufs) +//! Phase 3B: Serial registry build + edge creation from cached results +//! Phase 4: Parallel call/usage/semantic resolution (per-worker edge bufs) +//! +//! Each file is read and parsed ONCE (Phase 3A). The CBMFileResult is cached +//! and reused for resolution (Phase 4), eliminating 3x redundant I/O + parsing. +//! +//! Depends on: worker_pool, graph_buffer (shared IDs + merge), extraction (cbm.h) + +const std = @import("std"); +const c = @import("c"); +const interop = @import("../foundation/interop.zig"); +const util = @import("util.zig"); + +const CStr = util.CStr; +const MutStr = util.MutStr; +const ix = util.ix; +const streq = util.streq; +const ptrDiff = util.ptrDiff; +const HashTable = c.CBMHashTable; +const Arena = c.CBMArena; +const Gbuf = c.cbm_gbuf_t; +const Node = c.cbm_gbuf_node_t; +const Edge = c.cbm_gbuf_edge_t; +const Ctx = c.cbm_pipeline_ctx_t; +const FileInfo = c.cbm_file_info_t; +const FileError = c.cbm_file_error_t; +const FileResult = c.CBMFileResult; +const Definition = c.CBMDefinition; +const LspDef = c.CBMLSPDef; +const Registry = c.cbm_registry_t; +const Language = c.CBMLanguage; +const Spill = c.cbm_result_spill_t; +const Timespec = c.struct_timespec; +/// A NULL-terminated or counted array of borrowed C strings. +const StrList = [*c]CStr; + +/// int fields of ResolveWorkerState +const PP_RESOLVE_COUNTERS = 5; +const PP_JSON_MARGIN = 10; +const PP_ESC_MARGIN = 3; +const PP_ESC_SPACE = 2; +/// Fixed bytes around a serialized JSON field: ,"key":"value" / ,"key":[...] +/// -> comma + 2 key quotes + colon + 2 value quotes (resp. brackets). +const PP_JSON_FIELD_OVERHEAD = 6; +const PP_ARGS_MARGIN = 20; +/// ,"line": -> comma + key (7) + colon + up to 10 digits + NUL. +const PP_LINE_MARGIN = 24; +const PP_LOG_THRESH = 24; +const PP_LOG_INTERVAL = 10; +const PP_TIMER_THRESH = 1000; +/// Extraction memory back-pressure: when the process is over its RSS budget, +/// a worker reclaims + naps before pulling another file so peers can finish +/// and return pages. Bounded spins avoid deadlock when the resident graph +/// itself is near budget: then ONE confirmation cycle decides — drained +/// means continue, still over means the attempt fails whole (upstream +/// decision A, #1997 #832) instead of the former open-ended soft overshoot. +const PP_BACKPRESSURE_MAX_SPINS = 40; +const PP_BACKPRESSURE_NAP_NS = 3000000; // 3 ms +const PP_NSEC_PER_SEC: u64 = 1000000000; +const PP_USEC_PER_MS: u64 = 1000000; +const PP_HALF_CONF = 0.5; +const PP_FIELD_HINT_CONF = 0.85; +const PP_CSHARP_M_PREFIX_LEN = 2; + +// Absolute source-retention ceilings for the parallel extract pipeline. +// +// The extract worker copies each file's source bytes into result->arena so +// the fused cross-file LSP step in resolve_worker can re-parse without +// re-opening the file. That retention is TRANSIENT (freed at run end) but it +// is a PEAK-RSS driver: every retained byte is resident at once across the +// extract→resolve handoff. +// +// A cap here is a FLOOR as much as a ceiling: whatever total we allow, we +// WILL hold that much resident at peak on a large repo. rust-analyzer bounds +// retained file *text* to a small fixed budget and re-reads source on a miss +// rather than scaling the retained set with host RAM — because the re-read is +// cheap relative to holding tens of GB resident. We follow the same model: +// - derive the total budget from the process memory budget (budget/8), +// - BUT clamp the RAM-derived DEFAULT to a small absolute ceiling (1 GiB) +// so a 512 GiB host does not retain 64 GiB of source it would re-read +// cheaply anyway, and keep the per-file cap modest (32 MiB) so one +// pathological generated blob cannot monopolise the budget. +// A file dropped from retention is NOT lost to cross-file resolution: +// resolve_worker re-reads it on demand, bounded and freed immediately +// (source_reread fallback). Both caps are env-overridable — CBM_RETAIN_TOTAL_MB +// / CBM_RETAIN_PER_FILE_MB raise or lower the auto-derived defaults directly +// (the hard ceilings bound only the RAM-derived default, never a deliberate +// operator/caller choice). +const PP_RETAIN_TOTAL_HARD_MAX_BYTES: usize = 1024 * 1024 * 1024; // 1 GiB default ceiling +const PP_RETAIN_PER_FILE_HARD_MAX_BYTES: usize = 32 * 1024 * 1024; // 32 MiB per file + +/// Back-pressure nap-cycle counter (test observability): each execution of the +/// over-budget collect+nap gate counts one cycle. Lets tests assert the gate does +/// not re-pay the full nap tax on every file pull when napping cannot reclaim +/// memory (the resident floor, not in-flight transients, holds the budget). +var g_bp_nap_cycles: std.atomic.Value(c_long) = .init(0); +/// Defined here, declared in lsp_resolve.h: the uncapped tail-match scan's +/// cost, logged at the end of parallel resolve. +pub export var g_lsp_tail_lookups: u64 = 0; +pub export var g_lsp_tail_candidates: u64 = 0; + +pub export fn cbm_pp_bp_nap_cycles() c_long { + return g_bp_nap_cycles.load(.monotonic); +} + +pub export fn cbm_pp_bp_nap_cycles_reset() void { + g_bp_nap_cycles.store(0, .monotonic); +} + +/// Test seam: when set, a parallel extract with a spill store spills from its +/// first file instead of waiting for the budget signal. +var g_force_spill: std.atomic.Value(c_int) = .init(0); +/// Results parked by the parallel extract (test observability). +var g_spill_parked: std.atomic.Value(c_long) = .init(0); + +pub export fn cbm_parallel_force_spill_for_test(on: c_int) void { + g_force_spill.store(@intFromBool(on != 0), .monotonic); +} + +pub export fn cbm_pp_spill_parked() c_long { + return g_spill_parked.load(.monotonic); +} + +pub export fn cbm_pp_spill_parked_reset() void { + g_spill_parked.store(0, .monotonic); +} + +fn logError(msg: interop.Str, kvs: anytype) void { + interop.log(c.CBM_LOG_ERROR, msg, kvs); +} + +fn itoa(v: c_int) CStr { + return c.cbm_log_itoa(v); +} + +/// "true" or "false", as the properties JSON spells a bool. +fn boolText(b: bool) CStr { + return if (b) "true" else "false"; +} + +/// Parse a positive MB-valued retention env knob (CBM_RETAIN_*_MB) into bytes. +/// Follows the limits.c strtol convention: unset / unparseable / non-positive +/// → return 0 so the caller keeps its derived default. +fn cbmRetainEnvBytes(name: CStr) usize { + const raw: CStr = c.getenv(name); + if (raw == null or raw[0] == 0) return 0; + std.c._errno().* = 0; + var end: MutStr = null; + const v = c.strtol(raw, &end, 10); + if (std.c._errno().* != 0 or end == raw or end[0] != 0 or v <= 0) return 0; + return @as(usize, @intCast(v)) *% 1024 *% 1024; +} + +/// Auto-derived TOTAL retention budget: a fraction of the process memory +/// budget, clamped to the absolute ceiling. The clamp bounds ONLY this +/// RAM-derived default — a huge-RAM host must not retain tens of GB it would +/// re-read cheaply. When the budget is unset (tests / no cgroup) fall back to +/// the ceiling. +fn cbmParallelExtractDefaultTotalCap() usize { + var cap: usize = c.cbm_mem_budget(); + if (cap > 0) { + cap /= 8; + } else { + cap = PP_RETAIN_TOTAL_HARD_MAX_BYTES; + } + if (cap > PP_RETAIN_TOTAL_HARD_MAX_BYTES) cap = PP_RETAIN_TOTAL_HARD_MAX_BYTES; + return cap; +} + +/// Auto-derived PER-FILE cap: modest absolute ceiling, never above the total. +fn cbmParallelExtractDefaultPerFileCap(total_cap: usize) usize { + var cap: usize = PP_RETAIN_PER_FILE_HARD_MAX_BYTES; + if (cap > total_cap) cap = total_cap; + return cap; +} + +/// Resolve the effective retention options from (in precedence order): +/// explicit caller opts > CBM_RETAIN_*_MB env knobs > RAM-derived defaults. +/// The absolute hard ceilings apply only to the RAM-derived defaults; an +/// operator/caller that sets a value explicitly is trusted. The only invariant +/// enforced unconditionally is per-file ≤ total. +fn cbmParallelExtractResolveOpts(opts: ?*const c.cbm_parallel_extract_opts_t) c.cbm_parallel_extract_opts_t { + var resolved = std.mem.zeroes(c.cbm_parallel_extract_opts_t); + resolved.retain_sources = true; + resolved.retain_sources_set = true; + resolved.retain_total_budget_bytes = cbmParallelExtractDefaultTotalCap(); + resolved.retain_per_file_max_bytes = 0; + + const env_total = cbmRetainEnvBytes("CBM_RETAIN_TOTAL_MB"); + if (env_total > 0) resolved.retain_total_budget_bytes = env_total; + + resolved.retain_per_file_max_bytes = + cbmParallelExtractDefaultPerFileCap(resolved.retain_total_budget_bytes); + const env_per_file = cbmRetainEnvBytes("CBM_RETAIN_PER_FILE_MB"); + if (env_per_file > 0) resolved.retain_per_file_max_bytes = env_per_file; + + if (opts) |o| { + if (o.retain_sources_set) resolved.retain_sources = o.retain_sources; + if (o.retain_total_budget_bytes > 0) { + resolved.retain_total_budget_bytes = o.retain_total_budget_bytes; + } + if (o.retain_per_file_max_bytes > 0) { + resolved.retain_per_file_max_bytes = o.retain_per_file_max_bytes; + } + } + + // Correctness invariant: a single file can never exceed the total budget. + if (resolved.retain_per_file_max_bytes > resolved.retain_total_budget_bytes) { + resolved.retain_per_file_max_bytes = resolved.retain_total_budget_bytes; + } + return resolved; +} + +fn extractNowNs() u64 { + var ts: Timespec = undefined; + _ = c.cbm_clock_gettime(c.CLOCK_MONOTONIC, &ts); + return (@as(u64, @bitCast(@as(i64, ts.tv_sec))) *% PP_NSEC_PER_SEC) +% + @as(u64, @bitCast(@as(i64, ts.tv_nsec))); +} + +// ── Helpers (duplicated from pass files — kept static for isolation) ── + +/// Read file into a malloc'd buffer (= mimalloc in production). +/// *out_size receives the on-disk size and *out_status the failure reason so the +/// caller can attribute a skip to the right phase (read vs oversized) instead of +/// a silent drop. Both out params may be NULL. +fn readFile(path: CStr, out_len: *c_int, out_size: ?*c_long, out_status: ?*c.cbm_read_status_t) MutStr { + if (out_size) |s| s.* = 0; + if (out_status) |s| s.* = c.CBM_READ_OK; + const f = c.cbm_fopen(path, "rb") orelse { + if (out_status) |s| s.* = c.CBM_READ_OPEN_FAIL; + return null; + }; + _ = c.fseek(f, 0, c.SEEK_END); + const size = c.ftell(f); + _ = c.fseek(f, 0, c.SEEK_SET); + if (out_size) |s| s.* = size; + if (size <= 0) { + _ = c.fclose(f); + if (out_status) |s| s.* = c.CBM_READ_EMPTY; + return null; + } + if (size > c.cbm_max_file_bytes()) { // generous, env-configurable cap (B4) + _ = c.fclose(f); + if (out_status) |s| s.* = c.CBM_READ_OVERSIZED; + return null; + } + // +pad: tree-sitter lexer lookahead reads past EOF; keep it in-bounds + const CBM_TS_LOOKAHEAD_PAD = 16; + const usize_size: usize = @intCast(size); + const buf: MutStr = @ptrCast(c.malloc(usize_size + CBM_TS_LOOKAHEAD_PAD)); + if (buf == null) { + _ = c.fclose(f); + if (out_status) |s| s.* = c.CBM_READ_OOM; + return null; + } + var nread = c.fread(buf, 1, usize_size, f); + _ = c.fclose(f); + if (nread > usize_size) nread = usize_size; + @memset(buf[nread .. nread + CBM_TS_LOOKAHEAD_PAD], 0); + out_len.* = @intCast(nread); + return buf; +} + +// ── Per-worker skip list (Stage 2 / Track B) ──────────────────────── +// Each extract worker appends read/extract/oversized skips into its OWN list +// (no lock on the hot path); the lists are merged into the pipeline's +// cbm_file_error_t array in the existing sequential merge loop. +const PpErrList = extern struct { + items: [*c]FileError = null, + count: c_int = 0, + cap: c_int = 0, +}; + +/// NULL-safe heap strdup. +fn ppErrDup(s: CStr) MutStr { + if (s == null) return null; + const n = c.strlen(s) + 1; + const d: MutStr = @ptrCast(c.malloc(n)); + if (d != null) @memcpy(d[0..n], s[0..n]); + return d; +} + +fn ppErrAdd(list_c: ?*PpErrList, path: CStr, reason: CStr, phase: CStr) void { + const list = list_c orelse return; + if (list.count >= list.cap) { + const ncap: c_int = if (list.cap != 0) list.cap * 2 else 8; + const grown = interop.reallocN(FileError, list.items, ix(ncap)); + if (grown == null) return; // drop on OOM — never fail extraction to record a skip + list.items = grown; + list.cap = ncap; + } + const e = util.at(list.items, ix(list.count)); + e.path = ppErrDup(path); + e.reason = ppErrDup(reason); + e.phase = ppErrDup(phase); + list.count += 1; +} + +/// Free source buffer. +fn freeSource(buf: MutStr) void { + c.free(buf); +} + +// Append a JSON-escaped string value to buf at position *pos. +/// Escape one character for JSON. Returns bytes written (1 or 2). +fn jsonEscapeChar(buf: [*]u8, avail: usize, ch: u8) usize { + const esc: u8 = switch (ch) { + '"' => '"', + '\\' => '\\', + '\n' => 'n', + '\r' => 'r', + '\t' => 't', + else => { + // Any other raw control byte (e.g. form feed) is invalid inside a + // JSON string — degrade to a space. + if (avail >= 1) buf[0] = if (ch < 0x20) ' ' else ch; + return 1; + }, + }; + if (avail >= PP_ESC_SPACE) { + buf[0] = '\\'; + buf[1] = esc; + } + return PP_ESC_SPACE; +} + +/// Escaped length of a string under jsonEscapeChar's rules: escaped +/// characters expand to 2 bytes, everything else stays 1. +fn ppJsonEscapedLen(s_in: CStr) usize { + var n: usize = 0; + var s = s_in; + while (s[0] != 0) : (s += 1) { + n += switch (s[0]) { + '"', '\\', '\n', '\r', '\t' => PP_ESC_SPACE, + else => 1, + }; + } + return n; +} + +/// Appends are ATOMIC: a field is emitted only if the WHOLE serialized form +/// fits (with PP_ESC_SPACE bytes reserved for the closing '}' + NUL). Cutting a +/// field mid-value produced unterminated strings/arrays — malformed properties +/// JSON that aborts every json_extract()-based consumer downstream (seen on the +/// Linux kernel: 50-param functions truncated at the 2 KB cap). Dropping an +/// oversized optional field whole keeps the JSON valid. +fn appendJsonString(buf: [*]u8, bufsize: usize, pos: *usize, key: CStr, val: CStr) void { + if (val == null or val[0] == 0) return; + const required = c.strlen(key) + ppJsonEscapedLen(val) + PP_JSON_FIELD_OVERHEAD; + if (pos.* + required + PP_ESC_SPACE > bufsize) { + return; // whole field would not fit — skip it atomically + } + var p = pos.*; + const w = c.snprintf(buf + p, bufsize - p, ",\"%s\":\"", key); + if (w <= 0 or ix(w) >= bufsize - p) return; + p += ix(w); + var s = val; + while (s[0] != 0 and p < bufsize - PP_ESC_MARGIN) : (s += 1) { + p += jsonEscapeChar(buf + p, bufsize - p - PP_ESC_SPACE, s[0]); + } + if (p < bufsize - 1) { + buf[p] = '"'; + p += 1; + } + buf[p] = 0; + pos.* = p; +} + +/// Append a JSON array of strings: ,"key":["a","b","c"]. Atomic like +/// appendJsonString: emitted only if the whole array fits. +fn appendJsonStrArray(buf: [*]u8, bufsize: usize, pos: *usize, key: CStr, arr: [*c]const CStr) void { + if (arr == null or arr[0] == null or pos.* >= bufsize - PP_JSON_MARGIN) return; + // ,"key":[ + per item "" + separating commas + ] + var required = c.strlen(key) + PP_JSON_FIELD_OVERHEAD; + var i: usize = 0; + while (arr[i] != null) : (i += 1) { + required += ppJsonEscapedLen(arr[i]) + PP_ESC_SPACE + @intFromBool(i > 0); + } + if (pos.* + required + PP_ESC_SPACE > bufsize) { + return; // whole array would not fit — skip it atomically + } + var p = pos.*; + const n = c.snprintf(buf + p, bufsize - p, ",\"%s\":[", key); + if (n <= 0 or p + ix(n) >= bufsize - PP_ESC_SPACE) return; + p += ix(n); + i = 0; + while (arr[i] != null) : (i += 1) { + if (i > 0 and p < bufsize - 1) { + buf[p] = ','; + p += 1; + } + if (p < bufsize - 1) { + buf[p] = '"'; + p += 1; + } + // Full escaping (not just quote/backslash): items like C param types + // sliced from multi-line declarations carry raw \n/\t bytes, which are + // invalid inside JSON strings. + var s: CStr = arr[i]; + while (s[0] != 0 and p < bufsize - PP_ESC_SPACE) : (s += 1) { + p += jsonEscapeChar(buf + p, bufsize - p - PP_ESC_SPACE, s[0]); + } + if (p < bufsize - 1) { + buf[p] = '"'; + p += 1; + } + } + if (p < bufsize - 1) { + buf[p] = ']'; + p += 1; + } + buf[p] = 0; + pos.* = p; +} + +/// Append an already-serialized JSON value verbatim: ,"key":. Atomic like +/// appendJsonString. For `variants`, which extraction builds as a JSON array +/// of objects (cbm.h). +fn appendJsonRaw(buf: [*]u8, bufsize: usize, pos: *usize, key: CStr, json: CStr) void { + if (json == null or json[0] == 0) return; + const required = c.strlen(key) + c.strlen(json) + PP_JSON_FIELD_OVERHEAD; + if (pos.* + required + PP_ESC_SPACE > bufsize) { + return; // whole field would not fit — skip it atomically + } + const w = c.snprintf(buf + pos.*, bufsize - pos.*, ",\"%s\":%s", key, json); + if (w > 0 and ix(w) < bufsize - pos.*) pos.* += ix(w); +} + +fn buildDefProps(buf: [*]u8, bufsize: usize, def: *const Definition) void { + // Complexity/loop/recursion metrics are meaningful only for Function/Method. + // Gate the block so the millions of Macro/Field/Variable/Class/Enum nodes + // keep a lean properties blob (lossless — those fields are always zero for + // non-functions). Cuts RAM, gbuf-merge copy and dump volume. + const is_fn = def.label != null and (streq(def.label, "Function") or streq(def.label, "Method")); + const n = if (is_fn) + c.snprintf( + buf, + bufsize, + "{\"complexity\":%d,\"cognitive\":%d,\"loop_count\":%d,\"loop_depth\":%d," ++ + "\"self_recursive\":%s,\"param_count\":%d,\"max_access_depth\":%d," ++ + "\"linear_scan_in_loop\":%d,\"alloc_in_loop\":%d,\"recursion_in_loop\":%s," ++ + "\"unguarded_recursion\":%s," ++ + "\"lines\":%d,\"is_exported\":%s,\"is_test\":%s,\"is_entry_point\":%s", + def.complexity, + def.cognitive, + def.loop_count, + def.loop_depth, + boolText(def.is_recursive), + def.param_count, + def.max_access_depth, + def.linear_scan_in_loop, + def.alloc_in_loop, + boolText(def.recursion_in_loop), + boolText(def.unguarded_recursion), + def.lines, + boolText(def.is_exported), + boolText(def.is_test), + boolText(def.is_entry_point), + ) + else + c.snprintf( + buf, + bufsize, + "{\"complexity\":%d,\"lines\":%d,\"is_exported\":%s,\"is_test\":%s," ++ + "\"is_entry_point\":%s", + def.complexity, + def.lines, + boolText(def.is_exported), + boolText(def.is_test), + boolText(def.is_entry_point), + ); + if (n <= 0 or ix(n) >= bufsize) { + buf[0] = 0; + return; + } + var pos: usize = ix(n); + appendJsonString(buf, bufsize, &pos, "docstring", def.docstring); + // Right after the docstring, ahead of the bounded fields, so a long + // signature cannot squeeze it out of the fixed buffer. + appendJsonRaw(buf, bufsize, &pos, "variants", def.variants); + appendJsonString(buf, bufsize, &pos, "signature", def.signature); + appendJsonString(buf, bufsize, &pos, "return_type", def.return_type); + appendJsonString(buf, bufsize, &pos, "parent_class", def.parent_class); + appendJsonStrArray(buf, bufsize, &pos, "decorators", def.decorators); + appendJsonStrArray(buf, bufsize, &pos, "base_classes", def.base_classes); + appendJsonStrArray(buf, bufsize, &pos, "param_names", def.param_names); + appendJsonStrArray(buf, bufsize, &pos, "param_types", def.param_types); + appendJsonString(buf, bufsize, &pos, "route_path", def.route_path); + appendJsonString(buf, bufsize, &pos, "route_method", def.route_method); + appendJsonString(buf, bufsize, &pos, "http_client", def.http_client); + appendJsonString(buf, bufsize, &pos, "http_base_url", def.http_base_url); + + // MinHash fingerprint — append if present and buffer has room. + // Hex-encoded K=64 uint32 = 512 chars + key/quotes ≈ 520 chars. + if (def.fingerprint != null and def.fingerprint_k > 0 and + pos + c.CBM_MINHASH_HEX_LEN + c.CBM_MINHASH_JSON_OVERHEAD < bufsize) + { + var fp_hex: [c.CBM_MINHASH_HEX_BUF]u8 = undefined; + c.cbm_minhash_to_hex(@ptrCast(@alignCast(def.fingerprint)), &fp_hex, fp_hex.len); + appendJsonString(buf, bufsize, &pos, "fp", &fp_hex); + } + + // AST structural profile — append if present and buffer has room. + if (def.structural_profile != null and pos + c.CBM_AST_PROFILE_BUF < bufsize) { + appendJsonString(buf, bufsize, &pos, "sp", def.structural_profile); + } + + // Body tokens — raw identifiers from function body AST for semantic search. + if (def.body_tokens != null and pos + c.CBM_SZ_512 < bufsize) { + appendJsonString(buf, bufsize, &pos, "bt", def.body_tokens); + } + + if (pos < bufsize - 1) { + buf[pos] = '}'; + buf[pos + 1] = 0; + } +} + +/// Build import map from graph buffer IMPORTS edges (read-only access to gbuf). +fn buildImportMap( + gbuf: ?*const Gbuf, + project_name: CStr, + rel_path: CStr, + out_keys: *StrList, + out_vals: *StrList, + out_count: *c_int, +) c_int { + out_keys.* = null; + out_vals.* = null; + out_count.* = 0; + + const file_qn: MutStr = c.cbm_pipeline_fqn_compute(project_name, rel_path, "__file__"); + const file_node: ?*const Node = c.cbm_gbuf_find_by_qn(gbuf, file_qn); + c.free(file_qn); + const fnode = file_node orelse return 0; + + var edges: [*c][*c]const Edge = null; + var edge_count: c_int = 0; + const rc = c.cbm_gbuf_find_edges_by_source_type(gbuf, fnode.id, "IMPORTS", &edges, &edge_count); + if (rc != 0 or edge_count == 0) return 0; + + const keys = interop.callocN(CStr, ix(edge_count)); + const vals = interop.callocN(CStr, ix(edge_count)); + var count: usize = 0; + + const marker = "\"local_name\":\""; + for (0..ix(edge_count)) |i| { + const e: *const Edge = edges[i]; + const target: ?*const Node = c.cbm_gbuf_find_by_id(gbuf, e.target_id); + if (target == null or e.properties_json == null) continue; + var start: CStr = c.strstr(e.properties_json, marker); + if (start != null) { + start += marker.len; + const end: CStr = c.strchr(start, '"'); + if (end != null and @intFromPtr(end) > @intFromPtr(start)) { + keys[count] = c.cbm_strndup(start, ptrDiff(end, start)); + vals[count] = target.?.qualified_name; + count += 1; + } + } + } + + out_keys.* = keys; + out_vals.* = vals; + out_count.* = @intCast(count); + return 0; +} + +fn freeImportMap(keys: StrList, vals: StrList, count: c_int) void { + if (keys != null) { + for (0..ix(count)) |i| c.free(@constCast(keys[i])); + c.free(@ptrCast(keys)); + } + if (vals != null) c.free(@ptrCast(vals)); +} + +/// True for languages whose module QN derives from the CONTAINING DIRECTORY +/// (Java/Go package). MUST match cbm_lang_module_is_dir() (internal/cbm/helpers.c) +/// and pxcModuleIsDir() (pass_lsp_cross) so same-module callee resolution +/// keys against the directory-based def-node QNs in the registry. +fn ppModuleIsDir(lang: Language) bool { + return lang == c.CBM_LANG_JAVA or lang == c.CBM_LANG_GO; +} + +fn isCheckedException(name: CStr) bool { + if (name == null) return false; + if (c.strstr(name, "Error") != null or c.strstr(name, "Panic") != null or + c.strstr(name, "error") != null or c.strstr(name, "panic") != null) + { + return false; + } + return true; +} + +fn resolveAsClass(reg: ?*const Registry, name: CStr, module_qn: CStr, imp_keys: StrList, imp_vals: StrList, imp_count: c_int) CStr { + const res = c.cbm_registry_resolve(reg, name, module_qn, imp_keys, imp_vals, imp_count); + if (res.qualified_name == null or res.qualified_name[0] == 0) return null; + // Accept any type-like container (Class/Struct/Interface/Enum/Type/Trait): + // base classes, Rust `impl Trait for S` struct receivers, and Go struct + // embedding all resolve through here. Struct included so the struct receiver + // of an IMPLEMENTS edge is not dropped. + const label: CStr = c.cbm_registry_label_of(reg, res.qualified_name); + if (!c.cbm_label_is_type_like(label)) return null; + return res.qualified_name; +} + +fn extractDecoratorFunc(dec: CStr, out: [*]u8, outsz: usize) void { + out[0] = 0; + if (dec == null) return; + var start = dec; + if (start[0] == '@') start += 1; + const paren: CStr = c.strchr(start, '('); + const len = if (paren != null) ptrDiff(paren, start) else c.strlen(start); + if (len == 0 or len >= outsz) return; + @memcpy(out[0..len], start[0..len]); + out[len] = 0; +} + +// ── File sort for tail-latency reduction ──────────────────────────── + +const FileSortEntry = extern struct { + idx: c_int, + size: i64, +}; + +fn compareBySizeDesc(a: ?*const anyopaque, b: ?*const anyopaque) callconv(.c) c_int { + const fa: *const FileSortEntry = @ptrCast(@alignCast(a)); + const fb: *const FileSortEntry = @ptrCast(@alignCast(b)); + if (fb.size > fa.size) return 1; + if (fb.size < fa.size) return c.CBM_NOT_FOUND; + return 0; +} + +// ── Phase 3A: Parallel Extract ────────────────────────────────────── + +const CBM_CACHE_LINE = c.CBM_SZ_128; + +/// One per extract worker, padded to a cache line of its own. +const ExtractWorkerState = extern struct { + local_gbuf: ?*Gbuf align(CBM_CACHE_LINE), + nodes_created: c_int, + errors: c_int, +}; + +const AtomicInt = std.atomic.Value(c_int); + +const ExtractCtx = struct { + files: [*c]const FileInfo, + sorted: [*c]FileSortEntry, + file_count: c_int, + project_name: CStr, + repo_path: CStr, + + workers: [*c]ExtractWorkerState, + max_workers: c_int, + next_worker_id: AtomicInt = .init(0), + + result_cache: [*c][*c]FileResult, + shared_ids: [*c]i64, + cancelled: [*c]c_int, + next_file_idx: AtomicInt = .init(0), + + /// per-worker manifest arrays (separate allocation) + pkg_entries: [*c]c.cbm_pkg_entries_t, + /// which Module definitions take `qn#ext` + stem_marks: ?*const c.cbm_stem_marks_t, + + /// copy source into result->arena for cross-file LSP + retain_sources: bool, + /// project-wide retention cap (peak-RSS bound) + retain_total_budget_bytes: usize, + /// per-file retention cap + retain_per_file_max_bytes: usize, + /// total source bytes copied into result arenas + retained_bytes: std.atomic.Value(i64) = .init(0), + /// WARN index.retain_capped emitted once per run + retain_cap_warned: AtomicInt = .init(0), + + /// Per-worker skip lists (separate allocation, indexed by worker_id — no hot- + /// path lock). Merged into the pipeline in the sequential merge loop. + err_lists: [*c]PpErrList, + /// throttle for the index.file_oversized WARN + oversized_warned: AtomicInt = .init(0), + + /// Back-pressure futility latch: set when a full collect+nap cycle ended + /// still over budget — the resident floor (graph + retained sources), not + /// in-flight transients, holds the memory, so napping cannot reclaim it. + /// While set, pulls skip the nap (the designed soft overshoot); the cheap + /// over-budget probe re-arms the gate once RSS drains under budget. + bp_futile: AtomicInt = .init(0), + /// Set once futility was CONFIRMED by a second full cycle that still ended + /// over budget. Every pull loop breaks on it and extract returns + /// CBM_PIPELINE_ABORT_OVER_BUDGET. Deliberately not the shared cancel + /// token — that one belongs to client cancellation. + over_budget_abort: AtomicInt = .init(0), + + /// Spill (F1). With a store, the first futility latch turns spilling on + /// instead of failing the attempt: every finished result is then parked on + /// disk and its result_cache slot cleared. result_owner[file_idx] is the + /// extracting worker id + 1, so each worker parks its own backlog once + /// (its own allocations, which its next cbm_mem_collect returns). Only a + /// latch that fires again while spilling fails the attempt whole. + spill: ?*Spill, + result_owner: [*c]c_int, + spilling: AtomicInt = .init(0), +}; + +/// Relaxed load of a C atomic_int reached through a pointer. +inline fn loadInt(p: *const c_int) c_int { + return @atomicLoad(c_int, p, .monotonic); +} + +/// Cap on the number of index.file_oversized WARN lines (the full list still goes +/// to the response/logfile — this only throttles the stderr noise). +const PP_OVERSIZED_WARN_MAX = 32; + +/// Insert one definition node (and its route if present) into the local gbuf. +fn insertDefIntoGbuf(ws: *ExtractWorkerState, marks: ?*const c.cbm_stem_marks_t, fi: *const FileInfo, def: *Definition) void { + var props: [c.CBM_SZ_2K]u8 = undefined; + buildDefProps(&props, props.len, def); + const rel: CStr = if (def.file_path != null) def.file_path else fi.rel_path; + var marked: [c.CBM_SZ_4K]u8 = undefined; + const qn: CStr = if (def.label != null and streq(def.label, "Module")) + c.cbm_stem_marks_module_qn(marks, def.qualified_name, rel, fi.language, &marked, marked.len) + else + def.qualified_name; + const func_id = c.cbm_gbuf_upsert_node( + ws.local_gbuf, + if (def.label != null) def.label else "Function", + def.name, + qn, + rel, + @bitCast(def.start_line), + @bitCast(def.end_line), + &props, + ); + ws.nodes_created += 1; + if (def.route_path != null and def.route_path[0] != 0) { + const rm: CStr = if (def.route_method != null) def.route_method else "ANY"; + var route_qn: [c.CBM_ROUTE_QN_SIZE]u8 = undefined; + var cpath: [c.CBM_SZ_256]u8 = undefined; + _ = c.snprintf(&route_qn, route_qn.len, "__route__%s__%s", rm, c.cbm_route_canon_path(def.route_path, &cpath, cpath.len)); + var rprops: [c.CBM_SZ_256]u8 = undefined; + _ = c.snprintf(&rprops, rprops.len, "{\"method\":\"%s\",\"source\":\"decorator\"}", rm); + const route_id = c.cbm_gbuf_upsert_node( + ws.local_gbuf, + "Route", + def.route_path, + &route_qn, + if (def.file_path != null) def.file_path else fi.rel_path, + 0, + 0, + &rprops, + ); + var hprops: [c.CBM_SZ_512]u8 = undefined; + var esc_h: [c.CBM_SZ_512]u8 = undefined; + _ = c.cbm_json_escape(&esc_h, esc_h.len, def.qualified_name); + _ = c.snprintf(&hprops, hprops.len, "{\"handler\":\"%s\"}", @as(CStr, &esc_h)); + _ = c.cbm_gbuf_insert_edge(ws.local_gbuf, func_id, route_id, "HANDLES", &hprops); + } +} + +/// (int)ms for a millisecond count, as C converts it. +fn msInt(ms: u64) c_int { + return @bitCast(@as(c_uint, @truncate(ms))); +} + +/// (int)(bytes / 1 MiB), as C converts it. +fn mbInt(bytes: usize) c_int { + return @bitCast(@as(c_uint, @truncate(bytes / (1024 * 1024)))); +} + +fn logExtractFail(pos: c_int, ms: u64, path: CStr) void { + if (pos < PP_LOG_THRESH) { + interop.logWarn("parallel.extract.file.fail", .{ + "pos", itoa(pos), + "elapsed_ms", itoa(msInt(ms)), + "path", path, + }); + } +} + +fn logExtractDone(pos: c_int, ms: u64, defs: c_int, path: CStr) void { + if (pos < PP_LOG_THRESH or ms > PP_TIMER_THRESH) { + interop.logInfo("parallel.extract.file.done", .{ + "pos", itoa(pos), + "elapsed_ms", itoa(msInt(ms)), + "defs", itoa(defs), + "path", path, + }); + } +} + +/// One back-pressure cycle: reclaim this thread's freed pages, then nap in +/// bounded 3 ms steps while the process stays over budget. Counted for test +/// observability. Returns true when the FULL cycle elapsed still over budget — +/// the resident floor, not in-flight transients, holds the memory. +fn ppBackpressureCycleStillOver(ec: *ExtractCtx) bool { + c.cbm_mem_collect(); + _ = g_bp_nap_cycles.fetchAdd(1, .monotonic); + var bp: c_int = 0; + while (bp < PP_BACKPRESSURE_MAX_SPINS and c.cbm_mem_over_budget() and + loadInt(ec.cancelled) == 0 and ec.over_budget_abort.load(.monotonic) == 0) : (bp += 1) + { + var nap: Timespec = .{ .tv_sec = 0, .tv_nsec = PP_BACKPRESSURE_NAP_NS }; + _ = c.cbm_nanosleep(&nap, null); + } + return bp == PP_BACKPRESSURE_MAX_SPINS and c.cbm_mem_over_budget(); +} + +/// Back-pressure was futile and the confirmation cycle still ended over budget. +/// The budget stops being advisory here: flag the attempt so every worker stops +/// pulling and extract returns CBM_PIPELINE_ABORT_OVER_BUDGET. Nothing is +/// published, so the previously served Zova generation keeps answering. +fn ppFailWholeOverBudget(ec: *ExtractCtx) void { + if (ec.over_budget_abort.swap(1, .monotonic) == 0) { + logError("mem.budget.exceeded", .{ + "rss_mb", itoa(mbInt(c.cbm_mem_rss())), + "budget_mb", itoa(mbInt(c.cbm_mem_budget())), + "phase", "parallel_extract", + "action", "fail_whole", + }); + } +} + +fn ppPark(ec: *ExtractCtx, file_idx: c_int, result: ?*FileResult) bool { + if (!c.cbm_result_spill_park(ec.spill, file_idx, result)) return false; + _ = g_spill_parked.fetchAdd(1, .monotonic); + return true; +} + +/// Park every result this worker extracted that is still in memory. +fn ppSpillBacklog(ec: *ExtractCtx, worker_id: c_int) void { + for (0..ix(ec.file_count)) |i| { + if (loadInt(util.at(ec.result_owner, i)) != worker_id + 1) continue; + if (ec.result_cache[i] != null and ppPark(ec, @intCast(i), ec.result_cache[i])) { + ec.result_cache[i] = null; + } + } + c.cbm_mem_collect(); +} + +/// The budget gate's futility latch fired. With a spill store that is not yet +/// spilling, turn spilling on and re-arm the gate; otherwise run the B3 +/// confirmation cycle and fail the attempt whole if it still ends over. +fn ppBackpressureFutile(ec: *ExtractCtx, worker_id: c_int) void { + if (ec.spill != null and ec.spilling.swap(1, .monotonic) == 0) { + interop.logWarn("mem.spill.engaged", .{ + "rss_mb", itoa(mbInt(c.cbm_mem_rss())), + "budget_mb", itoa(mbInt(c.cbm_mem_budget())), + }); + ppSpillBacklog(ec, worker_id); + ec.bp_futile.store(0, .monotonic); + return; + } + interop.logWarn("mem.backpressure.futile", .{ "action", "confirm" }); + if (ppBackpressureCycleStillOver(ec)) { + ppFailWholeOverBudget(ec); + } else { + ec.bp_futile.store(0, .monotonic); + } +} + +fn extractWorker(worker_id: c_int, ctx_ptr: ?*anyopaque) callconv(.c) void { + const ec: *ExtractCtx = @ptrCast(@alignCast(ctx_ptr)); + const ws = util.at(ec.workers, ix(worker_id)); + var backlog_spilled = false; + + // Lazy gbuf creation + if (ws.local_gbuf == null) { + ws.local_gbuf = c.cbm_gbuf_new_shared_ids(ec.project_name, ec.repo_path, ec.shared_ids); + } + + // Pull files from shared atomic counter + while (true) { + const sort_pos = ec.next_file_idx.fetchAdd(1, .monotonic); + if (sort_pos >= ec.file_count) break; + if (loadInt(ec.cancelled) != 0 or ec.over_budget_abort.load(.monotonic) != 0) break; + + // Memory back-pressure (large repos): if the process is over its RSS + // budget, reclaim this thread's freed pages and nap so peer workers can + // finish their current file and return memory before this worker adds + // another parse working set. Caps the concurrent extraction transient + // near the budget instead of letting all workers parse their biggest + // files at once. Self-disabling when the budget is unset (tests) or RSS + // is under budget; bounded spins avoid deadlock when the resident graph + // is itself near budget. + // + // Futility latch: when a FULL nap cycle ends still over budget, the + // resident floor — not transients — holds the memory; napping again on + // the next pull cannot reclaim it and only idles workers (linux kernel: + // one full cycle per pull ≈ 390 s at 79% avg CPU). The worker that + // latches bp_futile (0→1) runs ONE confirmation cycle while its peers + // skip the nap. A confirmation that ends under budget re-arms the gate + // (RSS drained after all); one that still ends over budget fails the + // attempt whole: every worker stops pulling, extract returns + // CBM_PIPELINE_ABORT_OVER_BUDGET, nothing is published and the + // previously served generation keeps answering. + if (c.cbm_mem_budget() > 0) { + const over = c.cbm_mem_over_budget(); + const futile = ec.bp_futile.load(.monotonic) != 0; + if (over and !futile) { + // Act only on the 0→1 transition: all workers race into the + // gate before anyone latches, so a plain store would confirm + // (and WARN) once per worker. + if (ppBackpressureCycleStillOver(ec) and ec.bp_futile.swap(1, .monotonic) == 0) { + ppBackpressureFutile(ec, worker_id); + } + } else if (!over and futile) { + ec.bp_futile.store(0, .monotonic); + } + } + if (ec.over_budget_abort.load(.monotonic) != 0) break; + const spilling = ec.spilling.load(.monotonic) != 0; + if (spilling and !backlog_spilled) { + ppSpillBacklog(ec, worker_id); + backlog_spilled = true; + } + + const file_idx = ec.sorted[ix(sort_pos)].idx; + const fi = util.at(ec.files, ix(file_idx)); + const errs: ?*PpErrList = if (ec.err_lists != null) util.at(ec.err_lists, ix(worker_id)) else null; + + // Crash-quarantine skip (Stage 3c): a file the supervisor pinned as a + // crasher must never be extracted again. Record it as a phase="crash" + // skip in this worker's list (merged into the pipeline's file-error list + // later, surfacing in skipped[]) and move on — the good files still + // index and status stays "indexed". No-op unless the supervisor set + // CBM_INDEX_QUARANTINE_FILE. cbm_extract_file's hard guard backstops + // this check. + if (c.cbm_index_is_quarantined(fi.rel_path)) { + var phase: CStr = c.cbm_index_quarantine_phase(fi.rel_path); + if (phase == null) phase = "crash"; + const reason: CStr = + if (streq(phase, "hang")) "quarantined after hang" else "quarantined after crash"; + ppErrAdd(errs, fi.rel_path, reason, phase); + ws.errors += 1; + continue; + } + + // Read + extract + var source_len: c_int = 0; + var file_size: c_long = 0; + var rst: c.cbm_read_status_t = c.CBM_READ_OK; + const source = readFile(fi.path, &source_len, &file_size, &rst); + if (source == null) { + ws.errors += 1; + if (rst == c.CBM_READ_OVERSIZED) { + // Never a silent drop: record the oversized skip + a throttled + // WARN so the file surfaces in the response/logfile. + const cap = c.cbm_max_file_bytes(); + const mib: c_long = c.CBM_SZ_1K * c.CBM_SZ_1K; + var reason: [96]u8 = undefined; + _ = c.snprintf( + &reason, + reason.len, + "oversized (%lld MB > %lld MB)", + @as(c_longlong, @divTrunc(file_size, mib)), + @as(c_longlong, @divTrunc(cap, mib)), + ); + ppErrAdd(errs, fi.rel_path, &reason, "oversized"); + if (ec.oversized_warned.fetchAdd(1, .monotonic) < PP_OVERSIZED_WARN_MAX) { + interop.logWarn("index.file_oversized", .{ + "path", fi.rel_path, + "size_mb", itoa(@truncate(@divTrunc(file_size, mib))), + "cap_mb", itoa(@truncate(@divTrunc(cap, mib))), + }); + } + } else if (rst == c.CBM_READ_OPEN_FAIL or rst == c.CBM_READ_OOM) { + ppErrAdd(errs, fi.rel_path, "read failed", "read"); + } + // CBM_READ_EMPTY: benign 0-byte file — not reported. + continue; + } + + // Per-file start log: shows which file each worker is processing. + // Critical for diagnosing stuck workers on large vendored files. + if (sort_pos < PP_LOG_THRESH) { // first 2 rounds of workers = most interesting + interop.logInfo("parallel.extract.file.start", .{ + "pos", itoa(sort_pos), + "size_kb", itoa(@divTrunc(source_len, c.CBM_SZ_1K)), + "path", fi.rel_path, + }); + } + + const file_t0 = extractNowNs(); + + const result_opt: ?*FileResult = c.cbm_extract_file(source, source_len, fi.language, ec.project_name, fi.rel_path, c.CBM_EXTRACT_BUDGET, null, null); + + const file_elapsed_ms = (extractNowNs() -% file_t0) / PP_USEC_PER_MS; + + const result = result_opt orelse { + logExtractFail(sort_pos, file_elapsed_ms, fi.rel_path); + freeSource(source); + ws.errors += 1; + ppErrAdd(errs, fi.rel_path, "extract failed", "extract"); + continue; + }; + logExtractDone(sort_pos, file_elapsed_ms, result.defs.count, fi.rel_path); + + // Consume the previously-ignored has_error flag: a parse timeout / parse + // failure / unsupported-grammar result carries no defs but must still be + // reported (phase "extract", reason = the extractor's message). The empty + // result flows through unchanged below (the defs loop is a no-op). + if (result.has_error) { + ppErrAdd(errs, fi.rel_path, if (result.error_msg != null) result.error_msg else "extract failed", "extract"); + ws.errors += 1; + } else if (result.parse_incomplete) { + // Indexed, but with unrecovered ERROR/MISSING regions. Not a skip: + // recorded under the distinct "parse_partial" phase with the + // line-range list as the reason, or "parse_unusable" when one range + // covers most of the file (see parse_unusable in cbm.h). + ppErrAdd( + errs, + fi.rel_path, + if (result.error_ranges != null) result.error_ranges else "", + if (result.parse_unusable) "parse_unusable" else "parse_partial", + ); + } + + // Create definition nodes in local gbuf + for (0..ix(result.defs.count)) |d| { + const def = util.at(result.defs.items, d); + if (def.qualified_name != null and def.name != null) { + insertDefIntoGbuf(ws, ec.stem_marks, fi, def); + } + } + + // Free TSTree immediately — arena strings survive for registry+resolve. + // This makes slab reset safe: tree-sitter's internal nodes (in slab) + // are released before the slab is bulk-reclaimed. + c.cbm_free_tree(result); + + // Detect and parse manifest files for package map + { + const bn: CStr = c.strrchr(fi.rel_path, '/'); + _ = c.cbm_pkgmap_try_parse( + if (bn != null) bn + 1 else fi.rel_path, + fi.rel_path, + source, + source_len, + util.at(ec.pkg_entries, ix(worker_id)), + ); + } + + // Retain source bytes in result->arena so the fused cross-file LSP + // step in resolve_worker can re-parse without re-reading from disk. + // Bounded per-file (retain_per_file_max_bytes) and by a project-wide + // budget (retain_total_budget_bytes) to bound peak RSS — see the + // retention-cap comment at the top of this file. A file DROPPED here + // is NOT lost to cross-file resolution: resolve_worker re-reads it on + // demand (bounded, freed immediately). WARN once per run so the + // operator knows retention was capped (index.retain_capped). + if (ec.retain_sources and source_len > 0) { + var dropped = false; + const len: usize = ix(source_len); + if (len > ec.retain_per_file_max_bytes) { + dropped = true; // over the per-file cap + } else { + const prior = ec.retained_bytes.fetchAdd(source_len, .monotonic); + if (@as(usize, @bitCast(prior + source_len)) <= ec.retain_total_budget_bytes) { + const copy: MutStr = @ptrCast(c.cbm_arena_alloc(&result.arena, len + 1)); + if (copy != null) { + @memcpy(copy[0..len], source[0..len]); + copy[len] = 0; + result.source = copy; + result.source_len = source_len; + } else { + // Arena OOM — not a cap drop; re-read still covers + // cross-file resolution, so don't emit the WARN. + _ = ec.retained_bytes.fetchSub(source_len, .monotonic); + } + } else { + _ = ec.retained_bytes.fetchSub(source_len, .monotonic); + dropped = true; // project-wide budget exhausted + } + } + if (dropped and ec.retain_cap_warned.swap(1, .monotonic) == 0) { + interop.logWarn("index.retain_capped", .{ + "path", @as(CStr, if (fi.rel_path != null) fi.rel_path else "?"), + "bytes", itoa(source_len), + }); + } + } + + // Free source buffer — extraction captured everything needed, + // and the retention copy (if any) lives in result->arena. + freeSource(source); + + // Cache result (arena + extracted data, no tree) for Phase 3B and Phase 4. + // While spilling it goes straight to disk; a failed park keeps it. + if (!spilling or !ppPark(ec, file_idx, result)) { + ec.result_cache[ix(file_idx)] = result; + if (ec.result_owner != null) { + @atomicStore(c_int, util.at(ec.result_owner, ix(file_idx)), worker_id + 1, .monotonic); + } + } + + // Progress logging: log every 10 files (atomic read, no contention) + if (@rem(sort_pos + 1, PP_LOG_INTERVAL) == 0 or sort_pos + 1 == ec.file_count) { + interop.logInfo("parallel.extract.progress", .{ + "done", itoa(sort_pos + 1), + "total", itoa(ec.file_count), + }); + } + + // Reclaim all slab + tier2 memory between files. + // + // After cbm_free_tree(result), all tree nodes are on free lists. + // We then destroy the parser (frees its internal allocations too), + // leaving ZERO live slab/tier2 pointers. At that point, we can + // safely munmap/free every page, bounding peak memory per-file + // instead of accumulating across all 644 files. + // + // get_thread_parser() in cbm_extract_file will create a fresh + // parser for the next file — cost is microseconds vs seconds + // for parsing. This prevents unbounded memory accumulation and works + // identically on macOS, Linux, and Windows. + c.cbm_destroy_thread_parser(); + c.cbm_slab_reclaim(); + c.cbm_mem_collect(); + } + + // Final cleanup (parser already destroyed in loop, just slab state) + c.cbm_slab_destroy_thread(); + c.cbm_kind_in_set_free_cache(); // free this worker thread's node-type bitset cache +} + +fn mergePkgEntries(ctx: *Ctx, pkg_entries: [*c]c.cbm_pkg_entries_t, worker_count: c_int) void { + if (pkg_entries == null) return; + // Supplement with a repo-wide filesystem walk so manifests filtered + // by the main discoverer (package.json, composer.json — in + // IGNORED_JSON_FILES) still feed pkgmap. Append into worker 0's + // array so the existing merge below sees them. + _ = c.cbm_pkgmap_scan_repo(ctx.repo_path, util.at(pkg_entries, 0), ctx.excluded_dirs, ctx.excluded_count); + ctx.pkgmap = c.cbm_pkgmap_build(pkg_entries, worker_count, ctx.project_name); + c.cbm_pipeline_set_pkgmap(ctx.pkgmap); + for (0..ix(worker_count)) |i| c.cbm_pkg_entries_free(util.at(pkg_entries, i)); + c.free(pkg_entries); +} + +fn logExtractMemStats(worker_count: c_int) void { + if (c.cbm_mem_budget() > 0) { + interop.logInfo("parallel.extract.mem", .{ + "rss_mb", itoa(mbInt(c.cbm_mem_rss())), + "peak_mb", itoa(mbInt(c.cbm_mem_peak_rss())), + "budget_mb", itoa(mbInt(c.cbm_mem_budget())), + "per_worker_mb", itoa(mbInt(c.cbm_mem_worker_budget(worker_count))), + }); + } +} + +pub export fn cbm_parallel_extract_ex( + ctx: *Ctx, + files: [*c]const FileInfo, + file_count: c_int, + result_cache: [*c][*c]FileResult, + shared_ids: [*c]i64, + worker_count: c_int, + opts: ?*const c.cbm_parallel_extract_opts_t, +) c_int { + const resolved_opts = cbmParallelExtractResolveOpts(opts); + + if (file_count == 0) return 0; + + interop.logInfo("parallel.extract.start", .{ + "files", itoa(file_count), + "workers", itoa(worker_count), + }); + interop.logInfo("parallel.extract.retention", .{ + "retain_sources", boolText(resolved_opts.retain_sources), + "total_mb", itoa(mbInt(resolved_opts.retain_total_budget_bytes)), + "per_file_mb", itoa(mbInt(resolved_opts.retain_per_file_max_bytes)), + }); + + // Log per-worker memory budget + if (c.cbm_mem_budget() > 0) { + const worker_budget = c.cbm_mem_worker_budget(worker_count); + interop.logInfo("parallel.mem.budget", .{ + "total_mb", itoa(mbInt(c.cbm_mem_budget())), + "per_worker_mb", itoa(mbInt(worker_budget)), + }); + } + + // Sub-phase: Ensure extraction library is initialized + const t_init = util.profStart(); + _ = c.cbm_init(); + + // Slab allocator for tree-sitter (thread-safe via TLS). + c.cbm_slab_install(); + util.profEnd("parallel_extract", "1_init_libs", &t_init, 0); + + // Sub-phase: Sort files by descending size for tail-latency reduction + const t_sort = util.profStart(); + const sorted = interop.mallocN(FileSortEntry, ix(file_count)); + if (sorted == null) return c.CBM_NOT_FOUND; + for (0..ix(file_count)) |i| { + sorted[i].idx = @intCast(i); + sorted[i].size = files[i].size; + } + if (file_count > 1) { + c.qsort(sorted, ix(file_count), @sizeOf(FileSortEntry), compareBySizeDesc); + } + util.profEnd("parallel_extract", "2_sort_files", &t_sort, file_count); + + // Allocate per-worker state (cache-line aligned via posix_memalign) + var workers_raw: ?*anyopaque = null; + const workers_bytes = ix(worker_count) * @sizeOf(ExtractWorkerState); + if (c.cbm_aligned_alloc(&workers_raw, CBM_CACHE_LINE, workers_bytes) != 0) { + c.free(sorted); + return c.CBM_NOT_FOUND; + } + const workers: [*c]ExtractWorkerState = @ptrCast(@alignCast(workers_raw)); + @memset(@as([*]u8, @ptrCast(workers))[0..workers_bytes], 0); + + // Per-worker manifest entry arrays (separate from cache-line-aligned worker state) + const pkg_entries = interop.callocN(c.cbm_pkg_entries_t, ix(worker_count)); + if (pkg_entries == null) { + c.cbm_aligned_free(workers); + c.free(sorted); + return c.CBM_NOT_FOUND; + } + + // Per-worker skip lists (separate allocation; merged into the pipeline in the + // sequential merge loop below). + const err_lists = interop.callocN(PpErrList, ix(worker_count)); + + var ec: ExtractCtx = .{ + .files = files, + .sorted = sorted, + .file_count = file_count, + .project_name = ctx.project_name, + .repo_path = ctx.repo_path, + .workers = workers, + .max_workers = worker_count, + .result_cache = result_cache, + .shared_ids = shared_ids, + .cancelled = ctx.cancelled, + .pkg_entries = pkg_entries, + .stem_marks = ctx.stem_marks, + .err_lists = err_lists, + .retain_sources = resolved_opts.retain_sources, + .retain_total_budget_bytes = resolved_opts.retain_total_budget_bytes, + .retain_per_file_max_bytes = resolved_opts.retain_per_file_max_bytes, + .spill = ctx.spill, + .result_owner = null, + }; + ec.result_owner = if (ec.spill != null) interop.callocN(c_int, ix(file_count)) else null; + if (ec.result_owner == null) { + ec.spill = null; // no owner map, no spilling: the B3 path stays + } + ec.spilling = .init(@intFromBool(ec.spill != null and g_force_spill.load(.monotonic) != 0)); + + // Sub-phase: Dispatch workers (parse + extract per file, PARALLEL) + const t_dispatch = util.profStart(); + var parallel_opts = std.mem.zeroes(c.cbm_parallel_for_opts_t); + parallel_opts.max_workers = worker_count; + parallel_opts.force_pthreads = false; + c.cbm_parallel_for(worker_count, extractWorker, &ec, parallel_opts); + util.profEnd("parallel_extract", "3_dispatch_workers_parallel", &t_dispatch, file_count); + + // Sub-phase: Merge all local gbufs into main gbuf (SEQUENTIAL, gbuf not thread-safe) + const t_merge = util.profStart(); + var total_nodes: c_int = 0; + var total_errors: c_int = 0; + for (0..ix(worker_count)) |i| { + if (workers[i].local_gbuf != null) { + _ = c.cbm_gbuf_merge(ctx.gbuf, workers[i].local_gbuf); + total_nodes += workers[i].nodes_created; + total_errors += workers[i].errors; + c.cbm_gbuf_free(workers[i].local_gbuf); + } + } + util.profEnd("parallel_extract", "4_merge_gbufs_seq", &t_merge, total_nodes); + + // Merge per-worker skip lists into the pipeline (SEQUENTIAL — no lock). + // Runs unconditionally (not gated on local_gbuf) so a worker whose files all + // failed still surfaces its skips. + if (err_lists != null) { + for (0..ix(worker_count)) |i| { + for (0..ix(err_lists[i].count)) |j| { + const item = &err_lists[i].items[j]; + c.cbm_pipeline_add_file_error(ctx.pipeline, item.path, item.reason, item.phase); + c.free(item.path); + c.free(item.reason); + c.free(item.phase); + } + c.free(err_lists[i].items); + } + c.free(err_lists); + } + + mergePkgEntries(ctx, pkg_entries, worker_count); + + c.cbm_aligned_free(workers); + c.free(sorted); + c.free(ec.result_owner); + + // The over-budget verdict outranks the cancel sentinel: a caller must be + // able to name the cause. Every non-zero code stops before publication. + if (ec.over_budget_abort.load(.seq_cst) != 0) return c.CBM_PIPELINE_ABORT_OVER_BUDGET; + if (@atomicLoad(c_int, ctx.cancelled, .seq_cst) != 0) return c.CBM_NOT_FOUND; + + logExtractMemStats(worker_count); + + interop.logInfo("parallel.extract.done", .{ + "nodes", itoa(total_nodes), + "errors", itoa(total_errors), + }); + return 0; +} + +pub export fn cbm_parallel_extract( + ctx: *Ctx, + files: [*c]const FileInfo, + file_count: c_int, + result_cache: [*c][*c]FileResult, + shared_ids: [*c]i64, + worker_count: c_int, +) c_int { + return cbm_parallel_extract_ex(ctx, files, file_count, result_cache, shared_ids, worker_count, null); +} + +// ── Phase 3B: Serial Registry Build ───────────────────────────────── + +/// Register one definition and create DEFINES + DEFINES_METHOD edges. Returns edge count. +fn registerAndLinkDef(ctx: *Ctx, def: *const Definition, rel: CStr, lang: Language, reg_entries: *c_int) c_int { + var edges: c_int = 0; + if (def.name == null or def.qualified_name == null or def.label == null) return 0; + // Register callable symbols + every type-like container (Class/Struct/ + // Interface/Enum/Type/Trait). Type-like defs must be in the registry so + // `class Foo : IBar` (INHERITS), `impl Trait for S` (IMPLEMENTS), and + // method/field resolution can reach them — Struct included so Rust/Go/ + // Swift/D structs resolve as type targets. Variable/Field defs are + // registered too so READS/WRITES can resolve. + // KEEP IN SYNC with pipeline_incremental's seed set. + if (streq(def.label, "Function") or streq(def.label, "Method") or + c.cbm_label_is_type_like(def.label) or streq(def.label, "Variable") or + streq(def.label, "Field")) + { + c.cbm_registry_add(ctx.registry, def.name, def.qualified_name, def.label); + reg_entries.* += 1; + } + const file_qn: MutStr = c.cbm_pipeline_fqn_compute(ctx.project_name, rel, "__file__"); + const file_node: ?*const Node = c.cbm_gbuf_find_by_qn(ctx.gbuf, file_qn); + var marked: [c.CBM_SZ_4K]u8 = undefined; + const qn: CStr = if (streq(def.label, "Module")) + c.cbm_stem_marks_module_qn( + ctx.stem_marks, + def.qualified_name, + if (def.file_path != null) def.file_path else rel, + lang, + &marked, + marked.len, + ) + else + def.qualified_name; + const def_node: ?*const Node = c.cbm_gbuf_find_by_qn(ctx.gbuf, qn); + if (file_node != null and def_node != null) { + _ = c.cbm_gbuf_insert_edge(ctx.gbuf, file_node.?.id, def_node.?.id, "DEFINES", "{}"); + edges += 1; + } + c.free(file_qn); + if (def.parent_class != null and streq(def.label, "Method")) { + const parent: ?*const Node = c.cbm_gbuf_find_by_qn(ctx.gbuf, def.parent_class); + if (parent != null and def_node != null) { + _ = c.cbm_gbuf_insert_edge(ctx.gbuf, parent.?.id, def_node.?.id, "DEFINES_METHOD", "{}"); + } + } + return edges; +} + +/// Create IMPORTS edges for one file's imports (parallel path). +fn createImportsEdges(ctx: *Ctx, result: *const FileResult, rel: CStr, namespace_map: ?*HashTable) c_int { + var count: c_int = 0; + const file_qn: MutStr = c.cbm_pipeline_fqn_compute(ctx.project_name, rel, "__file__"); + const source_node: *const Node = c.cbm_gbuf_find_by_qn(ctx.gbuf, file_qn) orelse { + c.free(file_qn); + return 0; + }; + for (0..ix(result.imports.count)) |j| { + const imp = util.at(result.imports.items, j); + if (imp.module_path == null) continue; + const target: ?*const Node = c.cbm_pipeline_resolve_import_node(ctx, rel, file_qn, imp, namespace_map); + if (target != null and target.?.id != source_node.id) { + var esc_ln: [c.CBM_SZ_128]u8 = undefined; + _ = c.cbm_json_escape(&esc_ln, esc_ln.len, if (imp.local_name != null) imp.local_name else ""); + var imp_props: [c.CBM_SZ_256]u8 = undefined; + _ = c.snprintf(&imp_props, imp_props.len, "{\"local_name\":\"%s\"}", @as(CStr, &esc_ln)); + _ = c.cbm_gbuf_insert_edge(ctx.gbuf, source_node.id, target.?.id, "IMPORTS", &imp_props); + count += 1; + } + } + c.free(file_qn); + return count; +} + +/// Find channel source node (enclosing function or file). +fn findChannelSrc(ctx: *Ctx, ch: *const c.CBMChannel, rel: CStr) ?*const Node { + var node: ?*const Node = null; + if (ch.enclosing_func_qn != null and ch.enclosing_func_qn[0] != 0) { + node = c.cbm_gbuf_find_by_qn(ctx.gbuf, ch.enclosing_func_qn); + } + if (node == null) { + const file_qn: MutStr = c.cbm_pipeline_fqn_compute(ctx.project_name, rel, "__file__"); + node = c.cbm_gbuf_find_by_qn(ctx.gbuf, file_qn); + c.free(file_qn); + } + return node; +} + +/// Create Channel nodes + EMITS/LISTENS_ON edges for one file. +fn createChannelEdges(ctx: *Ctx, result: *const FileResult, rel: CStr) void { + for (0..ix(result.channels.count)) |j| { + const ch = util.at(result.channels.items, j); + if (ch.channel_name == null or ch.channel_name[0] == 0) continue; + const transport: CStr = if (ch.transport != null) ch.transport else "unknown"; + var channel_qn: [c.CBM_SZ_512]u8 = undefined; + _ = c.snprintf(&channel_qn, channel_qn.len, "__channel__%s__%s", transport, ch.channel_name); + var esc_cn: [c.CBM_SZ_256]u8 = undefined; + _ = c.cbm_json_escape(&esc_cn, esc_cn.len, ch.channel_name); + var channel_props: [c.CBM_SZ_512]u8 = undefined; + _ = c.snprintf(&channel_props, channel_props.len, "{\"transport\":\"%s\",\"name\":\"%s\"}", transport, @as(CStr, &esc_cn)); + const channel_id = c.cbm_gbuf_upsert_node(ctx.gbuf, "Channel", ch.channel_name, &channel_qn, "", 0, 0, &channel_props); + const src_node = findChannelSrc(ctx, ch, rel); + if (src_node != null and channel_id > 0) { + const edge_type: CStr = if (ch.direction == c.CBM_CHANNEL_EMIT) "EMITS" else "LISTENS_ON"; + var edge_props: [c.CBM_SZ_128]u8 = undefined; + _ = c.snprintf(&edge_props, edge_props.len, "{\"transport\":\"%s\"}", transport); + _ = c.cbm_gbuf_insert_edge(ctx.gbuf, src_node.?.id, channel_id, edge_type, &edge_props); + } + } +} + +/// Create CONFIGURES edges for one file's env accesses. +fn createEnvConfigures(ctx: *Ctx, result: *const FileResult, rel: CStr) void { + var file_qn: MutStr = null; + var file_node: ?*const Node = null; + for (0..ix(result.env_accesses.count)) |j| { + const ea = util.at(result.env_accesses.items, j); + if (ea.env_key == null or ea.env_key[0] == 0) continue; + var env_qn: [c.CBM_SZ_512]u8 = undefined; + _ = c.snprintf(&env_qn, env_qn.len, "__env__%s", ea.env_key); + var env_props: [c.CBM_SZ_512]u8 = undefined; + _ = c.snprintf(&env_props, env_props.len, "{\"env_key\":\"%s\"}", ea.env_key); + const env_id = c.cbm_gbuf_upsert_node(ctx.gbuf, "EnvVar", ea.env_key, &env_qn, "", 0, 0, &env_props); + if (env_id <= 0) continue; + var src: ?*const Node = null; + if (ea.enclosing_func_qn != null and ea.enclosing_func_qn[0] != 0) { + src = c.cbm_gbuf_find_by_qn(ctx.gbuf, ea.enclosing_func_qn); + } + if (src == null) { + if (file_qn == null) { + file_qn = c.cbm_pipeline_fqn_compute(ctx.project_name, rel, "__file__"); + file_node = c.cbm_gbuf_find_by_qn(ctx.gbuf, file_qn); + } + src = file_node; + } + if (src != null and src.?.id != env_id) { + _ = c.cbm_gbuf_insert_edge(ctx.gbuf, src.?.id, env_id, "CONFIGURES", "{\"strategy\":\"env_access\"}"); + } + } + c.free(file_qn); +} + +pub export fn cbm_build_registry_from_cache( + ctx: *Ctx, + files: [*c]const FileInfo, + file_count: c_int, + result_cache: [*c][*c]FileResult, +) c_int { + interop.logInfo("parallel.registry.start", .{ "files", itoa(file_count) }); + + var reg_entries: c_int = 0; + var defines_edges: c_int = 0; + var imports_edges: c_int = 0; + + // Namespace/package → File-QN map for namespace imports (C# `using`, + // Java/Kotlin `import`, PHP `use`). Built from the full result cache so + // every declaring file is visible regardless of loop order. + const rels = interop.callocN(CStr, ix(file_count)); + if (rels != null) { + for (0..ix(file_count)) |i| rels[i] = files[i].rel_path; + } + const namespace_map = + c.cbm_pipeline_namespace_map_build(ctx.project_name, result_cache, ctx.spill, rels, file_count); + c.free(@ptrCast(rels)); + + for (0..ix(file_count)) |i| { + if (util.checkCancel(ctx) != 0) { + c.cbm_pipeline_namespace_map_free(namespace_map); + return c.CBM_NOT_FOUND; + } + + const result: *FileResult = c.cbm_result_cache_acquire(result_cache, ctx.spill, @intCast(i)) orelse continue; + + const rel: CStr = files[i].rel_path; + + // Register callable symbols + DEFINES/DEFINES_METHOD edges + for (0..ix(result.defs.count)) |d| { + defines_edges += registerAndLinkDef(ctx, util.at(result.defs.items, d), rel, files[i].language, ®_entries); + } + + imports_edges += createImportsEdges(ctx, result, rel, namespace_map); + createChannelEdges(ctx, result, rel); + createEnvConfigures(ctx, result, rel); + c.cbm_result_cache_release(result_cache, @intCast(i), result); + } + + c.cbm_pipeline_namespace_map_free(namespace_map); + + interop.logInfo("parallel.registry.done", .{ + "entries", itoa(reg_entries), + "defines", itoa(defines_edges), + "imports", itoa(imports_edges), + }); + return 0; +} + +// ── Phase 4: Parallel Resolution ──────────────────────────────────── + +/// One per resolve worker, padded to a cache line of its own. +const ResolveWorkerState = extern struct { + local_edge_buf: ?*Gbuf align(CBM_CACHE_LINE), + calls_resolved: c_int, + usages_resolved: c_int, + semantic_resolved: c_int, + errors: c_int, + /// Subset of calls_resolved that were attributed via the LSP-override + /// path (cbm_pipeline_find_lsp_resolution hit) rather than the + /// registry's textual matcher. Surfaced in the parallel.resolve.done + /// log line so divergence between pipelines becomes observable. + lsp_overrides: c_int, +}; + +const AtomicU64 = std.atomic.Value(u64); + +const ResolveCtx = struct { + files: [*c]const FileInfo, + file_count: c_int, + project_name: CStr, + repo_path: CStr, + + workers: [*c]ResolveWorkerState, + max_workers: c_int, + + result_cache: [*c][*c]FileResult, + /// parked results (F1); NULL when nothing spilled + spill: ?*Spill, + /// READ-ONLY during Phase 4 + main_gbuf: ?*const Gbuf, + /// READ-ONLY during Phase 4 + registry: ?*const Registry, + shared_ids: [*c]i64, + cancelled: [*c]c_int, + next_file_idx: AtomicInt = .init(0), + + /// Cross-file LSP inputs — pre-built once by the caller in pipeline.zig + /// and shared read-only by usage across workers (typed non-const to + /// match the existing cbm_run_X_lsp_cross callee signatures the + /// worker forwards them to). NULL/0 → cross-LSP no-ops. + all_defs: [*c]LspDef, + def_count: c_int, + /// per-file module QN; def_modules[i] for files[i] + def_modules: [*c]const MutStr, + /// Optional inverted index for per-file def filtering (gopls pattern). + /// When non-NULL, the fused worker calls cbm_pxc_filter_defs_for_file + /// to shrink the def array passed to the LSP from O(all_defs) to + /// O(relevant_defs). NULL → each file sees the full all_defs[]. + module_def_index: ?*c.struct_CBMModuleDefIndex, + /// Tier 2 full: pre-built per-language registries (project-wide, + /// finalized, READ-ONLY). When non-NULL for a lang, the worker uses + /// cbm_run_X_lsp_cross_with_registry — skip per-file build entirely. + /// Stored as CBMCrossLspRegistries* (typedef from pass_lsp_cross.h). + cross_registries: ?*c.CBMCrossLspRegistries, + + /// Parsed once on the coordinator thread and borrowed read-only by resolve + /// workers. The pointer is installed into each worker's TLS slot so Rust's + /// manifest-aware cross-file resolver sees the same Cargo context. + rust_manifest: ?*const c.CBMCargoManifest = null, + + /// F4: LAZILY-built shared Rust registry (built ONCE, on the first NULL-filter + /// rust file — the ~all_defs amplifier files). Not eager: repos whose rust files + /// all filter to subsets never pay the O(all_defs) build + multi-GB RSS. Built + /// under rust_shared_mu into rust_shared_arena, published via rust_shared_reg; + /// torn down after the worker dispatch. + rust_shared_reg: std.atomic.Value(?*c.CBMTypeRegistry) = .init(null), + rust_shared_mu: c.cbm_mutex_t = undefined, + rust_shared_arena: Arena = undefined, + rust_shared_arena_live: bool = false, + + /// Counters for parallel.resolve.lsp_cross_done summary. + lsp_cross_processed: AtomicInt = .init(0), + lsp_cross_skipped_no_source: AtomicInt = .init(0), + /// Error message of a cross-file LSP walk that stopped incomplete, per + /// file index. Each file is owned by one worker; the coordinator reports + /// them after join, so the pipeline's error list is never mutated + /// concurrently. + lsp_failures: StrList = null, + + /// Per-sub-phase timing (ns aggregated across workers) — surfaces + /// exactly where parallel_resolve's wall time is spent so we stop + /// guessing about hot paths. Logged once at the end of + /// cbm_parallel_resolve. + time_ns_import_map: AtomicU64 = .init(0), + time_ns_cross_lsp: AtomicU64 = .init(0), + time_ns_calls: AtomicU64 = .init(0), + time_ns_usages: AtomicU64 = .init(0), + time_ns_throws: AtomicU64 = .init(0), + time_ns_rw: AtomicU64 = .init(0), + time_ns_semantic: AtomicU64 = .init(0), + /// Whole-iteration timer — captures everything from atomic file_idx + /// pickup through cleanup. If this >> sum of sub-phases, the + /// unmeasured cost is either in skip-eligibility checks, gbuf + /// setup, or — most likely — workers waiting on the + /// cbm_parallel_for synchronization barrier at the end. + time_ns_total_loop: AtomicU64 = .init(0), + total_files_visited: AtomicInt = .init(0), + /// Sub-breakdowns inside resolve_file_calls — finds the 553µs-per- + /// iteration hot path that the high-level resolve_calls counter + /// doesn't pinpoint. + /// lsp_idx + fallback scan + time_ns_rc_lsp_lookup: AtomicU64 = .init(0), + /// lsp_target_node OR registry_resolve + time_ns_rc_resolve: AtomicU64 = .init(0), + /// try_field_type_hint + time_ns_rc_hint: AtomicU64 = .init(0), + /// gbuf_find_by_qn for target + time_ns_rc_target: AtomicU64 = .init(0), + /// emit_service_edge + time_ns_rc_emit: AtomicU64 = .init(0), + /// find_source_node + time_ns_rc_source: AtomicU64 = .init(0), +}; + +/// Minimum buffer space needed per arg JSON object +const CBM_ARG_JSON_GUARD = c.CBM_SZ_32; + +// Append arg data as JSON to edge properties: ,"args":[{"i":0,"e":"x","v":"val"},...] +// Returns new position in buffer. +/// Sanitize expression string for JSON (in-place). +fn sanitizeExpr(expr_buf: *[c.CBM_SZ_128]u8, expr: CStr) void { + if (expr != null) { + _ = c.snprintf(expr_buf, 128, "%.*s", @as(c_int, 120), expr); + var i: usize = 0; + while (expr_buf[i] != 0) : (i += 1) { + if (expr_buf[i] == '"') expr_buf[i] = '\''; + if (expr_buf[i] == '\n' or expr_buf[i] == '\r') expr_buf[i] = ' '; + } + } else { + expr_buf[0] = 0; + } +} + +/// Format one call arg as JSON. Returns snprintf result. +fn formatCallArg(buf: [*]u8, bufsize: usize, a: *const c.CBMCallArg, expr: CStr) c_int { + var esc_k: [c.CBM_SZ_128]u8 = undefined; + var esc_e: [c.CBM_SZ_128]u8 = undefined; + var esc_v: [c.CBM_SZ_128]u8 = undefined; + const k: CStr = &esc_k; + const e: CStr = &esc_e; + const v: CStr = &esc_v; + _ = c.cbm_json_escape(&esc_e, esc_e.len, expr); + if (a.keyword != null and a.value != null) { + _ = c.cbm_json_escape(&esc_k, esc_k.len, a.keyword); + _ = c.cbm_json_escape(&esc_v, esc_v.len, a.value); + return c.snprintf(buf, bufsize, "{\"i\":%d,\"k\":\"%s\",\"e\":\"%s\",\"v\":\"%s\"}", a.index, k, e, v); + } + if (a.keyword != null) { + _ = c.cbm_json_escape(&esc_k, esc_k.len, a.keyword); + return c.snprintf(buf, bufsize, "{\"i\":%d,\"k\":\"%s\",\"e\":\"%s\"}", a.index, k, e); + } + if (a.value != null) { + _ = c.cbm_json_escape(&esc_v, esc_v.len, a.value); + return c.snprintf(buf, bufsize, "{\"i\":%d,\"e\":\"%s\",\"v\":\"%s\"}", a.index, e, v); + } + return c.snprintf(buf, bufsize, "{\"i\":%d,\"e\":\"%s\"}", a.index, e); +} + +fn appendArgsJson(buf: [*]u8, bufsize: usize, pos_in: usize, call: *const c.CBMCall) usize { + var pos = pos_in; + if (call.arg_count == 0 or pos >= bufsize - PP_ARGS_MARGIN) return pos; + var n = c.snprintf(buf + pos, bufsize - pos, ",\"args\":["); + if (n <= 0) return pos; + pos += ix(n); + var i: c_int = 0; + while (i < call.arg_count and pos < bufsize - CBM_ARG_JSON_GUARD) : (i += 1) { + const a = &call.args[ix(i)]; + const mark = pos; // rollback point (before the separator) + if (i > 0 and pos < bufsize - 1) { + buf[pos] = ','; + pos += 1; + } + var expr_buf: [c.CBM_SZ_128]u8 = undefined; + sanitizeExpr(&expr_buf, a.expr); + n = formatCallArg(buf + pos, bufsize - pos, a, &expr_buf); + // snprintf returns the UNtruncated length: if the arg did not fully + // fit, advancing pos by n would push it past buf and the buf[pos] + // writes below would overflow. Drop the arg whole (atomic field — + // keeps the array valid) and stop appending. + if (n <= 0 or ix(n) >= bufsize - pos) { + pos = mark; + break; + } + pos += ix(n); + } + if (pos < bufsize - 1) { + buf[pos] = ']'; + pos += 1; + } + buf[pos] = 0; + return pos; +} + +/// Scan call args for a URL-like route path and handler reference. +fn isPathKeyword(keyword: CStr) bool { + const path_keywords = [_][*:0]const u8{ + "prefix", "path", "route", "pattern", "url", + "endpoint", "rule", "mount_path", "route_path", "url_path", + }; + for (path_keywords) |kw| { + if (streq(keyword, kw)) return true; + } + return false; +} + +fn findRoutePathInArgs(call: *const c.CBMCall, out_handler: *CStr) CStr { + out_handler.* = null; + // 1. First string arg starting with / + if (call.first_string_arg != null and call.first_string_arg[0] == '/') { + out_handler.* = call.second_arg_name; + return call.first_string_arg; + } + // 2. Keyword args (prefix=, path=, route=, etc.) + var found: CStr = null; + for (0..ix(call.arg_count)) |ai| { + if (found != null) break; + const ca = &call.args[ai]; + const val: CStr = if (ca.value != null) ca.value else ca.expr; + if (val == null or val[0] != '/') continue; + if ((ca.keyword != null and isPathKeyword(ca.keyword)) or (ca.keyword == null and ca.index == 0)) { + found = val; + } + } + if (found == null) return null; + // 3. Handler: first identifier arg that's not a path/keyword + for (0..ix(call.arg_count)) |ai| { + const ca = &call.args[ai]; + if (ca.expr == null or ca.expr[0] == '/' or ca.expr[0] == '"' or ca.expr[0] == '\'') continue; + if (ca.keyword != null and (streq(ca.keyword, "prefix") or streq(ca.keyword, "name") or + streq(ca.keyword, "tags"))) + { + continue; + } + out_handler.* = ca.expr; + break; + } + return found; +} + +const PROPS_2K = c.CBM_SZ_2K; + +/// Build props JSON, append args, close brace, emit edge. +fn finalizeAndEmit(gbuf: ?*Gbuf, src_id: i64, tgt_id: i64, edge_type: CStr, props: *[PROPS_2K]u8, n: c_int, call: *const c.CBMCall) void { + if (n > 0 and ix(n) < PROPS_2K - PP_ESC_SPACE) { + var pos = appendArgsJson(props, PROPS_2K, ix(n), call); + if (call.start_line > 0 and streq(edge_type, "CALLS") and pos < PROPS_2K - PP_LINE_MARGIN) { + const ln = c.snprintf(@as([*]u8, props) + pos, PROPS_2K - pos, ",\"line\":%d", call.start_line); + if (ln > 0) pos += ix(ln); + } + if (pos < PROPS_2K - 1) { + props[pos] = '}'; + props[pos + 1] = 0; + } + } + _ = c.cbm_gbuf_insert_edge(gbuf, src_id, tgt_id, edge_type, props); +} + +/// Build Route node QN and properties for HTTP/async service edges. +fn buildServiceRoute(gbuf: ?*Gbuf, arg: CStr, method: CStr, broker: CStr, svc: c.cbm_svc_kind_t) i64 { + var route_qn: [c.CBM_ROUTE_QN_SIZE]u8 = undefined; + var prefix: CStr = undefined; + var cpath: [c.CBM_SZ_256]u8 = undefined; + var qpath: CStr = arg; + if (svc == c.CBM_SVC_HTTP) { + prefix = if (method != null) method else "ANY"; + qpath = c.cbm_route_canon_path(arg, &cpath, cpath.len); + } else { + prefix = if (broker != null) broker else "async"; + } + _ = c.snprintf(&route_qn, route_qn.len, "__route__%s__%s", prefix, qpath); + var route_props: [c.CBM_SZ_256]u8 = undefined; + if (method != null) { + _ = c.snprintf(&route_props, route_props.len, "{\"method\":\"%s\"}", method); + } else if (broker != null) { + _ = c.snprintf(&route_props, route_props.len, "{\"broker\":\"%s\"}", broker); + } else { + _ = c.snprintf(&route_props, route_props.len, "{}"); + } + return c.cbm_gbuf_upsert_node(gbuf, "Route", arg, &route_qn, "", 0, 0, &route_props); +} + +/// Emit HTTP_CALLS or ASYNC_CALLS edge via Route node. +fn emitHttpAsyncServiceEdge(gbuf: ?*Gbuf, source: *const Node, call: *const c.CBMCall, res: *const c.cbm_resolution_t, svc: c.cbm_svc_kind_t, arg: CStr) void { + const edge_type: CStr = if (svc == c.CBM_SVC_HTTP) "HTTP_CALLS" else "ASYNC_CALLS"; + const method: CStr = + if (svc == c.CBM_SVC_HTTP) c.cbm_service_pattern_http_method(call.callee_name) else null; + const broker: CStr = + if (svc == c.CBM_SVC_ASYNC) c.cbm_service_pattern_broker(res.qualified_name) else null; + + const route_id = buildServiceRoute(gbuf, arg, method, broker, svc); + + var esc_c: [c.CBM_SZ_256]u8 = undefined; + var esc_a: [c.CBM_SZ_256]u8 = undefined; + _ = c.cbm_json_escape(&esc_c, esc_c.len, call.callee_name); + _ = c.cbm_json_escape(&esc_a, esc_a.len, arg); + var props: [PROPS_2K]u8 = undefined; + const base: [*]u8 = &props; + var n = c.snprintf(&props, props.len, "{\"callee\":\"%s\",\"url_path\":\"%s\"", @as(CStr, &esc_c), @as(CStr, &esc_a)); + if (method != null) { + n += c.snprintf(base + ix(n), props.len -% ix(n), ",\"method\":\"%s\"", method); + } + if (broker != null) { + n += c.snprintf(base + ix(n), props.len -% ix(n), ",\"broker\":\"%s\"", broker); + } + finalizeAndEmit(gbuf, source.id, route_id, edge_type, &props, n, call); +} + +/// Emit CONFIGURES edge. +fn emitConfigEdge(gbuf: ?*Gbuf, source: *const Node, target_opt: ?*const Node, call: *const c.CBMCall, res: *const c.cbm_resolution_t, arg: CStr) void { + // emit_service_edge may be reached with target==NULL on the HTTP/ASYNC + // external-client bypass (#523); a CONFIGURES edge needs a real target, so + // never deref a NULL target here. + const target = target_opt orelse return; + var esc_c: [c.CBM_SZ_256]u8 = undefined; + var esc_k: [c.CBM_SZ_256]u8 = undefined; + _ = c.cbm_json_escape(&esc_c, esc_c.len, call.callee_name); + _ = c.cbm_json_escape(&esc_k, esc_k.len, if (arg != null) arg else ""); + var props: [PROPS_2K]u8 = undefined; + const n = c.snprintf( + &props, + props.len, + "{\"callee\":\"%s\",\"key\":\"%s\",\"confidence\":%.2f", + @as(CStr, &esc_c), + @as(CStr, &esc_k), + res.confidence, + ); + finalizeAndEmit(gbuf, source.id, target.id, "CONFIGURES", &props, n, call); +} + +/// Emit normal CALLS edge. +fn emitNormalCallsEdge(gbuf: ?*Gbuf, source: *const Node, target_opt: ?*const Node, call: *const c.CBMCall, res: *const c.cbm_resolution_t) void { + // A CALLS edge needs a real target; the HTTP/ASYNC external-client bypass + // (#523) can reach emit_service_edge with target==NULL, so guard the deref. + const target = target_opt orelse return; + var esc_c: [c.CBM_SZ_256]u8 = undefined; + _ = c.cbm_json_escape(&esc_c, esc_c.len, call.callee_name); + var props: [PROPS_2K]u8 = undefined; + const n = c.snprintf( + &props, + props.len, + "{\"callee\":\"%s\",\"confidence\":%.2f,\"strategy\":\"%s\",\"candidates\":%d", + @as(CStr, &esc_c), + res.confidence, + @as(CStr, if (res.strategy != null) res.strategy else "unknown"), + res.candidate_count, + ); + finalizeAndEmit(gbuf, source.id, target.id, "CALLS", &props, n, call); +} + +// Classify a resolved call by library identity and emit the appropriate edge. +/// Create Route node + CALLS + HANDLES edges for a route registration call. +/// route_mount is the framework mount of the registering file ("/api" for a +/// Laravel 11+ `withRouting(api: ...)` file, #1146) or "". +fn emitRouteRegistration( + gbuf: ?*Gbuf, + source: *const Node, + call: *const c.CBMCall, + route_path_in: CStr, + handler_ref: CStr, + module_qn: CStr, + registry: ?*const Registry, + main_gbuf: ?*const Gbuf, + ik: StrList, + iv: StrList, + ic: c_int, + route_mount: CStr, +) void { + const method: CStr = c.cbm_service_pattern_route_method(call.callee_name); + const method_or_any: CStr = if (method != null) method else "ANY"; + var mounted: [c.CBM_SZ_256]u8 = undefined; + const route_path: CStr = c.cbm_laravel_mount_route(route_mount, route_path_in, &mounted, mounted.len); + var rqn: [c.CBM_ROUTE_QN_SIZE]u8 = undefined; + var cpath: [c.CBM_SZ_256]u8 = undefined; + _ = c.snprintf(&rqn, rqn.len, "__route__%s__%s", method_or_any, c.cbm_route_canon_path(route_path, &cpath, cpath.len)); + var rp: [c.CBM_SZ_256]u8 = undefined; + _ = c.snprintf(&rp, rp.len, "{\"method\":\"%s\"}", method_or_any); + const rid = c.cbm_gbuf_upsert_node(gbuf, "Route", route_path, &rqn, "", 0, 0, &rp); + var esc_cn: [c.CBM_SZ_256]u8 = undefined; // sliced source text: escape quotes/newlines + var esc_rp: [c.CBM_SZ_512]u8 = undefined; + _ = c.cbm_json_escape(&esc_cn, esc_cn.len, call.callee_name); + _ = c.cbm_json_escape(&esc_rp, esc_rp.len, route_path); + var props: [c.CBM_SZ_1K]u8 = undefined; + _ = c.snprintf( + &props, + props.len, + "{\"callee\":\"%s\",\"url_path\":\"%s\",\"via\":\"route_registration\"}", + @as(CStr, &esc_cn), + @as(CStr, &esc_rp), + ); + _ = c.cbm_gbuf_insert_edge(gbuf, source.id, rid, "CALLS", &props); + if (handler_ref != null and handler_ref[0] != 0) { + const hres = c.cbm_registry_resolve_handler(registry, handler_ref, module_qn, ik, iv, ic, main_gbuf); + if (hres.qualified_name != null and hres.qualified_name[0] != 0) { + const h: ?*const Node = c.cbm_gbuf_find_by_qn(main_gbuf, hres.qualified_name); + if (h) |handler| { + // must exceed escaped value + wrapper or snprintf cuts the closing brace + var hp: [c.CBM_SZ_1K]u8 = undefined; + var esc_h2: [c.CBM_SZ_512]u8 = undefined; + _ = c.cbm_json_escape(&esc_h2, esc_h2.len, hres.qualified_name); + _ = c.snprintf(&hp, hp.len, "{\"handler\":\"%s\"}", @as(CStr, &esc_h2)); + _ = c.cbm_gbuf_insert_edge(gbuf, handler.id, rid, "HANDLES", &hp); + } + } + } +} + +/// Reject regex metacharacters, spaces, double-slashes in URL candidates. +fn isJunkUrl(s: CStr) bool { + var i: usize = 0; + while (s[i] != 0) : (i += 1) { + const ch = s[i]; + if (ch == '\\' or ch == '^' or ch == '$' or ch == '*' or ch == '+' or ch == '(' or + ch == ')' or ch == '[' or ch == ']' or ch == '|' or ch == ' ') + { + return true; + } + if (ch == '/' and i > 0 and s[i - 1] == '/') return true; + } + return false; +} + +/// Normalize a template literal URL and reject junk patterns. +/// Returns true if norm contains a valid API path. +fn normalizeUrlArg(url: CStr, norm: [*]u8, norm_sz: usize) bool { + var ni: usize = 0; + var p = url; + if (p[0] == '`' or p[0] == '"' or p[0] == '\'') p += 1; + if (p[0] != '/') return false; + while (p[0] != 0 and ni < norm_sz - 2) { + if (p[0] == '$' and p[1] == '{') { + norm[ni] = ':'; + ni += 1; + p += 2; + while (p[0] != 0 and p[0] != '}' and ni < norm_sz - 2) { + norm[ni] = p[0]; + ni += 1; + p += 1; + } + if (p[0] == '}') p += 1; + } else if (p[0] == '`' or p[0] == '"' or p[0] == '\'' or p[0] == '?') { + break; + } else { + norm[ni] = p[0]; + ni += 1; + p += 1; + } + } + norm[ni] = 0; + const MIN_URL_LEN = 4; + if (ni < MIN_URL_LEN or c.strchr(norm + 1, '/') == null) return false; + return !isJunkUrl(norm); +} + +fn argIsTopic(arg: *const c.CBMCallArg) bool { + const keywords = [_][*:0]const u8{ + "topic", "topic_id", "topic_name", "queue", "queue_name", "queue_id", "subject", "channel", + }; + if (arg.keyword == null) return false; + for (keywords) |kw| { + if (streq(arg.keyword, kw)) return true; + } + return false; +} + +/// File-language scope of the client-navigation heuristic (#1250): the JS/TS +/// family and the hosts that embed it. Kept explicit so changes to the +/// weak-member guard cannot broaden it. +fn ppClientNavLang(lang: Language) bool { + return lang == c.CBM_LANG_TYPESCRIPT or lang == c.CBM_LANG_TSX or lang == c.CBM_LANG_JAVASCRIPT or + lang == c.CBM_LANG_VUE or lang == c.CBM_LANG_SVELTE or lang == c.CBM_LANG_HTML or + lang == c.CBM_LANG_ASTRO; +} + +/// Detect API paths in call arguments and create HTTP_CALLS edges. +fn detectUrlInArgs(gbuf: ?*Gbuf, source: *const Node, call: *const c.CBMCall, lang: Language, svc: c.cbm_svc_kind_t) void { + // A resolved alias may be ASYNC even when its source spelling is unknown. + // Neither raw slash topics nor URL projections are HTTP evidence there. + if (svc == c.CBM_SVC_ASYNC or c.cbm_service_pattern_match(call.callee_name) == c.CBM_SVC_ASYNC) { + return; + } + if (c.cbm_service_pattern_is_client_navigation(ppClientNavLang(lang), call.callee_name)) return; + // A bare `fetch` reaching here resolved to the project's own `fetch`, + // which shadows the global API (#856): a plain call, not HTTP. + if (c.cbm_service_pattern_is_global_fetch(call.callee_name)) return; + for (0..ix(call.arg_count)) |ai| { + const ca = &call.args[ai]; + if (argIsTopic(ca)) continue; + // A slash-prefixed raw expression is not a URL string. In JS/TS this + // is notably a regex literal (`/
= raw.len) prefix_len = raw.len - 1; + @memcpy(raw[0..prefix_len], callee[0..prefix_len]); + raw[prefix_len] = 0; + + // Strip common prefixes: pb.New, New, pb. + var s: CStr = &raw; + if (c.strncmp(s, "pb.New", 6) == 0) { + s += 6; + } else if (c.strncmp(s, "pb.", 3) == 0 or c.strncmp(s, "New", 3) == 0) { + s += 3; + } + + // Strip the generated-stub/client suffix, preserving the canonical + // proto-declared service name. The proto service is `Service`; the + // generated client is `ServiceClient` / `ServiceGrpc`, so we strip + // only the trailing stub/client token (Client/Stub/Grpc/…), NOT "Service" + // itself — stripping "ServiceClient" yielded `` and broke cross-repo + // matching against the `Service` declared name (#294). Longest tokens + // first so e.g. BlockingStub wins over Stub. + // + // A match also serves as the gRPC stub-type signal: we ONLY emit a Route + // when a recognized suffix is actually present. Without this gate the + // fallback turned ordinary receiver vars (`_provider.GetGroup`, + // `_builder.AddSomeService`) into phantom `__grpc__provider/...` Routes + // that correspond to no .proto anywhere (#294). + _ = c.snprintf(service, srv_sz, "%s", s); + const slen = c.strlen(service); + const suffixes = [_][:0]const u8{ + "BlockingStub", "FutureStub", "AsyncStub", "AsyncClient", "Servicer", "Client", "Stub", "Grpc", + }; + var stripped = false; + for (suffixes) |sfx| { + if (slen > sfx.len and streq(service + slen - sfx.len, sfx.ptr)) { + service[slen - sfx.len] = 0; + stripped = true; + break; + } + } + + return stripped and service[0] != 0 and method[0] != 0; +} + +/// Emit GRPC_CALLS edge via gRPC Route node. +fn emitGrpcEdge(gbuf: ?*Gbuf, source: *const Node, call: *const c.CBMCall, res: *const c.cbm_resolution_t) void { + var service: [c.CBM_SZ_256]u8 = undefined; + var method: [c.CBM_SZ_256]u8 = undefined; + // Try callee_name first (e.g., "pb.NewCartServiceClient.GetCart") + if (!extract_grpc_service_method(call.callee_name, &service, service.len, &method, method.len)) { + // Fallback: try the resolved QN for Go chained calls. + // Go pattern: pb.NewCartServiceClient(conn).GetCart(ctx, req) + // callee_name = "GetCart", QN = "...CartServiceClient.GetCart" + // The QN contains the full ServiceClient.Method pattern. + if (res.qualified_name == null or + !extract_grpc_service_method(res.qualified_name, &service, service.len, &method, method.len)) + { + return; + } + } + const svc_name: CStr = &service; + const method_name: CStr = &method; + + var route_qn: [c.CBM_SZ_512]u8 = undefined; + _ = c.snprintf(&route_qn, route_qn.len, "__grpc__%s/%s", svc_name, method_name); + + var route_name: [c.CBM_SZ_256]u8 = undefined; + _ = c.snprintf(&route_name, route_name.len, "%s/%s", svc_name, method_name); + + const route_id = c.cbm_gbuf_upsert_node(gbuf, "Route", &route_name, &route_qn, "", 0, 0, "{\"source\":\"grpc\"}"); + + var esc_c: [c.CBM_SZ_256]u8 = undefined; + _ = c.cbm_json_escape(&esc_c, esc_c.len, call.callee_name); + var props: [c.CBM_SZ_1K]u8 = undefined; + _ = c.snprintf( + &props, + props.len, + "{\"callee\":\"%s\",\"service\":\"%s\",\"method\":\"%s\",\"confidence\":%.2f}", + @as(CStr, &esc_c), + svc_name, + method_name, + res.confidence, + ); + _ = c.cbm_gbuf_insert_edge(gbuf, source.id, route_id, "GRPC_CALLS", &props); +} + +/// Emit GRAPHQL_CALLS edge. The Route is keyed by the operation NAME only +/// (#598), never by the operation text: +/// named operation -> __graphql__ name "" +/// anonymous query/mutation/ -> __graphql____anonymous name "(anonymous )" +/// subscription (incl. the `{ ... }` query shorthand) +/// no GraphQL document -> __graphql__operation__anonymous name "(anonymous operation)" +/// (URL, variable, missing string argument) +/// Keys are bounded by the GraphQL Name grammar and deterministic. +fn emitGraphqlEdge(gbuf: ?*Gbuf, source: *const Node, call: *const c.CBMCall, res: *const c.cbm_resolution_t) void { + var op_type: CStr = "operation"; + var op_name: [c.CBM_SZ_128]u8 = undefined; + const named = c.cbm_service_pattern_graphql_operation(call.first_string_arg, &op_type, &op_name, op_name.len); + + var route_name: [c.CBM_SZ_256]u8 = undefined; + var route_qn: [c.CBM_SZ_256]u8 = undefined; + if (named) { + _ = c.snprintf(&route_name, route_name.len, "%s", @as(CStr, &op_name)); + _ = c.snprintf(&route_qn, route_qn.len, "__graphql__%s", @as(CStr, &op_name)); + } else { + _ = c.snprintf(&route_name, route_name.len, "(anonymous %s)", op_type); + _ = c.snprintf(&route_qn, route_qn.len, "__graphql__%s__anonymous", op_type); + } + + const route_id = c.cbm_gbuf_upsert_node(gbuf, "Route", &route_name, &route_qn, "", 0, 0, "{\"source\":\"graphql\"}"); + + var esc_c: [c.CBM_SZ_256]u8 = undefined; + var esc_op: [c.CBM_SZ_512]u8 = undefined; + _ = c.cbm_json_escape(&esc_c, esc_c.len, call.callee_name); + _ = c.cbm_json_escape(&esc_op, esc_op.len, &route_name); + var props: [c.CBM_SZ_1K]u8 = undefined; + _ = c.snprintf( + &props, + props.len, + "{\"callee\":\"%s\",\"operation\":\"%s\",\"operation_type\":\"%s\"," ++ + "\"confidence\":%.2f}", + @as(CStr, &esc_c), + @as(CStr, &esc_op), + op_type, + res.confidence, + ); + _ = c.cbm_gbuf_insert_edge(gbuf, source.id, route_id, "GRAPHQL_CALLS", &props); +} + +/// Emit TRPC_CALLS edge. Extract procedure path from callee chain. +fn emitTrpcEdge(gbuf: ?*Gbuf, source: *const Node, call: *const c.CBMCall, res: *const c.cbm_resolution_t) void { + // tRPC calls: trpc.user.getById.query() → extract "user.getById" + const callee: CStr = call.callee_name; + if (callee == null) return; + // Strip trailing .query/.mutate/.subscribe + var proc: [c.CBM_SZ_256]u8 = undefined; + _ = c.snprintf(&proc, proc.len, "%s", callee); + const last_dot: MutStr = c.strrchr(&proc, '.'); + if (last_dot != null and (streq(last_dot, ".query") or streq(last_dot, ".mutate") or + streq(last_dot, ".subscribe") or streq(last_dot, ".useQuery") or + streq(last_dot, ".useMutation"))) + { + last_dot[0] = 0; + } + // Strip leading trpc. + var p: CStr = &proc; + if (c.strncmp(p, "trpc.", 5) == 0) p += 5; + + var route_qn: [c.CBM_SZ_512]u8 = undefined; + _ = c.snprintf(&route_qn, route_qn.len, "__trpc__%s", p); + + const route_id = c.cbm_gbuf_upsert_node(gbuf, "Route", p, &route_qn, "", 0, 0, "{\"source\":\"trpc\"}"); + + var esc_c: [c.CBM_SZ_256]u8 = undefined; + _ = c.cbm_json_escape(&esc_c, esc_c.len, call.callee_name); + var props: [c.CBM_SZ_1K]u8 = undefined; + _ = c.snprintf( + &props, + props.len, + "{\"callee\":\"%s\",\"procedure\":\"%s\",\"confidence\":%.2f}", + @as(CStr, &esc_c), + p, + res.confidence, + ); + _ = c.cbm_gbuf_insert_edge(gbuf, source.id, route_id, "TRPC_CALLS", &props); +} + +/// When suppress_plain_calls is true (a TS/JS/TSX weak short-name member-call +/// match, #592/#606), every service classification below still runs — only the +/// plain CALLS fall-through (emitNormalCallsEdge) is skipped. detectUrlInArgs +/// and the HTTP/ASYNC/gRPC/GraphQL/tRPC/CONFIG/route branches are unaffected, so +/// a verb-suffix HTTP client (api.patch('/x')), broker, or route registration +/// keeps its edge; only the fabricated project CALLS edge is dropped. +fn emitServiceEdge( + gbuf: ?*Gbuf, + source: *const Node, + target: ?*const Node, + call: *const c.CBMCall, + res: *const c.cbm_resolution_t, + module_qn: CStr, + registry: ?*const Registry, + main_gbuf: ?*const Gbuf, + imp_keys: StrList, + imp_vals: StrList, + imp_count: c_int, + suppress_plain_calls: bool, + lang: Language, + route_mount: CStr, +) void { + var svc = c.cbm_service_pattern_match(res.qualified_name); + const arg: CStr = call.first_string_arg; + + // Also detect route registration by callee name suffix alone (handles unresolved + // local variables like app.include_router where QN resolution fails). + if (svc == c.CBM_SVC_NONE and c.cbm_service_pattern_route_method(call.callee_name) != null) { + svc = c.CBM_SVC_ROUTE_REG; + } + + // Detect gRPC stub method calls by resolved QN. + // Go pattern: pb.NewCartServiceClient(conn).GetCart(ctx, req) + // Tree-sitter extracts GetCart as the callee, which resolves to the + // generated pb interface method (QN contains "ServiceClient"). + if (svc == c.CBM_SVC_NONE and res.qualified_name != null) { + if (c.strstr(res.qualified_name, "ServiceClient") != null or + c.strstr(res.qualified_name, "ServiceGrpc") != null or + c.strstr(res.qualified_name, "Servicer") != null) + { + svc = c.CBM_SVC_GRPC; + } + } + + if (svc == c.CBM_SVC_ROUTE_REG) { + var handler_ref: CStr = null; + const route_path = findRoutePathInArgs(call, &handler_ref); + if (route_path != null) { + emitRouteRegistration(gbuf, source, call, route_path, handler_ref, module_qn, registry, main_gbuf, imp_keys, imp_vals, imp_count, route_mount); + return; + } + // No path found — fall through to normal CALLS edge + } + + const has_arg = arg != null and arg[0] != 0; + const has_url = has_arg and (arg[0] == '/' or c.strstr(arg, "://") != null); + const has_topic = has_arg and svc == c.CBM_SVC_ASYNC and c.strlen(arg) > PP_ESC_SPACE; + // Set when this call already has its HTTP_CALLS edge from the service + // patterns (typed with the verb its callee names). + var http_edge_emitted = false; + + if ((svc == c.CBM_SVC_HTTP or svc == c.CBM_SVC_ASYNC) and (has_url or has_topic)) { + emitHttpAsyncServiceEdge(gbuf, source, call, res, svc, arg); + http_edge_emitted = svc == c.CBM_SVC_HTTP; + } else if (svc == c.CBM_SVC_GRPC) { + emitGrpcEdge(gbuf, source, call, res); + } else if (svc == c.CBM_SVC_GRAPHQL) { + emitGraphqlEdge(gbuf, source, call, res); + } else if (svc == c.CBM_SVC_TRPC) { + emitTrpcEdge(gbuf, source, call, res); + } else if (svc == c.CBM_SVC_CONFIG) { + emitConfigEdge(gbuf, source, target, call, res, arg); + } else if (!suppress_plain_calls) { + emitNormalCallsEdge(gbuf, source, target, call, res); + } + + // The arg-URL heuristic is for calls no service pattern knows (a local + // fetch wrapper). On a classified HTTP call that already has its typed + // edge, skip it so we do not mint a second, method-less ANY edge. + if (!http_edge_emitted) detectUrlInArgs(gbuf, source, call, lang, svc); +} + +/// The #725 guard refuses a suffix_match binding across a language boundary, +/// which leaves the callee unresolved. An unresolved callee the route +/// classifier recognises is still a registration: `Route::get('/x', ...)` +/// beside a JS `get`, `app.get('/x', h)` beside a Python `get`. Emit exactly +/// the Route + CALLS + HANDLES that the unresolved-callee (callee_suffix) +/// fallback in resolveFileCalls emits for it, classified by callee name the +/// same way emitServiceEdge classifies that fallback, and nothing else: no +/// CALLS edge to the refused target and no URL-argument scan. Dropping the +/// whole call lost every GET registration in a mixed-language repo while POST +/// (no `post` to collide with) survived. +fn emitXlangRefusedRoute( + gbuf: ?*Gbuf, + source: *const Node, + call: *const c.CBMCall, + module_qn: CStr, + registry: ?*const Registry, + main_gbuf: ?*const Gbuf, + imp_keys: StrList, + imp_vals: StrList, + imp_count: c_int, + route_mount: CStr, +) void { + if (c.cbm_service_pattern_route_method(call.callee_name) == null) return; + const svc = c.cbm_service_pattern_match(call.callee_name); + if (svc != c.CBM_SVC_NONE and svc != c.CBM_SVC_ROUTE_REG) return; + var handler_ref: CStr = null; + const route_path = findRoutePathInArgs(call, &handler_ref); + if (route_path != null) { + emitRouteRegistration(gbuf, source, call, route_path, handler_ref, module_qn, registry, main_gbuf, imp_keys, imp_vals, imp_count, route_mount); + } +} + +/// Find the source node for an edge: enclosing function or file node. +fn findSourceNode(gbuf: ?*const Gbuf, project: CStr, rel: CStr, enclosing_qn: CStr) ?*const Node { + var src: ?*const Node = null; + if (enclosing_qn != null) { + src = c.cbm_gbuf_find_by_qn(gbuf, enclosing_qn); + // A class-level reference in a directory-module language carries the + // DIRECTORY module QN, which hits the shared Folder/Project node — + // attribute to this file's File node instead (#787). + if (c.cbm_pipeline_node_is_dir_container(src)) src = null; + } + if (src == null) { + const file_qn: MutStr = c.cbm_pipeline_fqn_compute(project, rel, "__file__"); + src = c.cbm_gbuf_find_by_qn(gbuf, file_qn); + c.free(file_qn); + } + return src; +} + +/// Field type hint resolution for obj.Method() with multiple candidates. +/// Strips C# field prefixes (_ / m_), capitalizes to get type name, and +/// checks if TypeName.Method or ITypeName.Method exists among candidates. +fn tryFieldTypeHint(rc: *ResolveCtx, res: *c.cbm_resolution_t, callee_name: CStr, source_id: i64) void { + if (res.qualified_name == null or res.candidate_count <= 1) return; + const dot: CStr = c.strchr(callee_name, '.'); + if (dot == null) return; + const plen = ptrDiff(dot, callee_name); + var obj_name: [c.CBM_SZ_256]u8 = undefined; + if (plen >= obj_name.len) return; + @memcpy(obj_name[0..plen], callee_name[0..plen]); + obj_name[plen] = 0; + + var type_hint: CStr = &obj_name; + if (type_hint[0] == '_') type_hint += 1; + if (type_hint[0] == 'm' and type_hint[1] == '_') type_hint += PP_CSHARP_M_PREFIX_LEN; + + var type_name: [c.CBM_SZ_256]u8 = undefined; + _ = c.snprintf(&type_name, type_name.len, "%s", type_hint); + if (type_name[0] >= 'a' and type_name[0] <= 'z') type_name[0] -= 'a' - 'A'; + + var iface_name: [c.CBM_SZ_256]u8 = undefined; + _ = c.snprintf(&iface_name, iface_name.len, "I%s", @as(CStr, &type_name)); + + const method: CStr = dot + 1; + var cands: StrList = null; + var cand_count: c_int = 0; + _ = c.cbm_registry_find_by_name(rc.registry, method, &cands, &cand_count); + for (0..ix(cand_count)) |ci| { + if (c.strstr(cands[ci], &type_name) != null or c.strstr(cands[ci], &iface_name) != null) { + const better: ?*const Node = c.cbm_gbuf_find_by_qn(rc.main_gbuf, cands[ci]); + if (better != null and better.?.id != source_id) { + res.qualified_name = cands[ci]; + res.confidence = PP_FIELD_HINT_CONF; + res.strategy = "field_type_hint"; + return; + } + } + } +} + +/// Free a strdup'd key stored in the per-file lsp_idx hash table. +fn lspIdxFreeKey(key: CStr, value: ?*anyopaque, ud: ?*anyopaque) callconv(.c) void { + _ = value; + _ = ud; + c.free(@constCast(key)); +} + +/// Adds the time since `t0` to a per-phase counter. +inline fn addElapsed(counter: *AtomicU64, t0: u64) void { + _ = counter.fetchAdd(extractNowNs() -% t0, .monotonic); +} + +/// A resolution that stands in for a call classified by its callee name. +fn nameResolution(name: CStr, strategy: CStr) c.cbm_resolution_t { + var res = std.mem.zeroes(c.cbm_resolution_t); + res.qualified_name = name; + res.confidence = PP_HALF_CONF; + res.strategy = strategy; + return res; +} + +/// A non-empty string argument that is a URL path or an absolute URL. +fn isUrlArg(u: CStr) bool { + return u != null and u[0] != 0 and (u[0] == '/' or c.strstr(u, "://") != null); +} + +/// isUrlArg, or for an async service any topic longer than two bytes. +fn isUrlOrTopicArg(u: CStr, svc: c.cbm_svc_kind_t) bool { + return u != null and u[0] != 0 and + (u[0] == '/' or c.strstr(u, "://") != null or + (svc == c.CBM_SVC_ASYNC and c.strlen(u) > PP_ESC_SPACE)); +} + +/// Resolve calls for one file and emit CALLS/HTTP_CALLS/ASYNC_CALLS edges. +fn resolveFileCalls( + rc: *ResolveCtx, + ws: *ResolveWorkerState, + result: *FileResult, + rel: CStr, + module_qn: CStr, + imp_keys: StrList, + imp_vals: StrList, + imp_count: c_int, + lang: Language, +) void { + // Framework mount of this file's routes (Laravel 11+ withRouting, #1146). + var route_mount: [c.CBM_SZ_128]u8 = undefined; + _ = c.cbm_laravel_file_route_mount(rc.repo_path, rel, lang, result, &route_mount, route_mount.len); + + // Build a per-file hash index of resolved_calls keyed by + // "caller_qn|callee_short" for O(1) lookup. cbm_pipeline_find_lsp_ + // resolution would otherwise do an O(N) linear scan over + // resolved_calls for EACH of result->calls.count calls — the + // dominant cost in parallel_resolve on kubernetes (~50s of pure + // scanning). On insert, keep the highest-confidence entry per key + // (matches the original "best" tie-break). Skip the build entirely + // when there are no calls (nothing to look up) or no resolved + // entries (lookups would all miss). + var lsp_idx: ?*HashTable = null; + if (result.calls.count > 0 and result.resolved_calls.count > 0) { + lsp_idx = c.cbm_ht_create(@as(u32, @bitCast(result.resolved_calls.count)) *% 2 +% 16); + if (lsp_idx != null) { + for (0..ix(result.resolved_calls.count)) |i| { + const rc_e = util.at(result.resolved_calls.items, i); + if (rc_e.caller_qn == null or rc_e.callee_qn == null or + rc_e.kind != c.CBM_RESOLVED_INVOCATION or + rc_e.confidence < c.CBM_LSP_CONFIDENCE_FLOOR) + { + continue; + } + var short_name: CStr = c.strrchr(rc_e.callee_qn, '.'); + short_name = if (short_name != null) short_name + 1 else rc_e.callee_qn; + var key: [1024]u8 = undefined; + // A row with a site span proves one call site: key it by that + // site so it never answers a same-named call elsewhere in the + // caller (cbm_pipeline_lsp_site_admits). + const kn = if (rc_e.site_end_byte > rc_e.site_start_byte) + c.snprintf(&key, key.len, "%s|%s@%u", rc_e.caller_qn, short_name, rc_e.site_start_byte) + else + c.snprintf(&key, key.len, "%s|%s", rc_e.caller_qn, short_name); + if (kn <= 0 or kn >= @as(c_int, key.len)) continue; + const existing: ?*c.CBMResolvedCall = @ptrCast(@alignCast(c.cbm_ht_get(lsp_idx, &key))); + if (existing == null) { + // New entry — strdup so the key outlives the loop body. + const kdup: MutStr = c.strdup(&key); + if (kdup != null) _ = c.cbm_ht_set(lsp_idx, kdup, rc_e); + } else if (rc_e.confidence > existing.?.confidence) { + // Update value; reuse stored key pointer to avoid leak. + const skey: CStr = c.cbm_ht_get_key(lsp_idx, &key); + if (skey != null) _ = c.cbm_ht_set(lsp_idx, skey, rc_e); + } + } + } + } + + for (0..ix(result.calls.count)) |ci| { + const call = util.at(result.calls.items, ci); + if (call.callee_name == null) continue; + var t0 = extractNowNs(); + const source_node_opt = findSourceNode(rc.main_gbuf, rc.project_name, rel, call.enclosing_func_qn); + addElapsed(&rc.time_ns_rc_source, t0); + const source_node = source_node_opt orelse continue; + + // LSP-resolved calls take precedence over registry textual matching. + // The shared helper and CBM_LSP_CONFIDENCE_FLOOR (lsp_resolve.h) + // decide which LSP overrides are admitted. Unique-tail fallbacks + // are JVM-only (see cbm_pipeline_lsp_allow_tail_match). + const allow_tail = c.cbm_pipeline_lsp_allow_tail_match(lang); + var res = std.mem.zeroes(c.cbm_resolution_t); + var lsp: ?*const c.CBMResolvedCall = null; + t0 = extractNowNs(); + if (lsp_idx != null and call.enclosing_func_qn != null) { + const call_leaf: CStr = c.cbm_pipeline_call_callee_leaf(call.callee_name); + var key: [1024]u8 = undefined; + var kn: c_int = if (call_leaf != null) + c.snprintf(&key, key.len, "%s|%s@%u", call.enclosing_func_qn, call_leaf, call.site_start_byte) + else + -1; + if (kn > 0 and kn < @as(c_int, key.len)) { + lsp = @ptrCast(@alignCast(c.cbm_ht_get(lsp_idx, &key))); + } + kn = if (call_leaf != null and lsp == null) + c.snprintf(&key, key.len, "%s|%s", call.enclosing_func_qn, call_leaf) + else + -1; + if (kn > 0 and kn < @as(c_int, key.len)) { + lsp = @ptrCast(@alignCast(c.cbm_ht_get(lsp_idx, &key))); + } + } + if (lsp == null) { + // Fallback to the linear scan for edge cases the index may + // miss (e.g. callee_name that wasn't the registered short + // name). Keeps semantics identical. + lsp = c.cbm_pipeline_find_lsp_resolution(&result.resolved_calls, call, allow_tail); + } + addElapsed(&rc.time_ns_rc_lsp_lookup, t0); + t0 = extractNowNs(); + var lsp_target: ?*const Node = null; + if (lsp) |l| { + // Canonicalise to the gbuf node's QN so res.qualified_name matches + // the gbuf even when the cross-file fallback had to prefix the + // project name. + lsp_target = c.cbm_pipeline_lsp_target_node(rc.main_gbuf, rc.project_name, l.callee_qn, allow_tail); + if (lsp_target) |target| { + res.qualified_name = target.qualified_name; + res.strategy = if (l.strategy != null) l.strategy else "lsp_override"; + res.confidence = l.confidence; + res.candidate_count = 1; + ws.lsp_overrides += 1; + } + } + // #1085: fall back to the registry resolver whenever the LSP did not + // yield a gbuf-resolvable target — whether no LSP resolution existed, + // OR the LSP was confident but its callee_qn isn't a node in the gbuf + // (the JSX-via-tsconfig-alias case: the TS LSP resolves the element + // ref to an alias-path QN that never matches a def node, so lsp_target + // is NULL). The old `else` ran the registry ONLY when lsp was null, so + // an LSP-with-unresolvable-target dropped the edge outright — silently + // losing every alias-imported JSX component edge (~21% of a Next.js + // call graph). Falling THROUGH to the registry keeps them via the + // import_map / unique_name resolution. + if (res.qualified_name == null or res.qualified_name[0] == 0) { + res = c.cbm_registry_resolve(rc.registry, call.callee_name, module_qn, imp_keys, imp_vals, imp_count); + } + addElapsed(&rc.time_ns_rc_resolve, t0); + + t0 = extractNowNs(); + tryFieldTypeHint(rc, &res, call.callee_name, source_node.id); + addElapsed(&rc.time_ns_rc_hint, t0); + + // Perl call-graph noise guard (#476). Perl has no LSP resolver; for + // builtins (push/shift/ + // keys/...) and method calls ($obj->m, unresolved receiver), suppress + // only WEAK cross-file short-name matches and keep the high-confidence + // same_module / import_map strategies so a genuine same-file or + // imported call to a builtin-named sub still resolves. Placed after the + // field-type hint so a hint cannot re-introduce a suppressed edge. + // Gated to Perl — other languages are unaffected. + if (c.cbm_perl_suppress_generic_match(lang == c.CBM_LANG_PERL, call.is_method, call.callee_name, res.strategy)) { + continue; + } + + // Dynamic-language weak-member suppression (#592/#606/#1276). The + // receiver-aware guard must NOT drop this call here: doing so would also + // skip the #523 callee-name service bypass below, emit_service_edge's + // route/gRPC/config branches, and its unconditional detect_url_in_args + // (which classifies verb-suffix HTTP clients like api.patch('/x')). + // Instead, defer to the emit path and suppress ONLY the plain-CALLS + // fall-through (emit_normal_calls_edge), so every service edge stays + // main-identical by construction. res.strategy may carry an lsp_* value + // here (LSP-resolved calls keep res through this point); the helper's + // EXPLICIT drop-list leaves lsp_ts_method / lsp_cross untouched. See + // #606 direction. + const suppress_weak_member = lang == c.CBM_LANG_PYTHON or lang == c.CBM_LANG_JAVASCRIPT or + lang == c.CBM_LANG_TYPESCRIPT or lang == c.CBM_LANG_TSX or lang == c.CBM_LANG_ZIG; + // Bare-call local-binding suppression. A member call has a receiver the + // guard above can reason about; a bare `run()` has none, so that guard + // cannot see this class at all. Python-only today because the + // extraction flag is set only for Python. + const suppress_weak_local_binding = lang == c.CBM_LANG_PYTHON; + const drop_plain_call = + c.cbm_suppress_weak_member_match(suppress_weak_member, call.is_method, res.strategy) or + c.cbm_suppress_zig_qualified_member(lang == c.CBM_LANG_ZIG, call.is_method, res.strategy) or + c.cbm_suppress_weak_local_binding_call(suppress_weak_local_binding, call.callee_is_locally_bound, res.strategy); + + // First: a call on an axios.create() instance (#1916) — decided by the + // receiver's binding, before the #523 spelling check and the + // route-registration suffix fallback. + var client_url: [c.CBM_SZ_512]u8 = undefined; + if (c.cbm_pipeline_http_client_call_url(rc.main_gbuf, rc.project_name, rel, result, imp_keys, imp_vals, imp_count, call, &client_url, client_url.len)) { + const svc_res = nameResolution(call.callee_name, "http_client_instance"); + emitHttpAsyncServiceEdge(ws.local_edge_buf, source_node, call, &svc_res, c.CBM_SVC_HTTP, &client_url); + continue; + } + // Service-pattern HTTP/ASYNC client call (`requests.get(url)`): the + // service signal lives in the callee_name. The registry can mis-resolve + // it to a spurious builtin short-name match (`requests.get` -> + // `builtins.dict.get` via "get"), which is non-empty and not an HTTP + // pattern, so the resolved-QN service checks below miss it and the call + // is dropped. Detect it on the callee_name FIRST so the HTTP_CALLS/ + // ASYNC_CALLS edge is emitted regardless (target is a synthesized route + // node, not the unindexed library). (#523) + const csvc = c.cbm_service_pattern_match(call.callee_name); + if (csvc == c.CBM_SVC_HTTP or csvc == c.CBM_SVC_ASYNC) { + if (isUrlOrTopicArg(call.first_string_arg, csvc)) { + const svc_res = nameResolution(call.callee_name, "service_pattern"); + emitServiceEdge(ws.local_edge_buf, source_node, source_node, call, &svc_res, module_qn, rc.registry, rc.main_gbuf, imp_keys, imp_vals, imp_count, false, lang, &route_mount); + continue; + } + } + + const unresolved = res.qualified_name == null or res.qualified_name[0] == 0; + if (unresolved or c.cbm_registry_strategy_is_weak(res.strategy)) { + // A callee an import binds to an external client library + // (`send` under `from paho.mqtt import publish as send`): + // spelled through its import, then classified. The binding + // outranks a weak name match. + var spelled: [c.CBM_SZ_512]u8 = undefined; + const isvc = c.cbm_pipeline_external_import_service(lang, &result.imports, imp_keys, imp_count, call, &spelled, spelled.len); + if (isvc != c.CBM_SVC_NONE) { + var through = call.*; + through.callee_name = &spelled; + const imp_res = nameResolution(&spelled, "import_binding"); + emitHttpAsyncServiceEdge(ws.local_edge_buf, source_node, &through, &imp_res, isvc, through.first_string_arg); + ws.calls_resolved += 1; + continue; + } + } + if (unresolved) { + if (c.cbm_service_pattern_route_method(call.callee_name) != null) { + const fake_res = nameResolution(call.callee_name, "callee_suffix"); + // The caller stands in for the unknown target, so a plain + // CALLS fall-through would be a self-loop. + emitServiceEdge(ws.local_edge_buf, source_node, source_node, call, &fake_res, module_qn, rc.registry, rc.main_gbuf, imp_keys, imp_vals, imp_count, true, lang, &route_mount); + } else if (c.cbm_service_pattern_is_global_fetch(call.callee_name)) { + // Native `fetch()` (#856): only the global API once resolution + // has failed to find a local/imported `fetch`. Call the low-level + // emitter directly — emit_service_edge re-derives its own kind + // from res->qualified_name via cbm_service_pattern_match, which + // "fetch" deliberately never matches. + const u: CStr = call.first_string_arg; + if (isUrlArg(u)) { + const fake_res = nameResolution(call.callee_name, "service_pattern"); + emitHttpAsyncServiceEdge(ws.local_edge_buf, source_node, call, &fake_res, c.CBM_SVC_HTTP, u); + } + } + continue; + } + // Reuse lsp_target as target_node when LSP resolved — avoids a + // second cbm_gbuf_find_by_qn lookup. try_field_type_hint may have + // upgraded res.qualified_name to a different candidate, in which + // case we must re-resolve. + t0 = extractNowNs(); + const target_node: ?*const Node = + if (lsp_target != null and res.qualified_name == lsp_target.?.qualified_name) + lsp_target + else + c.cbm_gbuf_find_by_qn(rc.main_gbuf, res.qualified_name); + addElapsed(&rc.time_ns_rc_target, t0); + if (target_node != null and source_node.id != target_node.?.id and + c.cbm_suppress_cross_language_suffix_match(lang, target_node.?.file_path, res.strategy)) + { + // #725: do not emit a suffix_match + // CALLS edge across a language boundary. A route registration + // behind the refused binding still gets its Route. + emitXlangRefusedRoute(ws.local_edge_buf, source_node, call, module_qn, rc.registry, rc.main_gbuf, imp_keys, imp_vals, imp_count, &route_mount); + continue; + } + if (target_node == null or source_node.id == target_node.?.id) { + // HTTP/ASYNC calls to an EXTERNAL client library (`requests.get(url)`) + // resolve to an unindexed QN (target_node == NULL), but their edge + // target is a synthesized route node, not the library — emit them + // anyway so cross-repo matching has an HTTP_CALLS edge to work with + // (#523). + const psvc = c.cbm_service_pattern_match(res.qualified_name); + if ((psvc == c.CBM_SVC_HTTP or psvc == c.CBM_SVC_ASYNC) and target_node == null) { + if (isUrlOrTopicArg(call.first_string_arg, psvc)) { + emitServiceEdge(ws.local_edge_buf, source_node, null, call, &res, module_qn, rc.registry, rc.main_gbuf, imp_keys, imp_vals, imp_count, false, lang, &route_mount); + ws.calls_resolved += 1; + } + } + continue; + } + t0 = extractNowNs(); + emitServiceEdge(ws.local_edge_buf, source_node, target_node, call, &res, module_qn, rc.registry, rc.main_gbuf, imp_keys, imp_vals, imp_count, drop_plain_call, lang, &route_mount); + addElapsed(&rc.time_ns_rc_emit, t0); + ws.calls_resolved += 1; + } + if (lsp_idx != null) { + c.cbm_ht_foreach(lsp_idx, lspIdxFreeKey, null); + c.cbm_ht_free(lsp_idx); + } +} + +/// Resolve USAGE and CALL_REFERENCE edges for one file. CALL_REFERENCE only +/// through the exact LSP join, never from the textual registry. +fn resolveFileUsages( + rc: *ResolveCtx, + ws: *ResolveWorkerState, + result: *FileResult, + rel: CStr, + module_qn: CStr, + imp_keys: StrList, + imp_vals: StrList, + imp_count: c_int, + lang: Language, +) void { + var ref_index: c.cbm_pipeline_ref_index_t = undefined; + _ = c.cbm_pipeline_ref_index_build(&result.resolved_calls, &ref_index); + const allow_tail = c.cbm_pipeline_lsp_allow_tail_match(lang); + for (0..ix(result.usages.count)) |u| { + const usage = util.at(result.usages.items, u); + if (usage.ref_name == null) continue; + const src = findSourceNode(rc.main_gbuf, rc.project_name, rel, usage.enclosing_func_qn) orelse continue; + var tgt: ?*const Node = null; + var row: ?*const c.CBMResolvedCall = null; + const join = c.cbm_pipeline_join_lsp_reference(rc.main_gbuf, rc.project_name, &ref_index, usage, allow_tail, @ptrCast(&tgt), @ptrCast(&row)); + if (join == c.CBM_REF_JOIN_SKIP or (join == c.CBM_REF_JOIN_VALUE and usage.reference_only)) { + continue; + } + if (join == c.CBM_REF_JOIN_NONE) { + const res = c.cbm_registry_resolve(rc.registry, usage.ref_name, module_qn, imp_keys, imp_vals, imp_count); + if (res.qualified_name == null or res.qualified_name[0] == 0) continue; + if (!c.cbm_pipeline_reference_fallback_allowed(lang, usage, res.strategy, imp_keys, imp_count)) { + continue; + } + tgt = c.cbm_gbuf_find_by_qn(rc.main_gbuf, res.qualified_name); + // #1928: the registry fallback is a bare-name guess — never let it + // bind a reference across a language boundary. + if (tgt != null and c.cbm_suppress_cross_language_ref(lang, tgt.?.file_path)) continue; + // #1942/#1962: a bare Go reference can never denote a struct + // field; the member half of a selector may. + if (tgt != null and c.cbm_go_suppress_bare_field_ref(lang == c.CBM_LANG_GO, usage.is_member_access, tgt.?.label)) { + continue; + } + } + const target = tgt orelse continue; + if (src.id == target.id) continue; + var uprops: [c.CBM_SZ_512]u8 = undefined; + var esc_ref: [c.CBM_SZ_256]u8 = undefined; // sliced source text: escape quotes/newlines + _ = c.cbm_json_escape(&esc_ref, esc_ref.len, usage.ref_name); + const callable = join == c.CBM_REF_JOIN_CALLABLE; + _ = c.cbm_pipeline_reference_props(&uprops, uprops.len, &esc_ref, if (callable) row else null); + _ = c.cbm_gbuf_insert_edge(ws.local_edge_buf, src.id, target.id, if (callable) "CALL_REFERENCE" else "USAGE", &uprops); + ws.usages_resolved += 1; + } + c.cbm_pipeline_ref_index_free(&ref_index); +} + +/// Resolve throws/raises for one file. +fn resolveFileThrows( + rc: *ResolveCtx, + ws: *ResolveWorkerState, + result: *FileResult, + module_qn: CStr, + imp_keys: StrList, + imp_vals: StrList, + imp_count: c_int, +) void { + for (0..ix(result.throws.count)) |t| { + const thr = util.at(result.throws.items, t); + if (thr.exception_name == null or thr.enclosing_func_qn == null) continue; + const src: *const Node = c.cbm_gbuf_find_by_qn(rc.main_gbuf, thr.enclosing_func_qn) orelse continue; + const edge_type: CStr = if (isCheckedException(thr.exception_name)) "THROWS" else "RAISES"; + const res = c.cbm_registry_resolve(rc.registry, thr.exception_name, module_qn, imp_keys, imp_vals, imp_count); + if (res.qualified_name == null or res.qualified_name[0] == 0) continue; + const tgt: *const Node = c.cbm_gbuf_find_by_qn(rc.main_gbuf, res.qualified_name) orelse continue; + if (src.id == tgt.id) continue; + _ = c.cbm_gbuf_insert_edge(ws.local_edge_buf, src.id, tgt.id, edge_type, "{}"); + } +} + +/// Resolve reads/writes for one file. +fn resolveFileRw( + rc: *ResolveCtx, + ws: *ResolveWorkerState, + result: *FileResult, + rel: CStr, + module_qn: CStr, + imp_keys: StrList, + imp_vals: StrList, + imp_count: c_int, + lang: Language, +) void { + for (0..ix(result.rw.count)) |r| { + const rw = util.at(result.rw.items, r); + if (rw.var_name == null) continue; + const src = findSourceNode(rc.main_gbuf, rc.project_name, rel, rw.enclosing_func_qn) orelse continue; + const res = c.cbm_registry_resolve(rc.registry, rw.var_name, module_qn, imp_keys, imp_vals, imp_count); + if (res.qualified_name == null or res.qualified_name[0] == 0) continue; + const tgt: *const Node = c.cbm_gbuf_find_by_qn(rc.main_gbuf, res.qualified_name) orelse continue; + if (src.id == tgt.id) continue; + // #1928: every resolution here is a bare-name registry guess — never + // let it bind a read/write across a language boundary. + if (c.cbm_suppress_cross_language_ref(lang, tgt.file_path)) continue; + // #1942/#1962: a bare Go reference can never denote a struct field; + // a selector-LHS write (`t.err = x`) may bind it. + if (c.cbm_go_suppress_bare_field_ref(lang == c.CBM_LANG_GO, rw.is_member_access, tgt.label)) continue; + const etype: CStr = if (rw.is_write) "WRITES" else "READS"; + _ = c.cbm_gbuf_insert_edge(ws.local_edge_buf, src.id, tgt.id, etype, "{}"); + } +} + +/// Resolve base_classes → INHERITS / IMPLEMENTS edges for one definition. +fn resolveDefInherits(rc: *ResolveCtx, ws: *ResolveWorkerState, def: *const Definition, node: *const Node, mq: CStr, ik: StrList, iv: StrList, ic: c_int) void { + if (def.base_classes == null) return; + var b: usize = 0; + while (def.base_classes[b] != null) : (b += 1) { + const bqn = resolveAsClass(rc.registry, def.base_classes[b], mq, ik, iv, ic); + if (bqn == null) continue; + const bn: ?*const Node = c.cbm_gbuf_find_by_qn(rc.main_gbuf, bqn); + if (bn != null and node.id != bn.?.id) { + // A base that resolves to an Interface is an IMPLEMENTS relation + // (Java `implements`, C# `: IFace`, TS `implements`); a Class/ + // Type/Enum base is plain INHERITS. + const base_label: CStr = c.cbm_registry_label_of(rc.registry, bqn); + const edge_type: CStr = + if (base_label != null and streq(base_label, "Interface")) "IMPLEMENTS" else "INHERITS"; + _ = c.cbm_gbuf_insert_edge(ws.local_edge_buf, node.id, bn.?.id, edge_type, "{}"); + ws.semantic_resolved += 1; + } + } +} + +/// Resolve decorators → DECORATES edges for one definition. +fn resolveDefDecorators(rc: *ResolveCtx, ws: *ResolveWorkerState, def: *const Definition, node: *const Node, mq: CStr, ik: StrList, iv: StrList, ic: c_int) void { + if (def.decorators == null) return; + var dc: usize = 0; + while (def.decorators[dc] != null) : (dc += 1) { + var func: [c.CBM_SZ_256]u8 = undefined; + extractDecoratorFunc(def.decorators[dc], &func, func.len); + if (func[0] == 0) continue; + const func_name: CStr = &func; + var res = c.cbm_registry_resolve(rc.registry, func_name, mq, ik, iv, ic); + if ((res.qualified_name == null or res.qualified_name[0] == 0) and c.strchr(func_name, '.') == null) { + // C# attributes are referenced by their short name (`[Log]`) but + // declared with an `Attribute` suffix (`class LogAttribute`). + var with_suffix: [c.CBM_SZ_256]u8 = undefined; + const wn = c.snprintf(&with_suffix, with_suffix.len, "%sAttribute", func_name); + if (wn > 0 and ix(wn) < with_suffix.len) { + res = c.cbm_registry_resolve(rc.registry, &with_suffix, mq, ik, iv, ic); + } + } + var dn: ?*const Node = null; + if (res.qualified_name != null and res.qualified_name[0] != 0) { + dn = c.cbm_gbuf_find_by_qn(rc.main_gbuf, res.qualified_name); + } + var dn_id: i64 = 0; + if (dn) |decorator| { + dn_id = decorator.id; + } else { + // External/stdlib decorator (Rust `#[derive(Debug)]`, Swift + // `@discardableResult`, Scala `@deprecated`, Python `@cache`, + // Java `@Override`, ...): no local symbol resolves. Materialise a + // synthetic "Decorator" node so the DECORATES relation is recorded. + // The node is created in the per-worker local_edge_buf (shared-ID + // gbuf); the sequential merge dedupes by QN across workers, so all + // uses of the same decorator name collapse to one node project-wide + // and the edge target IDs are remapped consistently. + var syn_qn: [c.CBM_SZ_512]u8 = undefined; + _ = c.snprintf(&syn_qn, syn_qn.len, "", func_name); + dn_id = c.cbm_gbuf_upsert_node(ws.local_edge_buf, "Decorator", func_name, &syn_qn, "", 0, 0, "{}"); + } + if (dn_id != 0 and node.id != dn_id) { + // Decorator SOURCE TEXT can contain quotes and raw newlines + // (e.g. @register.tag("block"), multi-line @override_settings) — + // interpolating it raw produced malformed properties JSON that + // aborts every json_extract consumer (django: 3826 such edges). + var esc_dec: [c.CBM_SZ_256]u8 = undefined; + _ = c.cbm_json_escape(&esc_dec, esc_dec.len, def.decorators[dc]); + var dp: [c.CBM_SZ_512]u8 = undefined; + _ = c.snprintf(&dp, dp.len, "{\"decorator\":\"%s\"}", @as(CStr, &esc_dec)); + _ = c.cbm_gbuf_insert_edge(ws.local_edge_buf, node.id, dn_id, "DECORATES", &dp); + // Ensure a reference-style edge exists so the decorator appears in queries + // without being misclassified as a real call by downstream passes. + _ = c.cbm_gbuf_insert_edge(ws.local_edge_buf, node.id, dn_id, "USAGE", "{}"); + ws.semantic_resolved += 1; + } + } +} + +/// Resolve INHERITS + DECORATES + IMPLEMENTS for one file. +fn resolveFileSemantic(rc: *ResolveCtx, ws: *ResolveWorkerState, result: *FileResult, module_qn: CStr, imp_keys: StrList, imp_vals: StrList, imp_count: c_int) void { + for (0..ix(result.defs.count)) |d| { + const def = util.at(result.defs.items, d); + if (def.qualified_name == null) continue; + const node: *const Node = c.cbm_gbuf_find_by_qn(rc.main_gbuf, def.qualified_name) orelse continue; + resolveDefInherits(rc, ws, def, node, module_qn, imp_keys, imp_vals, imp_count); + resolveDefDecorators(rc, ws, def, node, module_qn, imp_keys, imp_vals, imp_count); + } + for (0..ix(result.impl_traits.count)) |t| { + const it = util.at(result.impl_traits.items, t); + if (it.trait_name == null or it.struct_name == null) continue; + const tqn = resolveAsClass(rc.registry, it.trait_name, module_qn, imp_keys, imp_vals, imp_count); + const sqn = resolveAsClass(rc.registry, it.struct_name, module_qn, imp_keys, imp_vals, imp_count); + if (tqn == null or sqn == null) continue; + const tn: ?*const Node = c.cbm_gbuf_find_by_qn(rc.main_gbuf, tqn); + const sn: ?*const Node = c.cbm_gbuf_find_by_qn(rc.main_gbuf, sqn); + if (tn != null and sn != null and tn.?.id != sn.?.id) { + _ = c.cbm_gbuf_insert_edge(ws.local_edge_buf, sn.?.id, tn.?.id, "IMPLEMENTS", "{}"); + ws.semantic_resolved += 1; + } + } +} + +// ── FastAPI Depends() tracking ────────────────────────────────────── +// Scans Python function signatures for Depends(func_ref) and creates CALLS +// edges from the endpoint to the dependency function. Without this, FastAPI +// auth/DI functions appear as dead code (in_degree=0). + +const PP_SIG_SCAN_LINES = 15; +const PP_REGEX_GROUPS = 2; + +/// Python signature text of the def starting at start_line. Caller frees. +fn extractPySignature(source: CStr, start_line: c_int, end_line: c_int) MutStr { + var sig_end = start_line + PP_SIG_SCAN_LINES; + if (end_line > 0 and sig_end > end_line) sig_end = end_line; + var p = source; + var line: c_int = 1; + while (p[0] != 0 and line < start_line) { + if (p[0] == '\n') line += 1; + p += 1; + } + const sig_start = p; + while (p[0] != 0 and line < sig_end) { + if (p[0] == '\n') line += 1; + p += 1; + if (@intFromPtr(p) > @intFromPtr(sig_start) + 1 and (p - 1)[0] == ':' and (p - 2)[0] == ')') { + break; + } + } + const sig_len = ptrDiff(p, sig_start); + const sig: MutStr = @ptrCast(c.malloc(sig_len + 1)); + if (sig == null) return null; + @memcpy(sig[0..sig_len], sig_start[0..sig_len]); + sig[sig_len] = 0; + return sig; +} + +/// CALLS edges for each Depends(func_ref) in one def's signature. +fn scanDependsInSig( + rc: *ResolveCtx, + ws: *ResolveWorkerState, + re: *const c.cbm_regex_t, + sig: CStr, + source_node: *const Node, + module_qn: CStr, + imp_keys: StrList, + imp_vals: StrList, + imp_count: c_int, +) void { + var match: [PP_REGEX_GROUPS]c.cbm_regmatch_t = undefined; + var scan = sig; + while (c.cbm_regexec(re, scan, PP_REGEX_GROUPS, &match, 0) == 0) { + var ref_len: c_int = match[1].rm_eo - match[1].rm_so; + var func_ref: [c.CBM_SZ_256]u8 = undefined; + if (ref_len >= @as(c_int, func_ref.len)) ref_len = @as(c_int, func_ref.len) - 1; + const len = ix(ref_len); + @memcpy(func_ref[0..len], (scan + ix(match[1].rm_so))[0..len]); + func_ref[len] = 0; + const res = c.cbm_registry_resolve(rc.registry, &func_ref, module_qn, imp_keys, imp_vals, imp_count); + if (res.qualified_name != null and res.qualified_name[0] != 0) { + const tn: ?*const Node = c.cbm_gbuf_find_by_qn(rc.main_gbuf, res.qualified_name); + if (tn != null and source_node.id != tn.?.id) { + _ = c.cbm_gbuf_insert_edge(ws.local_edge_buf, source_node.id, tn.?.id, "CALLS", "{\"confidence\":0.95,\"strategy\":\"fastapi_depends\"}"); + ws.calls_resolved += 1; + } + } + scan += ix(match[0].rm_eo); + } +} + +fn resolveFileFastapiDepends( + rc: *ResolveCtx, + ws: *ResolveWorkerState, + result: *const FileResult, + path: CStr, + module_qn: CStr, + imp_keys: StrList, + imp_vals: StrList, + imp_count: c_int, +) void { + var has_depends = false; + for (0..ix(result.calls.count)) |ci| { + if (has_depends) break; + const callee: CStr = result.calls.items[ci].callee_name; + has_depends = callee != null and streq(callee, "Depends"); + } + if (!has_depends) return; + var depends_re: c.cbm_regex_t = undefined; + if (c.cbm_regcomp(&depends_re, "Depends\\(([A-Za-z_][A-Za-z0-9_.]*)", c.CBM_REG_EXTENDED) != 0) { + return; + } + var source_owned: MutStr = null; + var source: CStr = result.source; + if (source == null or result.source_len <= 0) { + var source_len: c_int = 0; + source_owned = if (path != null) readFile(path, &source_len, null, null) else null; + source = source_owned; + } + if (source != null) for (0..ix(result.defs.count)) |d| { + const def = util.at(result.defs.items, d); + if (def.qualified_name == null or def.start_line <= 0 or def.label == null or + (!streq(def.label, "Function") and !streq(def.label, "Method"))) + { + continue; + } + const source_node: *const Node = c.cbm_gbuf_find_by_qn(rc.main_gbuf, def.qualified_name) orelse continue; + const sig = extractPySignature(source, @bitCast(def.start_line), @bitCast(def.end_line)); + if (sig == null) continue; + scanDependsInSig(rc, ws, &depends_re, sig, source_node, module_qn, imp_keys, imp_vals, imp_count); + c.free(sig); + }; + freeSource(source_owned); + c.cbm_regfree(&depends_re); +} + +/// F4: get (or lazily build ONCE) the shared all_defs Rust registry. Called by the +/// first worker that hits a NULL-filter rust file; later null-files reuse it. Fast +/// path is a lock-free atomic load; the build happens under rust_shared_mu into the +/// dedicated rust_shared_arena (never a shared pipeline arena from a worker thread). +/// Returns NULL if there are no defs (caller falls back to the per-file build). +fn ppRustSharedRegistry(rc: *ResolveCtx) ?*c.CBMTypeRegistry { + var p = rc.rust_shared_reg.load(.acquire); + if (p != null) return p; + if (rc.all_defs == null or rc.def_count <= 0) return null; + c.cbm_mutex_lock(&rc.rust_shared_mu); + p = rc.rust_shared_reg.load(.monotonic); + if (p == null) { + c.cbm_arena_init(&rc.rust_shared_arena); + rc.rust_shared_arena_live = true; + p = c.cbm_rust_build_cross_registry(&rc.rust_shared_arena, rc.all_defs, rc.def_count); + if (p) |reg| { + var sb: [96]u8 = undefined; + _ = c.snprintf(&sb, sb.len, "types=%d funcs=%d", reg.type_count, reg.func_count); + interop.logInfo("cross_lsp.rust_registry", .{ "scale", @as(CStr, &sb) }); + } + rc.rust_shared_reg.store(p, .release); + } + c.cbm_mutex_unlock(&rc.rust_shared_mu); + return p; +} + +/// void*-typed adapter so the shared dispatch helper (pass_lsp_cross) can +/// borrow the lazily-built shared Rust registry without knowing ResolveCtx. +fn ppRustSharedRegistryGet(ctx: ?*anyopaque) callconv(.c) [*c]c.CBMTypeRegistry { + return ppRustSharedRegistry(@ptrCast(@alignCast(ctx))); +} + +fn resolveWorker(worker_id: c_int, ctx_ptr: ?*anyopaque) callconv(.c) void { + const rc: *ResolveCtx = @ptrCast(@alignCast(ctx_ptr)); + const ws = util.at(rc.workers, ix(worker_id)); + + c.cbm_pxc_set_rust_manifest(rc.rust_manifest); + + if (ws.local_edge_buf == null) { + ws.local_edge_buf = c.cbm_gbuf_new_shared_ids(rc.project_name, rc.repo_path, rc.shared_ids); + } + + // Per-worker service-pattern result cache. The same resolved QN + // (e.g. "fmt.Errorf", "context.Context.Done") appears in many + // call edges across many files within a project — caching turns + // cbm_service_pattern_match's 6 × 30 × strstr scan into one hash + // lookup after the first miss for each QN. Scoped to the worker's + // lifetime in the parallel_resolve phase. + c.cbm_service_pattern_cache_begin(); + + // A result loaded from the spill store lives for one iteration; it is + // released at the top of the next one (every `continue` passes there) + // and after the loop. + var loaded: ?*FileResult = null; + var loaded_idx: c_int = -1; + while (true) { + c.cbm_result_cache_release(rc.result_cache, loaded_idx, loaded); + loaded = null; + loaded_idx = -1; + const file_idx = rc.next_file_idx.fetchAdd(1, .monotonic); + if (file_idx >= rc.file_count) break; + if (loadInt(rc.cancelled) != 0) break; + const fidx = ix(file_idx); + + const loop_t0 = extractNowNs(); + + const result_opt: ?*FileResult = c.cbm_result_cache_acquire(rc.result_cache, rc.spill, file_idx); + if (result_opt != null and result_opt != @as(?*FileResult, rc.result_cache[fidx])) { + loaded = result_opt; + loaded_idx = file_idx; + // cbm_pxc_collect_all_defs stored the inferred JVM namespace on + // the in-memory result; a loaded copy needs it again. + if (rc.all_defs != null) { + _ = c.cbm_pxc_apply_inferred_namespace(result_opt, util.at(rc.files, fidx)); + } + } + const result = result_opt orelse { + addElapsed(&rc.time_ns_total_loop, loop_t0); + continue; + }; + _ = rc.total_files_visited.fetchAdd(1, .monotonic); + + const lang: Language = rc.files[fidx].language; + const rel: CStr = rc.files[fidx].rel_path; + + // Skip cross-LSP for machine-generated files — they're huge (10k- + // 70k lines for k8s protobuf/openapi), have low semantic value for + // graph navigation (boilerplate getters/setters/marshal), and + // dominate the cross-LSP wall time when they have even one + // unresolved call (tree-sitter parse on a 70k-line file is ~1-2s). + // The per-file LSP during extract still indexes their defs/calls + // normally — only the cross-file resolution refinement is skipped. + var is_generated = false; + if (rel != null) { + inline for (.{ + ".pb.go", "zz_generated", "_generated.go", ".gen.go", "/applyconfigurations/", + "_pb2.py", "_pb2_grpc.py", ".pb.cc", ".pb.h", ".pb-c.c", + ".pb-c.h", + }) |marker| { + is_generated = is_generated or c.strstr(rel, marker) != null; + } + } + + // Cross-file LSP is a per-file tree-sitter re-parse + AST walk + + // registry lookups — ~50-150ms per file. It can ONLY find calls + // that exist in the AST. If the per-file extract found zero calls, + // cross-LSP will too: the AST is the same. For non-JVM languages, + // skip when per-file LSP already produced at least as many resolved + // entries as textual calls. Java/Kotlin per-file LSP can fill the + // count with constructors or same-file calls while a mixed-source-root + // Java↔Kotlin call remains unresolved, so JVM callers run whenever + // calls exist. Callable-reference candidates (`accept(handler)` with + // an imported `handler`) count the same way against reference rows. + const jvm_cross_lsp = lang == c.CBM_LANG_JAVA or lang == c.CBM_LANG_KOTLIN; + const rust_workspace_cross_lsp = + lang == c.CBM_LANG_RUST and rc.rust_manifest != null and rc.rust_manifest.?.member_count > 0; + const invocation_rows = c.cbm_pipeline_invocation_row_count(&result.resolved_calls); + var ref_candidates: c_int = 0; + for (0..ix(result.usages.count)) |u| { + ref_candidates += @intFromBool(result.usages.items[u].may_be_call_reference); + } + const cross_lsp_eligible = + rc.all_defs != null and rc.def_count > 0 and c.cbm_pxc_has_cross_lsp(lang) and + (result.calls.count > 0 or ref_candidates > 0) and + (jvm_cross_lsp or rust_workspace_cross_lsp or invocation_rows < result.calls.count or + c.cbm_pipeline_reference_row_count(&result.resolved_calls) < ref_candidates or + (lang == c.CBM_LANG_ZIG and rc.def_modules != null and + c.cbm_pxc_zig_args_in_module(rc.cross_registries, rc.def_modules[fidx]))) and + !is_generated; + + // Skip files with nothing else to resolve and no cross-LSP work. + if (result.calls.count == 0 and result.usages.count == 0 and result.throws.count == 0 and + result.rw.count == 0 and result.defs.count == 0 and result.impl_traits.count == 0 and + !cross_lsp_eligible) + { + continue; + } + + // Build import map ONCE (read-only access to main_gbuf). The + // same imp_keys/imp_vals feed both the fused cross-file LSP + // step below AND the resolve_file_* chain — no duplicate build. + var imp_keys: StrList = null; + var imp_vals: StrList = null; + var imp_count: c_int = 0; + const imp_t0 = extractNowNs(); + _ = buildImportMap(rc.main_gbuf, rc.project_name, rel, &imp_keys, &imp_vals, &imp_count); + addElapsed(&rc.time_ns_import_map, imp_t0); + + // Per-file is_import_reachable memoization. Spans all 5 resolve + // sub-passes (calls/usages/throws/rw/semantic) which all flow + // through cbm_registry_resolve. Same callee_name appears in + // many call sites — first eval pays the strstr cost, repeats + // are O(1) hash. Imports are constant within a file so the + // cache is sound; invalidated at file exit. + c.cbm_registry_reach_cache_begin(result.calls.count + result.usages.count + 64); + + // Per-file import-map prefix → module-qn hash. resolve_import_map + // was doing O(imports) linear strcmp per call; with this it + // becomes O(1). Keys/values borrowed from imp_keys/imp_vals + // which outlive this scope. + c.cbm_registry_import_map_cache_begin(imp_keys, imp_vals, imp_count); + + // THE BIG ONE: per-file cache of cbm_registry_resolve results. + // Same callee_name in multiple call sites resolves identically + // within a file (module_qn is fixed) — first call walks the + // strategy chain, repeats are O(1). On K8s this targets the + // 98.7% hot spot in resolve_file_calls (881 of 893s CPU). + c.cbm_registry_resolve_cache_begin(result.calls.count + result.usages.count + 64); + + const module_qn: MutStr = c.cbm_pipeline_fqn_module_dir(rc.project_name, rel, ppModuleIsDir(lang)); + + // ── Cross-file LSP (FUSED) ───────────────────────────── + // Runs BEFORE resolve_file_calls so its additions to + // result->resolved_calls are picked up by + // cbm_pipeline_find_lsp_resolution when calls become CALLS + // edges. Prefers source bytes retained in result->arena during + // extract; when the low-RAM retention cap dropped this file + // (result->source==NULL) it FALLS BACK to a bounded per-file + // read from disk, freed immediately after the LSP call. This is + // the correctness guarantee: lowering the retention cap trades + // retained RAM for a bounded re-read, it NEVER drops a cross-file + // edge. Only a genuine read failure (deleted/unreadable/oversized) + // leaves source NULL and is counted as skipped_no_source; defs/calls + // already in the extract are unaffected either way. + // + // Slab reclaim afterward: the LSP re-parses via tree-sitter, + // which allocates through this worker's TLS slab. Reclaiming + // here keeps the slab high-water bounded as the resolve phase + // walks across thousands of files in a single worker thread. + if (cross_lsp_eligible) { + var lsp_source_owned: MutStr = null; + var lsp_source: CStr = result.source; + var lsp_source_len: c_int = result.source_len; + if ((lsp_source == null or lsp_source_len <= 0) and rc.files[fidx].path != null) { + // Retention cap skipped this file — re-read on demand (bounded + // by readFile's cbm_max_file_bytes cap), freed below. + lsp_source_owned = readFile(rc.files[fidx].path, &lsp_source_len, null, null); + lsp_source = lsp_source_owned; + } + if (lsp_source != null and lsp_source_len > 0) { + const def_module: CStr = if (rc.def_modules != null) rc.def_modules[fidx] else module_qn; + + const lsp_t0 = extractNowNs(); + + // Shared per-file dispatch (pass_lsp_cross): module-def + // filter → shared prebuilt registry (overlay pattern) → + // filtered per-file fallback. Journal the + // file around the resolve so a hang HERE is attributed to + // this file, not to a stale extraction marker. + c.cbm_index_mark_start(rel); + const had_error = result.has_error; + c.cbm_pxc_dispatch_file(lang, result, lsp_source, lsp_source_len, rel, def_module, rc.cross_registries, rc.module_def_index, rc.all_defs, rc.def_count, imp_keys, imp_vals, imp_count, ppRustSharedRegistryGet, rc); + c.cbm_index_mark_done(rel); + if (!had_error and result.has_error) { + rc.lsp_failures[fidx] = result.error_msg; + } + // Free the on-demand re-read (no-op when source was retained). + freeSource(lsp_source_owned); + // Contract: cbm_slab_reclaim() requires the thread parser to be + // destroyed first; otherwise its lexer holds slab pointers + // (lexer.included_ranges) that get freed underneath it, causing + // a heap-use-after-free on the next ts_lexer_goto. The next + // cbm_extract_file on this thread will recreate the parser. + c.cbm_destroy_thread_parser(); + c.cbm_slab_reclaim(); + const lsp_elapsed_ns = extractNowNs() -% lsp_t0; + _ = rc.time_ns_cross_lsp.fetchAdd(lsp_elapsed_ns, .monotonic); + const lsp_elapsed_ms = lsp_elapsed_ns / PP_USEC_PER_MS; + if (lsp_elapsed_ms > PP_TIMER_THRESH) { + interop.logInfo("parallel.resolve.lsp_cross.slow", .{ + "elapsed_ms", itoa(msInt(lsp_elapsed_ms)), + "path", rel, + }); + } + _ = rc.lsp_cross_processed.fetchAdd(1, .monotonic); + } else { + // Source unavailable even after the re-read fallback (file + // deleted / unreadable / oversized) → the cross-file LSP + // refinement no-ops for this file. This is a bounded skip, NOT + // a file failure: defs/calls were already extracted and are + // unaffected. Deliberately NOT recorded as a cbm_file_error — + // doing so would flood skipped[] with false positives (itself a + // false-guard bug). The "cross_lsp" phase string is reserved for + // Track C's real crash-attribution signal; leave it unwired. + _ = rc.lsp_cross_skipped_no_source.fetchAdd(1, .monotonic); + } + } + + // Per-sub-phase wall-clock so we can attribute the dominant cost. + var ph_t0: u64 = undefined; + + // ── CALLS resolution ──────────────────────────────────── + ph_t0 = extractNowNs(); + resolveFileCalls(rc, ws, result, rel, module_qn, imp_keys, imp_vals, imp_count, lang); + if (lang == c.CBM_LANG_PYTHON) { + resolveFileFastapiDepends(rc, ws, result, rc.files[fidx].path, module_qn, imp_keys, imp_vals, imp_count); + } + addElapsed(&rc.time_ns_calls, ph_t0); + + // ── USAGE resolution ──────────────────────────────────── + ph_t0 = extractNowNs(); + resolveFileUsages(rc, ws, result, rel, module_qn, imp_keys, imp_vals, imp_count, lang); + addElapsed(&rc.time_ns_usages, ph_t0); + + // ── THROWS / RAISES ───────────────────────────────────── + ph_t0 = extractNowNs(); + resolveFileThrows(rc, ws, result, module_qn, imp_keys, imp_vals, imp_count); + addElapsed(&rc.time_ns_throws, ph_t0); + + // ── READS / WRITES ────────────────────────────────────── + ph_t0 = extractNowNs(); + resolveFileRw(rc, ws, result, rel, module_qn, imp_keys, imp_vals, imp_count, lang); + addElapsed(&rc.time_ns_rw, ph_t0); + + // ── INHERITS + DECORATES + IMPLEMENTS ──────────────────── + ph_t0 = extractNowNs(); + resolveFileSemantic(rc, ws, result, module_qn, imp_keys, imp_vals, imp_count); + addElapsed(&rc.time_ns_semantic, ph_t0); + + c.cbm_registry_reach_cache_end(); + c.cbm_registry_import_map_cache_end(); + c.cbm_registry_resolve_cache_end(); + + c.free(module_qn); + freeImportMap(imp_keys, imp_vals, imp_count); + + addElapsed(&rc.time_ns_total_loop, loop_t0); + } + c.cbm_result_cache_release(rc.result_cache, loaded_idx, loaded); + + // Tear down this worker's thread-local parser + slab state, mirroring + // extractWorker. Without this, resolve-worker pages keep an owner pointer + // into dead TLS and are never retired, so a later cross-thread free can + // never bring their refcount to zero (leak). Retiring them here releases + // each page as its final chunk returns. + c.cbm_pxc_set_rust_manifest(null); + c.cbm_destroy_thread_parser(); + c.cbm_slab_destroy_thread(); + c.cbm_service_pattern_cache_end(); +} + +/// snprintf("%llu") of `v` into `buf`. +fn u64Text(buf: *[32]u8, v: u64) CStr { + _ = c.snprintf(buf, buf.len, "%llu", @as(c_ulonglong, v)); + return buf; +} + +/// A nanosecond counter in whole milliseconds, as text in `buf`. +fn nsAsMsText(buf: *[32]u8, counter: *const AtomicU64) CStr { + return u64Text(buf, counter.load(.monotonic) / 1000000); +} + +pub export fn cbm_parallel_resolve( + ctx: *Ctx, + files: [*c]const FileInfo, + file_count: c_int, + result_cache: [*c][*c]FileResult, + shared_ids: [*c]i64, + worker_count: c_int, + all_defs: [*c]LspDef, + def_count: c_int, + def_modules: [*c]const MutStr, + module_def_index: ?*c.struct_CBMModuleDefIndex, + cross_registries_v: ?*anyopaque, +) c_int { + // See header: typed as void* across the TU boundary; cast back here. + const cross_registries: ?*c.CBMCrossLspRegistries = @ptrCast(@alignCast(cross_registries_v)); + if (file_count == 0) return 0; + + interop.logInfo("parallel.resolve.start", .{ + "files", itoa(file_count), + "workers", itoa(worker_count), + }); + + var workers_raw: ?*anyopaque = null; + const workers_bytes = ix(worker_count) * @sizeOf(ResolveWorkerState); + if (c.cbm_aligned_alloc(&workers_raw, CBM_CACHE_LINE, workers_bytes) != 0) return c.CBM_NOT_FOUND; + const workers: [*c]ResolveWorkerState = @ptrCast(@alignCast(workers_raw)); + @memset(@as([*]u8, @ptrCast(workers))[0..workers_bytes], 0); + const lsp_failures = interop.callocN(CStr, ix(file_count)); + if (lsp_failures == null) { + c.cbm_aligned_free(workers); + return c.CBM_NOT_FOUND; + } + + var have_rust = false; + for (0..ix(file_count)) |i| { + if (files[i].language == c.CBM_LANG_RUST and + c.cbm_result_cache_present(result_cache, ctx.spill, @intCast(i))) + { + have_rust = true; + break; + } + } + var rust_manifest_arena: Arena = undefined; + var rust_manifest: c.CBMCargoManifest = undefined; + var rust_manifest_arena_live = false; + var rust_manifest_ptr: ?*const c.CBMCargoManifest = null; + if (have_rust) { + c.cbm_arena_init(&rust_manifest_arena); + rust_manifest_arena_live = true; + if (c.cbm_pxc_build_rust_manifest(ctx, &rust_manifest_arena, &rust_manifest)) { + rust_manifest_ptr = &rust_manifest; + } + } + + var rc: ResolveCtx = .{ + .files = files, + .file_count = file_count, + .project_name = ctx.project_name, + .repo_path = ctx.repo_path, + .workers = workers, + .max_workers = worker_count, + .result_cache = result_cache, + .spill = ctx.spill, + .main_gbuf = ctx.gbuf, + .registry = ctx.registry, + .shared_ids = shared_ids, + .cancelled = ctx.cancelled, + .all_defs = all_defs, + .def_count = def_count, + .def_modules = def_modules, + .module_def_index = module_def_index, + .cross_registries = cross_registries, + .rust_manifest = rust_manifest_ptr, + .lsp_failures = lsp_failures, + }; + @atomicStore(u64, &g_lsp_tail_lookups, 0, .monotonic); + @atomicStore(u64, &g_lsp_tail_candidates, 0, .monotonic); + c.cbm_pxc_filter_stats_reset(); + // F4 lazy shared Rust registry: mutex up before workers spawn. + c.cbm_mutex_init(&rc.rust_shared_mu); + rc.rust_shared_arena_live = false; + + // Sub-phase: Dispatch resolve workers (per-file call/usage resolution, PARALLEL) + const t_resolve_dispatch = util.profStart(); + var opts = std.mem.zeroes(c.cbm_parallel_for_opts_t); + opts.max_workers = worker_count; + opts.force_pthreads = false; + c.cbm_parallel_for(worker_count, resolveWorker, &rc, opts); + util.profEnd("parallel_resolve", "1_dispatch_workers_parallel", &t_resolve_dispatch, file_count); + for (0..ix(file_count)) |i| { + if (lsp_failures[i] != null and ctx.pipeline != null) { + c.cbm_pipeline_add_file_error(ctx.pipeline, files[i].rel_path, lsp_failures[i], "lsp_incomplete"); + } + } + c.free(@ptrCast(lsp_failures)); + // Workers joined: the shared Rust registry (if built) is no longer read. + // Free its dedicated arena + the lock (registry was self-contained: it strdup'd + // all QNs, so freeing all_defs afterward is safe). + if (rc.rust_shared_arena_live) { + c.cbm_arena_destroy(&rc.rust_shared_arena); + rc.rust_shared_arena_live = false; + } + c.cbm_mutex_destroy(&rc.rust_shared_mu); + if (rust_manifest_arena_live) c.cbm_arena_destroy(&rust_manifest_arena); + + // Sub-phase: Merge all local edge bufs into main gbuf (SEQUENTIAL) + const t_resolve_merge = util.profStart(); + var total_calls: c_int = 0; + var total_usages: c_int = 0; + var total_semantic: c_int = 0; + var total_lsp_overrides: c_int = 0; + for (0..ix(worker_count)) |i| { + if (workers[i].local_edge_buf != null) { + _ = c.cbm_gbuf_merge(ctx.gbuf, workers[i].local_edge_buf); + total_calls += workers[i].calls_resolved; + total_usages += workers[i].usages_resolved; + total_semantic += workers[i].semantic_resolved; + total_lsp_overrides += workers[i].lsp_overrides; + c.cbm_gbuf_free(workers[i].local_edge_buf); + } + } + util.profEnd("parallel_resolve", "2_merge_edge_bufs_seq", &t_resolve_merge, total_calls + total_usages); + + c.cbm_aligned_free(workers); + + // Go-style implicit interface satisfaction (needs full graph, serial) + const go_impl = c.cbm_pipeline_implements_go(ctx); + + if (@atomicLoad(c_int, ctx.cancelled, .seq_cst) != 0) return c.CBM_NOT_FOUND; + + // Summary metric that replaces the removed `pass.timing pass=lsp_cross` + // log line — surfaces how many files the fused cross-file LSP step + // actually processed vs skipped (e.g. because their source bytes + // were not retained at extract time due to the per-file/total cap). + interop.logInfo("parallel.resolve.lsp_cross_done", .{ + "files_processed", itoa(rc.lsp_cross_processed.load(.monotonic)), + "files_skipped_no_source", itoa(rc.lsp_cross_skipped_no_source.load(.monotonic)), + "defs_total", itoa(def_count), + }); + + // Cross-LSP cost normalised per file (upstream f6e3af43/cffb4e37). Wall + // time cannot tell "big repo" from "superlinear pass"; us_per_file can: + // for a pass linear in file count it stays roughly constant across repo + // sizes. defs_per_file is the lever behind it: if it tracks defs_total + // rather than the file's own module plus imports, the module filter is + // not containing the work and cross-file LSP is O(files x corpus_defs). + const cross_files = rc.lsp_cross_processed.load(.monotonic); + const cross_us = rc.time_ns_cross_lsp.load(.monotonic) / 1000; + if (cross_files > 0) { + const cross_files_u: u64 = @intCast(cross_files); + var cf_buf: [32]u8 = undefined; + var nf_buf: [32]u8 = undefined; + var cu_buf: [32]u8 = undefined; + var pk_buf: [32]u8 = undefined; + _ = c.snprintf(&nf_buf, nf_buf.len, "%d", cross_files); + const per_kdef: u64 = if (def_count > 0) + (cross_us *% 1000) / (cross_files_u *% @as(u64, @intCast(def_count))) + else + 0; + interop.logInfo("parallel.resolve.cross_lsp_cost", .{ + "cross_lsp_ms", u64Text(&cu_buf, cross_us / 1000), + "files", @as(CStr, &nf_buf), + "us_per_file", u64Text(&cf_buf, cross_us / cross_files_u), + "defs_total", itoa(def_count), + "us_per_file_per_kdef", u64Text(&pk_buf, per_kdef), + }); + + var reg_defs: u64 = 0; + var reg_files: u64 = 0; + var flt_files: u64 = 0; + var flt_failed: u64 = 0; + c.cbm_pxc_filter_stats(®_defs, ®_files, &flt_files, &flt_failed); + if (reg_files > 0) { + var rd_buf: [32]u8 = undefined; + var ff_buf: [32]u8 = undefined; + var fp_buf: [32]u8 = undefined; + interop.logInfo("parallel.resolve.perfile_registry", .{ + "defs_per_file", u64Text(&rd_buf, reg_defs / reg_files), + "defs_total", itoa(def_count), + "filtered_files", u64Text(&fp_buf, flt_files), + "filter_failed", u64Text(&ff_buf, flt_failed), + }); + } + } + + interop.logInfo("parallel.resolve.done", .{ + "calls", itoa(total_calls), + "usages", itoa(total_usages), + "semantic", itoa(total_semantic + go_impl), + "lsp_overrides", itoa(total_lsp_overrides), + }); + + // Per-sub-phase breakdown so we stop guessing about hot paths. + // Numbers are summed across workers (total CPU-ms, not wall-time). + var loop_buf: [32]u8 = undefined; + var visits_buf: [32]u8 = undefined; + _ = c.snprintf(&visits_buf, visits_buf.len, "%d", rc.total_files_visited.load(.monotonic)); + interop.logInfo("parallel.resolve.phase_summary", .{ + "total_loop_cpu_ms", nsAsMsText(&loop_buf, &rc.time_ns_total_loop), + "files_visited", @as(CStr, &visits_buf), + }); + + var imp_buf: [32]u8 = undefined; + var xls_buf: [32]u8 = undefined; + var cal_buf: [32]u8 = undefined; + interop.logInfo("parallel.resolve.phase_ms_a", .{ + "import_map", nsAsMsText(&imp_buf, &rc.time_ns_import_map), + "cross_lsp", nsAsMsText(&xls_buf, &rc.time_ns_cross_lsp), + "resolve_calls", nsAsMsText(&cal_buf, &rc.time_ns_calls), + }); + + var use_buf: [32]u8 = undefined; + var thr_buf: [32]u8 = undefined; + var rw_buf: [32]u8 = undefined; + var sem_buf: [32]u8 = undefined; + interop.logInfo("parallel.resolve.phase_ms_b", .{ + "resolve_usages", nsAsMsText(&use_buf, &rc.time_ns_usages), + "resolve_throws", nsAsMsText(&thr_buf, &rc.time_ns_throws), + "resolve_rw", nsAsMsText(&rw_buf, &rc.time_ns_rw), + "resolve_semantic", nsAsMsText(&sem_buf, &rc.time_ns_semantic), + }); + + var src_buf: [32]u8 = undefined; + var lsp_buf: [32]u8 = undefined; + var rsv_buf: [32]u8 = undefined; + var hnt_buf: [32]u8 = undefined; + var tgt_buf: [32]u8 = undefined; + var emt_buf: [32]u8 = undefined; + interop.logInfo("parallel.resolve.calls_breakdown", .{ + "find_source", nsAsMsText(&src_buf, &rc.time_ns_rc_source), + "lsp_lookup", nsAsMsText(&lsp_buf, &rc.time_ns_rc_lsp_lookup), + "resolve", nsAsMsText(&rsv_buf, &rc.time_ns_rc_resolve), + }); + interop.logInfo("parallel.resolve.calls_breakdown2", .{ + "field_hint", nsAsMsText(&hnt_buf, &rc.time_ns_rc_hint), + "find_target", nsAsMsText(&tgt_buf, &rc.time_ns_rc_target), + "emit_edge", nsAsMsText(&emt_buf, &rc.time_ns_rc_emit), + }); + + // Candidate-scan cost (upstream f6e3af43). per_lookup is a property of + // the corpus, not the machine; if it grows with repo size the tail-match + // scan is turning resolve superlinear. + const tail_lookups = @atomicLoad(u64, &g_lsp_tail_lookups, .monotonic); + const tail_cands = @atomicLoad(u64, &g_lsp_tail_candidates, .monotonic); + var tl_buf: [32]u8 = undefined; + var tc_buf: [32]u8 = undefined; + var tp_buf: [32]u8 = undefined; + interop.logInfo("parallel.resolve.scan_cost", .{ + "tail_lookups", u64Text(&tl_buf, tail_lookups), + "tail_candidates", u64Text(&tc_buf, tail_cands), + "per_lookup", u64Text(&tp_buf, if (tail_lookups != 0) tail_cands / tail_lookups else 0), + }); + return 0; +} + +pub export fn cbm_parallel_resolve_calls(ctx_c: ?*Ctx, files: [*c]const FileInfo, file_count: c_int) void { + const ctx = ctx_c orelse return; + if (files == null or file_count <= 0) return; + var shared_ids: i64 = c.cbm_gbuf_next_id(ctx.gbuf); + var rc: ResolveCtx = .{ + .files = files, + .file_count = file_count, + .project_name = ctx.project_name, + .repo_path = ctx.repo_path, + .workers = null, + .max_workers = 0, + .result_cache = null, + .spill = null, + .main_gbuf = ctx.gbuf, + .registry = ctx.registry, + .shared_ids = &shared_ids, + .cancelled = ctx.cancelled, + .all_defs = null, + .def_count = 0, + .def_modules = null, + .module_def_index = null, + .cross_registries = null, + }; + var ws = std.mem.zeroes(ResolveWorkerState); + ws.local_edge_buf = c.cbm_gbuf_new_shared_ids(ctx.project_name, ctx.repo_path, &shared_ids); + if (ws.local_edge_buf == null) return; + for (0..ix(file_count)) |i| { + if (util.checkCancel(ctx) != 0) break; + const fi = util.at(files, i); + var source_len: c_int = 0; + const source = readFile(fi.path, &source_len, null, null); + if (source == null) continue; + const result_opt: ?*FileResult = c.cbm_extract_file(source, source_len, fi.language, ctx.project_name, fi.rel_path, c.CBM_EXTRACT_BUDGET, null, null); + freeSource(source); + const result = result_opt orelse continue; + if (result.calls.count > 0) { + var imp_keys: StrList = null; + var imp_vals: StrList = null; + var imp_count: c_int = 0; + _ = buildImportMap(ctx.gbuf, ctx.project_name, fi.rel_path, &imp_keys, &imp_vals, &imp_count); + const module_qn: MutStr = c.cbm_pipeline_fqn_module_dir(ctx.project_name, fi.rel_path, ppModuleIsDir(fi.language)); + resolveFileCalls(&rc, &ws, result, fi.rel_path, module_qn, imp_keys, imp_vals, imp_count, fi.language); + if (fi.language == c.CBM_LANG_PYTHON) { + resolveFileFastapiDepends(&rc, &ws, result, fi.path, module_qn, imp_keys, imp_vals, imp_count); + } + c.free(module_qn); + freeImportMap(imp_keys, imp_vals, imp_count); + } + c.cbm_free_result(result); + } + _ = c.cbm_gbuf_merge(ctx.gbuf, ws.local_edge_buf); + c.cbm_gbuf_free(ws.local_edge_buf); + c.cbm_pipeline_sync_next_id(ctx.gbuf, &shared_ids); + interop.logInfo("parallel.resolve_calls.done", .{ + "files", itoa(file_count), + "calls", itoa(ws.calls_resolved), + }); +} diff --git a/src/pipeline/pass_pkgmap.c b/src/pipeline/pass_pkgmap.c deleted file mode 100644 index 4f776ca7..00000000 --- a/src/pipeline/pass_pkgmap.c +++ /dev/null @@ -1,2233 +0,0 @@ -/* - * pass_pkgmap.c — Generic package/module manifest resolution. - * - * Scans discovered files for manifest files (package.json, go.mod, Cargo.toml, - * pyproject.toml, composer.json, pubspec.yaml, pom.xml, build.gradle, mix.exs, - * *.gemspec) and builds a hash table mapping bare package specifiers to resolved - * module QNs. This enables IMPORTS edges for non-relative imports like - * "@myorg/pkg", "github.com/foo/bar", "use my_crate::foo". - * - * Integration: called from parallel extract workers (per-worker local arrays) - * and merged sequentially before registry build. - */ -#include "pipeline/pipeline.h" -#include "pipeline/pipeline_internal.h" -#include "discover/discover.h" -#include "foundation/compat.h" -#include "foundation/compat_fs.h" -#include "foundation/constants.h" -#include "foundation/hash_table.h" -#include "foundation/log.h" -#include "foundation/platform.h" -#include "foundation/str_util.h" -#include "foundation/win_utf8.h" -#include "foundation/yaml.h" - -#include - -#include -#include -#include -#include -#include - -/* Read an entire file into a malloc'd buffer. Returns NULL on failure. */ -static char *pkgmap_read_file(const char *path, int *out_len) { - FILE *f = cbm_fopen(path, "rb"); - if (!f) { - return NULL; - } - (void)fseek(f, 0, SEEK_END); - long size = ftell(f); - (void)fseek(f, 0, SEEK_SET); - if (size <= 0 || size > (long)CBM_SZ_1K * CBM_SZ_1K) { /* 1MB cap for manifests */ - (void)fclose(f); - return NULL; - } - char *buf = (char *)malloc((size_t)size + SKIP_ONE); - if (!buf) { - (void)fclose(f); - return NULL; - } - size_t nread = fread(buf, SKIP_ONE, (size_t)size, f); - (void)fclose(f); - buf[nread] = '\0'; - *out_len = (int)nread; - return buf; -} - -/* ── Constants ─────────────────────────────────────────────────── */ - -enum { - PKGMAP_INIT_CAP = 16, - PKGMAP_PATH_BUF = 1024, - PKGMAP_LINE_BUF = 512, - PKGMAP_HT_INIT = 64, - /* Hard ceiling on recursive directory descent. This is the - * non-negotiable termination guarantee for the manifest walk: even - * if the filesystem contains directory junctions / symlink cycles - * (the documented reason the walk was once disabled on Windows), - * descent stops at this depth so the walk can never hang. 64 is far - * deeper than any real source tree. */ - PKGMAP_WALK_MAX_DEPTH = 64, - /* String lengths for manifest parsing (avoid magic numbers in memcmp) */ - TOML_NAME_LEN = 4, /* strlen("name") */ - TOML_NAME_SP = 5, /* strlen("name ") */ - TOML_NAME_EQ = 5, /* strlen("name=") */ - XML_PARENT_OPEN = 8, /* strlen("") */ - XML_PARENT_CLOSE = 9, /* strlen("") */ - XML_GROUP_OPEN = 9, /* strlen("") */ - XML_ARTIFACT_OPEN = 12, /* strlen("") */ -}; - -/* Check if src at position p starts with literal str of known length. */ -static bool at_prefix(const char *p, const char *end, const char *prefix, int prefix_len) { - return p + prefix_len <= end && memcmp(p, prefix, (size_t)prefix_len) == 0; -} - -/* ── Per-worker collection ─────────────────────────────────────── */ - -void cbm_pkg_entries_init(cbm_pkg_entries_t *e) { - e->items = NULL; - e->count = 0; - e->cap = 0; -} - -static void pkg_entries_push(cbm_pkg_entries_t *e, char *pkg_name, char *entry_rel) { - if (e->count >= e->cap) { - int new_cap = e->cap == 0 ? PKGMAP_INIT_CAP : e->cap * SKIP_ONE * PAIR_LEN; - cbm_pkg_entry_t *tmp = realloc(e->items, new_cap * sizeof(cbm_pkg_entry_t)); - if (!tmp) { - free(pkg_name); - free(entry_rel); - return; - } - e->items = tmp; - e->cap = new_cap; - } - e->items[e->count].pkg_name = pkg_name; - e->items[e->count].entry_rel = entry_rel; - e->count++; -} - -void cbm_pkg_entries_free(cbm_pkg_entries_t *e) { - for (int i = 0; i < e->count; i++) { - free(e->items[i].pkg_name); - free(e->items[i].entry_rel); - } - free(e->items); - e->items = NULL; - e->count = 0; - e->cap = 0; -} - -/* ── Helpers ───────────────────────────────────────────────────── */ - -/* Get the directory part of a relative path (without trailing slash). - * Returns heap-allocated string. For "foo.json" returns "". */ -static char *path_dirname(const char *rel_path) { - const char *last = strrchr(rel_path, '/'); - if (!last) { - return strdup(""); - } - return cbm_strndup(rel_path, (size_t)(last - rel_path)); -} - -/* Strip file extension from a path. Returns heap-allocated string. - * "src/index.ts" → "src/index", "lib/main" → "lib/main" */ -static char *strip_extension(const char *path) { - size_t len = strlen(path); - for (size_t i = len; i > 0; i--) { - if (path[i - SKIP_ONE] == '.') { - return cbm_strndup(path, i - SKIP_ONE); - } - if (path[i - SKIP_ONE] == '/') { - break; - } - } - return strdup(path); -} - -/* Join directory + relative entry path, normalize. - * "packages/foo" + "src/index.ts" → "packages/foo/src/index" (stripped ext) */ -static char *join_and_strip(const char *dir, const char *entry) { - if (!entry || entry[0] == '\0') { - return NULL; - } - /* Skip leading ./ from entry */ - if (entry[0] == '.' && entry[SKIP_ONE] == '/') { - entry += PAIR_LEN; - } - char buf[PKGMAP_PATH_BUF]; - if (dir[0] == '\0') { - snprintf(buf, sizeof(buf), "%s", entry); - } else { - snprintf(buf, sizeof(buf), "%s/%s", dir, entry); - } - return strip_extension(buf); -} - -/* Find a line starting with a prefix in source. Returns pointer to first char - * after prefix, or NULL. Handles leading whitespace. */ -static const char *find_line_value(const char *src, int src_len, const char *prefix) { - size_t plen = strlen(prefix); - const char *p = src; - const char *end = src + src_len; - while (p < end) { - /* Skip leading whitespace */ - while (p < end && (*p == ' ' || *p == '\t')) { - p++; - } - if (p + plen <= end && memcmp(p, prefix, plen) == 0) { - return p + plen; - } - /* Skip to next line */ - while (p < end && *p != '\n') { - p++; - } - if (p < end) { - p++; /* skip \n */ - } - } - return NULL; -} - -/* Extract a quoted string value from position. Handles both "..." and '...' - * Returns heap-allocated string, or NULL. */ -static char *extract_quoted(const char *p, const char *end) { - /* Skip whitespace and = sign */ - while (p < end && (*p == ' ' || *p == '\t' || *p == '=')) { - p++; - } - if (p >= end) { - return NULL; - } - char quote = *p; - if (quote != '"' && quote != '\'') { - return NULL; - } - p++; - const char *start = p; - while (p < end && *p != quote && *p != '\n') { - p++; - } - if (p >= end || *p != quote) { - return NULL; - } - return cbm_strndup(start, (size_t)(p - start)); -} - -/* ── Language-specific manifest parsers ────────────────────────── */ - -/* Resolve JS/TS entry point from exports["."] object. */ -static const char *resolve_exports_dot(yyjson_val *dot) { - if (yyjson_is_str(dot)) { - return yyjson_get_str(dot); - } - if (!yyjson_is_obj(dot)) { - return NULL; - } - static const char *keys[] = {"import", "default", "require"}; - for (int i = 0; i < (int)(sizeof(keys) / sizeof(keys[0])); i++) { - yyjson_val *v = yyjson_obj_get(dot, keys[i]); - if (yyjson_is_str(v)) { - return yyjson_get_str(v); - } - } - return NULL; -} - -/* Resolve JS/TS entry point from package.json root. */ -static const char *resolve_pkg_entry(yyjson_val *root) { - yyjson_val *exports = yyjson_obj_get(root, "exports"); - if (yyjson_is_obj(exports)) { - const char *e = resolve_exports_dot(yyjson_obj_get(exports, ".")); - if (e) { - return e; - } - } - static const char *fallback_keys[] = {"main", "module"}; - for (int i = 0; i < (int)(sizeof(fallback_keys) / sizeof(fallback_keys[0])); i++) { - yyjson_val *v = yyjson_obj_get(root, fallback_keys[i]); - if (yyjson_is_str(v)) { - return yyjson_get_str(v); - } - } - return "src/index.ts"; /* last resort default */ -} - -/* JS/TS: package.json — name + entry point resolution */ -static void parse_package_json(const char *source, int source_len, const char *rel_path, - cbm_pkg_entries_t *entries) { - yyjson_doc *doc = yyjson_read(source, (size_t)source_len, 0); - if (!doc) { - return; - } - yyjson_val *root = yyjson_doc_get_root(doc); - if (!yyjson_is_obj(root)) { - yyjson_doc_free(doc); - return; - } - - yyjson_val *name_val = yyjson_obj_get(root, "name"); - if (!yyjson_is_str(name_val)) { - yyjson_doc_free(doc); - return; - } - const char *name = yyjson_get_str(name_val); - if (!name || name[0] == '\0') { - yyjson_doc_free(doc); - return; - } - - const char *entry = resolve_pkg_entry(root); - if (entry) { - char *dir = path_dirname(rel_path); - char *resolved = join_and_strip(dir, entry); - if (resolved) { - pkg_entries_push(entries, strdup(name), resolved); - } - free(dir); - } - - yyjson_doc_free(doc); -} - -/* Go: go.mod — module directive */ -static void parse_go_mod(const char *source, int source_len, const char *rel_path, - cbm_pkg_entries_t *entries) { - const char *end = source + source_len; - const char *val = find_line_value(source, source_len, "module "); - if (!val) { - return; - } - /* Extract module path (rest of line, trimmed) */ - while (val < end && (*val == ' ' || *val == '\t')) { - val++; - } - const char *start = val; - while (val < end && *val != '\n' && *val != '\r' && *val != ' ') { - val++; - } - if (val <= start) { - return; - } - char *module_path = cbm_strndup(start, (size_t)(val - start)); - char *dir = path_dirname(rel_path); - - /* The module path maps to the directory containing go.mod. - * For "." dir, use empty string. */ - pkg_entries_push(entries, module_path, strdup(dir)); - free(dir); -} - -/* Extract "name" value from a TOML section. Scans from section_start until - * the next [...] header or EOF. Returns heap string or NULL. */ -static char *toml_extract_name(const char *section_start, const char *end) { - const char *p = section_start; - while (p < end) { - while (p < end && (*p == ' ' || *p == '\t')) { - p++; - } - if (p < end && *p == '[') { - break; - } - if (at_prefix(p, end, "name ", TOML_NAME_SP)) { - return extract_quoted(p + TOML_NAME_SP, end); - } - if (at_prefix(p, end, "name=", TOML_NAME_EQ)) { - return extract_quoted(p + TOML_NAME_EQ, end); - } - while (p < end && *p != '\n') { - p++; - } - if (p < end) { - p++; - } - } - return NULL; -} - -/* Build entry path: dir/suffix or just suffix if dir is empty. */ -static char *build_entry_path(const char *rel_path, const char *suffix) { - char *dir = path_dirname(rel_path); - char buf[PKGMAP_PATH_BUF]; - snprintf(buf, sizeof(buf), "%s%s%s", dir[0] ? dir : "", dir[0] ? "/" : "", suffix); - free(dir); - return strdup(buf); -} - -/* Rust: Cargo.toml — [package] name */ -static void parse_cargo_toml(const char *source, int source_len, const char *rel_path, - cbm_pkg_entries_t *entries) { - const char *section = find_line_value(source, source_len, "[package]"); - if (!section) { - return; - } - char *name = toml_extract_name(section, source + source_len); - if (!name) { - return; - } - char *entry = build_entry_path(rel_path, "src/lib"); - pkg_entries_push(entries, name, entry); -} - -/* Python: pyproject.toml — [project] name */ -/* Normalize Python package name: hyphens → underscores (PEP 503). */ -static void py_normalize_name(char *name) { - for (char *c = name; *c; c++) { - if (*c == '-') { - *c = '_'; - } - } -} - -static void parse_pyproject_toml(const char *source, int source_len, const char *rel_path, - cbm_pkg_entries_t *entries) { - const char *section = find_line_value(source, source_len, "[project]"); - if (!section) { - return; - } - char *name = toml_extract_name(section, source + source_len); - if (!name) { - return; - } - py_normalize_name(name); - - /* Register src//__init__ as primary entry */ - char suffix[PKGMAP_PATH_BUF]; - snprintf(suffix, sizeof(suffix), "src/%s/__init__", name); - char *entry = build_entry_path(rel_path, suffix); - char *name_copy = strdup(name); - pkg_entries_push(entries, name, entry); - - /* Also register /__init__ as alternative (no src/ prefix) */ - snprintf(suffix, sizeof(suffix), "%s/__init__", name_copy); - char *alt_entry = build_entry_path(rel_path, suffix); - if (name_copy && alt_entry) { - pkg_entries_push(entries, name_copy, alt_entry); - } else { - free(name_copy); - free(alt_entry); - } -} - -/* Extract PSR-4 entries from one composer.json autoload section ("autoload" - * or "autoload-dev": composer registers both, so test namespaces such as - * Tests\ map to their directory as well). */ -static void extract_psr4(yyjson_val *root, const char *section, const char *dir, - cbm_pkg_entries_t *entries) { - yyjson_val *autoload = yyjson_obj_get(root, section); - if (!yyjson_is_obj(autoload)) { - return; - } - yyjson_val *psr4 = yyjson_obj_get(autoload, "psr-4"); - if (!yyjson_is_obj(psr4)) { - return; - } - yyjson_val *key = NULL; - yyjson_obj_iter iter = yyjson_obj_iter_with(psr4); - while ((key = yyjson_obj_iter_next(&iter)) != NULL) { - yyjson_val *val = yyjson_obj_iter_get_val(key); - if (!yyjson_is_str(key) || !yyjson_is_str(val)) { - continue; - } - const char *ns_prefix = yyjson_get_str(key); - const char *ns_dir = yyjson_get_str(val); - while (ns_dir[0] == '.' && ns_dir[SKIP_ONE] == '/') { - ns_dir += PAIR_LEN; /* "./src/" names src/ */ - } - if (strcmp(ns_dir, ".") == 0) { - ns_dir = ""; /* "." names the manifest's own directory */ - } - char ns_entry[PKGMAP_PATH_BUF]; - if (dir[0]) { - snprintf(ns_entry, sizeof(ns_entry), "%s/%s", dir, ns_dir); - } else { - snprintf(ns_entry, sizeof(ns_entry), "%s", ns_dir); - } - size_t nelen = strlen(ns_entry); - if (nelen > 0 && ns_entry[nelen - SKIP_ONE] == '/') { - ns_entry[nelen - SKIP_ONE] = '\0'; - } - pkg_entries_push(entries, strdup(ns_prefix), strdup(ns_entry)); - } -} - -/* PHP: composer.json — name + PSR-4 autoload */ -static void parse_composer_json(const char *source, int source_len, const char *rel_path, - cbm_pkg_entries_t *entries) { - yyjson_doc *doc = yyjson_read(source, (size_t)source_len, 0); - if (!doc) { - return; - } - yyjson_val *root = yyjson_doc_get_root(doc); - if (!yyjson_is_obj(root)) { - yyjson_doc_free(doc); - return; - } - - char *dir = path_dirname(rel_path); - - /* Register package name → directory */ - yyjson_val *name_val = yyjson_obj_get(root, "name"); - if (yyjson_is_str(name_val)) { - const char *name = yyjson_get_str(name_val); - if (name && name[0] != '\0') { - pkg_entries_push(entries, strdup(name), strdup(dir)); - } - } - - extract_psr4(root, "autoload", dir, entries); - extract_psr4(root, "autoload-dev", dir, entries); - - free(dir); - yyjson_doc_free(doc); -} - -/* Dart: pubspec.yaml — name */ -static void parse_pubspec_yaml(const char *source, int source_len, const char *rel_path, - cbm_pkg_entries_t *entries) { - cbm_yaml_node_t *root = cbm_yaml_parse(source, source_len); - if (!root) { - return; - } - const char *name = cbm_yaml_get_str(root, "name"); - if (name && name[0] != '\0') { - char *dir = path_dirname(rel_path); - char entry[PKGMAP_PATH_BUF]; - snprintf(entry, sizeof(entry), "%s%slib", dir[0] ? dir : "", dir[0] ? "/" : ""); - pkg_entries_push(entries, strdup(name), strdup(entry)); - free(dir); - } - cbm_yaml_free(root); -} - -/* Extract text content of an XML tag at position p. Returns heap string or NULL. - * p must point to the char after the opening tag's '>'. */ -static char *xml_tag_content(const char *p, const char *end) { - const char *s = p; - while (p < end && *p != '<') { - p++; - } - if (p <= s) { - return NULL; - } - return cbm_strndup(s, (size_t)(p - s)); -} - -/* Java: pom.xml — + */ -/* Scan pom.xml for a top-level XML tag (outside ). Returns heap string or NULL. */ -static char *pom_find_tag(const char *source, const char *end, const char *tag, int tag_len) { - const char *p = source; - bool in_parent = false; - while (p < end) { - if (at_prefix(p, end, "", XML_PARENT_OPEN)) { - in_parent = true; - } - if (at_prefix(p, end, "", XML_PARENT_CLOSE)) { - in_parent = false; - } - if (!in_parent && at_prefix(p, end, tag, tag_len)) { - return xml_tag_content(p + tag_len, end); - } - p++; - } - return NULL; -} - -static void parse_pom_xml(const char *source, int source_len, const char *rel_path, - cbm_pkg_entries_t *entries) { - const char *end = source + source_len; - char *group_id = pom_find_tag(source, end, "", XML_GROUP_OPEN); - char *artifact_id = pom_find_tag(source, end, "", XML_ARTIFACT_OPEN); - - if (group_id && artifact_id) { - /* Map: "com.myorg.myapp" → src/main/java directory */ - char pkg_name[PKGMAP_PATH_BUF]; - snprintf(pkg_name, sizeof(pkg_name), "%s.%s", group_id, artifact_id); - char *dir = path_dirname(rel_path); - char entry[PKGMAP_PATH_BUF]; - snprintf(entry, sizeof(entry), "%s%ssrc/main/java", dir[0] ? dir : "", dir[0] ? "/" : ""); - pkg_entries_push(entries, strdup(pkg_name), strdup(entry)); - - /* Also register just the groupId for package-level imports */ - char grp_entry[PKGMAP_PATH_BUF]; - snprintf(grp_entry, sizeof(grp_entry), "%s%ssrc/main/java", dir[0] ? dir : "", - dir[0] ? "/" : ""); - pkg_entries_push(entries, strdup(group_id), strdup(grp_entry)); - free(dir); - } - - free(group_id); - free(artifact_id); -} - -/* Gradle: build.gradle / build.gradle.kts — group = '...' */ -static void parse_build_gradle(const char *source, int source_len, const char *rel_path, - cbm_pkg_entries_t *entries) { - const char *end = source + source_len; - /* Look for group = '...' or group '...' or group = "..." */ - const char *val = find_line_value(source, source_len, "group"); - if (!val) { - return; - } - char *group = extract_quoted(val, end); - if (!group) { - return; - } - char *dir = path_dirname(rel_path); - /* Check for src/main/java or src/main/kotlin */ - char entry[PKGMAP_PATH_BUF]; - snprintf(entry, sizeof(entry), "%s%ssrc/main/java", dir[0] ? dir : "", dir[0] ? "/" : ""); - pkg_entries_push(entries, group, strdup(entry)); - free(dir); -} - -/* Elixir: mix.exs — app: :name */ -static void parse_mix_exs(const char *source, int source_len, const char *rel_path, - cbm_pkg_entries_t *entries) { - const char *end = source + source_len; - /* Look for app: :app_name */ - const char *val = find_line_value(source, source_len, "app:"); - if (!val) { - return; - } - while (val < end && (*val == ' ' || *val == '\t')) { - val++; - } - if (val >= end || *val != ':') { - return; - } - val++; /* skip : */ - const char *start = val; - while (val < end && *val != ',' && *val != '\n' && *val != ' ' && *val != ')') { - val++; - } - if (val <= start) { - return; - } - char *app_name = cbm_strndup(start, (size_t)(val - start)); - char *dir = path_dirname(rel_path); - char entry[PKGMAP_PATH_BUF]; - snprintf(entry, sizeof(entry), "%s%slib/%s", dir[0] ? dir : "", dir[0] ? "/" : "", app_name); - /* Register with colon prefix as Elixir uses :atom syntax */ - char atom_name[PKGMAP_PATH_BUF]; - snprintf(atom_name, sizeof(atom_name), "%s", app_name); - pkg_entries_push(entries, strdup(atom_name), strdup(entry)); - free(app_name); - free(dir); -} - -/* Ruby: *.gemspec — spec.name = '...' */ -static void parse_gemspec(const char *source, int source_len, const char *rel_path, - cbm_pkg_entries_t *entries) { - const char *end = source + source_len; - /* Try spec.name, s.name, gem.name patterns */ - static const char *patterns[] = {".name", NULL}; - for (int i = 0; patterns[i]; i++) { - const char *p = source; - while (p < end) { - const char *found = strstr(p, patterns[i]); - if (!found || found >= end) { - break; - } - char *name = extract_quoted(found + strlen(patterns[i]), end); - if (name) { - char *dir = path_dirname(rel_path); - char entry[PKGMAP_PATH_BUF]; - snprintf(entry, sizeof(entry), "%s%slib/%s", dir[0] ? dir : "", dir[0] ? "/" : "", - name); - pkg_entries_push(entries, name, strdup(entry)); - free(dir); - return; - } - p = found + SKIP_ONE; - } - } -} - -/* Zig: build.zig — named modules for `@import("name")`. A module is declared - * with `b.addModule("name", .{ .root_source_file = b.path("src/root.zig") })`, - * or created into a variable (`const m = b.createModule(.{ .root_source_file = - * b.path(...) })`) that is registered under a name by `x.addImport("name", m)` - * or `.imports = &.{ .{ .name = "name", .module = m } }`. Keys carry a "zig:" - * prefix so a Zig module never collides with another ecosystem's package. - * - * A module of a dependency (`b.dependency("pkg", .{}).module("x")`) resolves - * when build.zig.zon declares the dependency by an in-repo `.path`: the - * dependency's build.zig `addModule("x", ...)` names its root. A dependency - * fetched by URL has no source in the repo and stays unmapped. Entries for - * this: "zigdep:|" -> the dependency's directory (build.zig.zon in - * ), "zigmod:|" -> module x of the build.zig in , and - * "zig:" -> "@||", resolved by cbm_pkgmap_build. */ - -static const char *bz_find(const char *p, const char *end, const char *needle) { - size_t n = strlen(needle); - for (; p && p + n <= end; p++) { - if (memcmp(p, needle, n) == 0) { - return p; - } - } - return NULL; -} - -static const char *bz_skip_ws(const char *p, const char *end) { - while (p < end && (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r')) { - p++; - } - return p; -} - -static size_t bz_ident_len(const char *p, const char *end) { - size_t n = 0; - while (p + n < end && (p[n] == '_' || (p[n] >= 'a' && p[n] <= 'z') || - (p[n] >= 'A' && p[n] <= 'Z') || (n > 0 && p[n] >= '0' && p[n] <= '9'))) { - n++; - } - return n; -} - -/* A "..." string at p (after whitespace): heap copy, *after past the quote. */ -static char *bz_quoted(const char *p, const char *end, const char **after) { - p = bz_skip_ws(p, end); - if (p >= end || *p != '"') { - return NULL; - } - const char *close = memchr(p + SKIP_ONE, '"', (size_t)(end - p - SKIP_ONE)); - if (!close) { - return NULL; - } - *after = close + SKIP_ONE; - return cbm_strndup(p + SKIP_ONE, (size_t)(close - p - SKIP_ONE)); -} - -/* The `.root_source_file = x.path("...zig")` of the statement starting at p. */ -static char *bz_root_source(const char *p, const char *end) { - const char *semi = memchr(p, ';', (size_t)(end - p)); - const char *stop = semi ? semi : end; - const char *r = bz_find(p, stop, "root_source_file"); - const char *call = r ? bz_find(r, stop, ".path(") : NULL; - const char *after = NULL; - char *path = call ? bz_quoted(call + strlen(".path("), stop, &after) : NULL; - if (path && (!cbm_str_ends_with(path, ".zig") || strstr(path, ".."))) { - free(path); - path = NULL; - } - return path; -} - -/* Root source of module variable `ident` (from its `const ident = ...;`). */ -static char *bz_var_source(const char *src, const char *end, const char *ident, size_t len) { - for (const char *p = bz_find(src, end, "const "); p; p = bz_find(p + SKIP_ONE, end, "const ")) { - const char *q = bz_skip_ws(p + strlen("const "), end); - if (bz_ident_len(q, end) != len || memcmp(q, ident, len) != 0) { - continue; - } - q = bz_skip_ws(q + len, end); - if (q < end && *q == '=') { - return bz_root_source(q, end); - } - } - return NULL; -} - -/* The module variable at p (`m` followed by `)`/`,`/`}`): its root source. */ -static char *bz_module_arg_source(const char *src, const char *p, const char *end) { - p = bz_skip_ws(p, end); - size_t len = bz_ident_len(p, end); - const char *q = bz_skip_ws(p + len, end); - if (len == 0 || q >= end || (*q != ')' && *q != ',' && *q != '}')) { - return NULL; - } - return bz_var_source(src, end, p, len); -} - -static void bz_push(cbm_pkg_entries_t *entries, const char *rel_path, const char *name, - const char *path, bool exported) { - char key[PKGMAP_PATH_BUF]; - snprintf(key, sizeof(key), "zig:%s", name); - char *dir = path_dirname(rel_path); - char mod_key[PKGMAP_PATH_BUF]; - snprintf(mod_key, sizeof(mod_key), "zigmod:%s|%s", dir, name); - char *mod_entry = exported ? join_and_strip(dir, path) : NULL; - if (mod_entry) { - pkg_entries_push(entries, strdup(mod_key), mod_entry); - } - char *entry = join_and_strip(dir, path); - free(dir); - if (entry) { - pkg_entries_push(entries, strdup(key), entry); - } -} - -/* End of the expression at p: the `,` `)` `}` or `;` that closes it. */ -static const char *bz_expr_end(const char *p, const char *end) { - int nest = 0; - for (; p < end; p++) { - if (*p == '"') { - const char *close = memchr(p + SKIP_ONE, '"', (size_t)(end - p - SKIP_ONE)); - p = close ? close : end; - } else if (*p == '(' || *p == '{' || *p == '[') { - nest++; - } else if ((*p == ')' || *p == '}' || *p == ']') && nest > 0) { - nest--; - } else if (nest == 0 && (*p == ',' || *p == ')' || *p == '}' || *p == ';')) { - return p; - } - } - return end; -} - -/* The value expression [*vb, *ve) of `const ident = ...;`, or false. */ -static bool bz_var_value(const char *src, const char *end, const char *ident, size_t len, - const char **vb, const char **ve) { - for (const char *p = bz_find(src, end, "const "); p; p = bz_find(p + SKIP_ONE, end, "const ")) { - const char *q = bz_skip_ws(p + strlen("const "), end); - if (bz_ident_len(q, end) != len || memcmp(q, ident, len) != 0) { - continue; - } - q = bz_skip_ws(q + len, end); - if (q < end && *q == '=') { - *vb = bz_skip_ws(q + SKIP_ONE, end); - *ve = bz_expr_end(*vb, end); - return true; - } - } - return false; -} - -/* The expression [b, e), or a bare identifier's `const` value. */ -static void bz_deref(const char *src, const char *end, const char **b, const char **e) { - size_t len = bz_ident_len(*b, *e); - if (len > 0 && bz_skip_ws(*b + len, *e) == *e) { - const char *vb = NULL; - const char *ve = NULL; - if (bz_var_value(src, end, *b, len, &vb, &ve)) { - *b = vb; - *e = ve; - } - } -} - -/* A module argument `dep.module("x")` / `b.dependency("pkg", ..).module("x")` - * (or a `const` holding one): push "zig:" -> "@||". */ -static void bz_push_dep_module(cbm_pkg_entries_t *entries, const char *rel_path, const char *name, - const char *src, const char *p, const char *end) { - const char *b = bz_skip_ws(p, end); - const char *e = bz_expr_end(b, end); - bz_deref(src, end, &b, &e); - const char *m = NULL; - for (const char *q = bz_find(b, e, ".module("); q; q = bz_find(q + SKIP_ONE, e, ".module(")) { - m = q; - } - const char *after = NULL; - char *mod = m ? bz_quoted(m + strlen(".module("), e, &after) : NULL; - const char *db = b; - const char *de = m; - if (mod) { - bz_deref(src, end, &db, &de); - } - const char *d = mod ? bz_find(db, de, ".dependency(") : NULL; - char *pkg = d ? bz_quoted(d + strlen(".dependency("), de, &after) : NULL; - if (pkg) { - char key[PKGMAP_PATH_BUF]; - char val[PKGMAP_PATH_BUF]; - char *dir = path_dirname(rel_path); - snprintf(key, sizeof(key), "zig:%s", name); - snprintf(val, sizeof(val), "@%s|%s|%s", dir, pkg, mod); - free(dir); - pkg_entries_push(entries, strdup(key), strdup(val)); - } - free(mod); - free(pkg); -} - -/* build.zig.zon: `.dependencies = .{ .pkg = .{ .path = "libs/pkg" }, ... }`. */ -static void parse_build_zig_zon(const char *source, int source_len, const char *rel_path, - cbm_pkg_entries_t *entries) { - /* Blank `//` comments (outside strings), so a commented-out dependency - * is not one. */ - char *text = cbm_strndup(source, (size_t)source_len); - if (!text) { - return; - } - bool in_str = false; - for (int i = 0; i < source_len; i++) { - if (in_str) { - in_str = !(text[i] == '"' && text[i - SKIP_ONE] != '\\'); - } else if (text[i] == '"') { - in_str = true; - } else if (text[i] == '/' && i + SKIP_ONE < source_len && text[i + SKIP_ONE] == '/') { - while (i < source_len && text[i] != '\n') { - text[i++] = ' '; - } - } - } - const char *end = text + source_len; - const char *deps = bz_find(text, end, ".dependencies"); - const char *open = deps ? bz_find(deps, end, ".{") : NULL; - const char *close = open ? bz_expr_end(open + PAIR_LEN, end) : NULL; - char *dir = path_dirname(rel_path); - for (const char *p = open ? open + PAIR_LEN : NULL; p && close && p < close;) { - p = bz_skip_ws(p, close); - if (p >= close || *p != '.') { - break; - } - p++; - const char *after = p; - char *name = NULL; - if (*p == '@') { - name = bz_quoted(p + SKIP_ONE, close, &after); - } else { - size_t n = bz_ident_len(p, close); - name = n > 0 ? cbm_strndup(p, n) : NULL; - after = p + n; - } - const char *eq = bz_skip_ws(after, close); - const char *body = eq < close && *eq == '=' ? bz_skip_ws(eq + SKIP_ONE, close) : close; - const char *body_end = bz_expr_end(body, close); - const char *path = name ? bz_find(body, body_end, ".path") : NULL; - const char *peq = path ? bz_skip_ws(path + strlen(".path"), body_end) : NULL; - char *rel = peq && peq < body_end && *peq == '=' ? bz_quoted(peq + SKIP_ONE, body_end, &after) - : NULL; - if (rel) { - size_t rl = strlen(rel); - while (rl > 0 && rel[rl - SKIP_ONE] == '/') { - rel[--rl] = '\0'; - } - const char *r = rel[0] == '.' && rel[SKIP_ONE] == '/' ? rel + PAIR_LEN : rel; - char key[PKGMAP_PATH_BUF]; - char val[PKGMAP_PATH_BUF]; - snprintf(key, sizeof(key), "zigdep:%s|%s", dir, name); - if (dir[0] == '\0') { - snprintf(val, sizeof(val), "%s", r); - } else { - snprintf(val, sizeof(val), "%s/%s", dir, r); - } - pkg_entries_push(entries, strdup(key), strdup(val)); - } - free(name); - free(rel); - p = body_end < close && *body_end == ',' ? body_end + SKIP_ONE : close; - } - free(dir); - free(text); -} - -static void parse_build_zig(const char *source, int source_len, const char *rel_path, - cbm_pkg_entries_t *entries) { - const char *end = source + source_len; - const char *after = NULL; - for (const char *p = bz_find(source, end, "addModule("); p; - p = bz_find(p + SKIP_ONE, end, "addModule(")) { - char *name = bz_quoted(p + strlen("addModule("), end, &after); - char *path = name ? bz_root_source(after, end) : NULL; - if (path) { - bz_push(entries, rel_path, name, path, true); - } - free(name); - free(path); - } - for (const char *p = bz_find(source, end, "addImport("); p; - p = bz_find(p + SKIP_ONE, end, "addImport(")) { - char *name = bz_quoted(p + strlen("addImport("), end, &after); - const char *comma = name ? bz_skip_ws(after, end) : NULL; - char *path = comma && comma < end && *comma == ',' - ? bz_module_arg_source(source, comma + SKIP_ONE, end) - : NULL; - if (path) { - bz_push(entries, rel_path, name, path, false); - } else if (comma && comma < end && *comma == ',') { - bz_push_dep_module(entries, rel_path, name, source, comma + SKIP_ONE, end); - } - free(name); - free(path); - } - for (const char *p = bz_find(source, end, ".name"); p; p = bz_find(p + SKIP_ONE, end, ".name")) { - const char *eq = bz_skip_ws(p + strlen(".name"), end); - char *name = eq < end && *eq == '=' ? bz_quoted(eq + SKIP_ONE, end, &after) : NULL; - const char *close = name ? memchr(after, '}', (size_t)(end - after)) : NULL; - const char *mod = close ? bz_find(after, close, ".module") : NULL; - const char *meq = mod ? bz_skip_ws(mod + strlen(".module"), close) : NULL; - char *path = meq && meq < close && *meq == '=' - ? bz_module_arg_source(source, meq + SKIP_ONE, end) - : NULL; - if (path) { - bz_push(entries, rel_path, name, path, false); - } else if (meq && meq < close && *meq == '=') { - bz_push_dep_module(entries, rel_path, name, source, meq + SKIP_ONE, end); - } - free(name); - free(path); - } -} - -/* ── Public: manifest detection + parsing ──────────────────────── */ - -bool cbm_pkgmap_try_parse(const char *basename, const char *rel_path, const char *source, - int source_len, cbm_pkg_entries_t *entries) { - if (!basename || !source || source_len <= 0) { - return false; - } - - if (strcmp(basename, "package.json") == 0) { - parse_package_json(source, source_len, rel_path, entries); - return true; - } - if (strcmp(basename, "go.mod") == 0) { - parse_go_mod(source, source_len, rel_path, entries); - return true; - } - if (strcmp(basename, "Cargo.toml") == 0) { - parse_cargo_toml(source, source_len, rel_path, entries); - return true; - } - if (strcmp(basename, "pyproject.toml") == 0) { - parse_pyproject_toml(source, source_len, rel_path, entries); - return true; - } - if (strcmp(basename, "composer.json") == 0) { - parse_composer_json(source, source_len, rel_path, entries); - return true; - } - if (strcmp(basename, "pubspec.yaml") == 0) { - parse_pubspec_yaml(source, source_len, rel_path, entries); - return true; - } - if (strcmp(basename, "pom.xml") == 0) { - parse_pom_xml(source, source_len, rel_path, entries); - return true; - } - if (strcmp(basename, "build.gradle") == 0 || strcmp(basename, "build.gradle.kts") == 0) { - parse_build_gradle(source, source_len, rel_path, entries); - return true; - } - if (strcmp(basename, "mix.exs") == 0) { - parse_mix_exs(source, source_len, rel_path, entries); - return true; - } - if (cbm_str_ends_with(basename, ".gemspec")) { - parse_gemspec(source, source_len, rel_path, entries); - return true; - } - if (strcmp(basename, "build.zig") == 0) { - parse_build_zig(source, source_len, rel_path, entries); - return true; - } - if (strcmp(basename, "build.zig.zon") == 0) { - parse_build_zig_zon(source, source_len, rel_path, entries); - return true; - } - return false; -} - -/* ── Merge: per-worker entries → hash table ────────────────────── */ - -/* A composer.json `autoload.psr-4` key: a namespace prefix ending in '\\'. */ -static bool pkgmap_is_psr4_prefix(const char *pkg_name) { - size_t n = pkg_name ? strlen(pkg_name) : 0; - return n > 0 && pkg_name[n - SKIP_ONE] == '\\'; -} - -/* The entry_rel recorded for key, or NULL. */ -static const char *pkgmap_raw_entry(cbm_pkg_entries_t *worker_entries, int worker_count, - const char *key) { - for (int w = 0; w < worker_count; w++) { - for (int i = 0; i < worker_entries[w].count; i++) { - if (strcmp(worker_entries[w].items[i].pkg_name, key) == 0) { - return worker_entries[w].items[i].entry_rel; - } - } - } - return NULL; -} - -/* Zig "zig:" -> "@||": module x of the build.zig in the - * directory build.zig.zon in gives dependency pkg. Returns the number - * of entries added. */ -static int pkgmap_resolve_zig_deps(CBMHashTable *map, cbm_pkg_entries_t *worker_entries, - int worker_count) { - int added = 0; - for (int w = 0; w < worker_count; w++) { - for (int i = 0; i < worker_entries[w].count; i++) { - const char *name = worker_entries[w].items[i].pkg_name; - const char *v = worker_entries[w].items[i].entry_rel; - const char *bar1 = v[0] == '@' ? strchr(v, '|') : NULL; - const char *bar2 = bar1 ? strchr(bar1 + SKIP_ONE, '|') : NULL; - if (!bar2 || cbm_ht_has(map, name)) { - continue; - } - char key[PKGMAP_PATH_BUF]; - snprintf(key, sizeof(key), "zigdep:%.*s", (int)(bar2 - v - SKIP_ONE), v + SKIP_ONE); - const char *dep_dir = pkgmap_raw_entry(worker_entries, worker_count, key); - if (!dep_dir) { - continue; - } - snprintf(key, sizeof(key), "zigmod:%s|%s", dep_dir, bar2 + SKIP_ONE); - const char *qn = (const char *)cbm_ht_get(map, key); - if (qn) { - cbm_ht_set(map, strdup(name), strdup(qn)); - added++; - } - } - } - return added; -} - -CBMHashTable *cbm_pkgmap_build(cbm_pkg_entries_t *worker_entries, int worker_count, - const char *project_name) { - /* Count total entries */ - int total = 0; - for (int w = 0; w < worker_count; w++) { - total += worker_entries[w].count; - } - if (total == 0) { - return NULL; - } - - CBMHashTable *map = cbm_ht_create(PKGMAP_HT_INIT); - int merged = 0; - - for (int w = 0; w < worker_count; w++) { - cbm_pkg_entries_t *we = &worker_entries[w]; - for (int i = 0; i < we->count; i++) { - /* Zig dependency entries are resolved below. */ - if (we->items[i].entry_rel[0] == '@' || - strncmp(we->items[i].pkg_name, "zigdep:", strlen("zigdep:")) == 0) { - continue; - } - /* Convert entry_rel to QN: project.dir.parts. A PSR-4 namespace - * prefix (key ends in '\\') keeps its repo-relative DIRECTORY - * instead: its readers map `Prefix\Sub\Class` onto a file path - * (/Sub/Class.php), which a dotted QN cannot express (#1186). */ - const char *entry_rel = we->items[i].entry_rel; - char *qn = pkgmap_is_psr4_prefix(we->items[i].pkg_name) - ? cbm_strndup(entry_rel, strlen(entry_rel)) - : cbm_pipeline_fqn_module(project_name, entry_rel); - if (!qn) { - continue; - } - - /* Check for duplicate — first wins */ - if (cbm_ht_has(map, we->items[i].pkg_name)) { - free(qn); - continue; - } - - /* Transfer ownership: key = strdup'd pkg_name, value = qn */ - char *key = strdup(we->items[i].pkg_name); - cbm_ht_set(map, key, qn); - merged++; - } - } - - merged += pkgmap_resolve_zig_deps(map, worker_entries, worker_count); - if (merged == 0) { - cbm_ht_free(map); - return NULL; - } - cbm_log_info("pkgmap.build", "entries", cbm_log_itoa(merged)); - return map; -} - -/* Returns true if basename is a package manifest we know how to parse. - * Used by the filesystem walker; cbm_pkgmap_try_parse is the source of - * truth for which basenames produce entries. */ -static bool is_pkgmap_manifest_basename(const char *basename) { - if (!basename) { - return false; - } - if (strcmp(basename, "package.json") == 0 || strcmp(basename, "go.mod") == 0 || - strcmp(basename, "Cargo.toml") == 0 || strcmp(basename, "pyproject.toml") == 0 || - strcmp(basename, "composer.json") == 0 || strcmp(basename, "pubspec.yaml") == 0 || - strcmp(basename, "pom.xml") == 0 || strcmp(basename, "build.gradle") == 0 || - strcmp(basename, "build.gradle.kts") == 0 || strcmp(basename, "mix.exs") == 0 || - strcmp(basename, "build.zig") == 0 || strcmp(basename, "build.zig.zon") == 0) { - return true; - } - return cbm_str_ends_with(basename, ".gemspec"); -} - -/* Stat a path, skipping symlinks. Returns 0 on success, -1 to skip. - * On POSIX, lstat + S_ISLNK avoids following symlink cycles. On Windows - * we use the UTF-8-safe wide stat (mirroring discover.c's wide_stat); - * reparse points (junctions/symlinks) are detected separately by - * pkgmap_is_reparse_point below before we descend. Mirrors discover.c's - * safe_stat. */ -static int pkgmap_safe_stat(const char *abs_path, struct stat *st) { -#ifdef _WIN32 - wchar_t *wpath = cbm_utf8_to_wide(abs_path); - if (!wpath) { - return CBM_NOT_FOUND; - } - struct _stat64 wst; - int ret = _wstat64(wpath, &wst); - free(wpath); - if (ret != 0) { - return CBM_NOT_FOUND; - } - st->st_mode = wst.st_mode; - st->st_size = wst.st_size; - st->st_mtime = wst.st_mtime; - return 0; -#else - if (lstat(abs_path, st) != 0) { - return CBM_NOT_FOUND; - } - if (S_ISLNK(st->st_mode)) { - return CBM_NOT_FOUND; - } - return 0; -#endif -} - -/* True if abs_path is a Windows reparse point (directory junction or - * symlink). Following these is the documented cause of the historic CI - * hang, so we skip them before recursing. On POSIX this is a no-op - * (symlinks are already filtered by pkgmap_safe_stat's S_ISLNK check). - * Note: the PKGMAP_WALK_MAX_DEPTH bound below is the hard termination - * guarantee; this check is a best-effort early skip on top of it. */ -#ifdef _WIN32 -static bool pkgmap_is_reparse_point(const char *abs_path) { - wchar_t *wpath = cbm_utf8_to_wide(abs_path); - if (!wpath) { - return false; - } - DWORD attrs = GetFileAttributesW(wpath); - free(wpath); - if (attrs == INVALID_FILE_ATTRIBUTES) { - return false; - } - return (attrs & FILE_ATTRIBUTE_REPARSE_POINT) != 0; -} -#endif - -/* Recursive filesystem walker that finds and parses package manifest - * files independently of the main discovery filter. The main discovery - * filter intentionally hides package.json / composer.json etc. from - * code indexing (they're config, not source), but pass_pkgmap still - * needs to read them to resolve workspace imports. Skips directories - * matched by the shared cbm_should_skip_dir helper so we don't walk - * node_modules, .git, build, etc. Returns the number of manifests - * parsed, accumulated across the whole walk. - * - * Cross-platform: uses the portable cbm_opendir/cbm_readdir/cbm_closedir - * API (same as src/discover/discover.c) and the symlink-skipping - * pkgmap_safe_stat. Termination is guaranteed by the PKGMAP_WALK_MAX_DEPTH - * recursion bound — even directory junctions / symlink cycles cannot make - * it hang. On Windows we additionally skip reparse points before - * descending as a best-effort early-out. */ -static int pkgmap_walk_dir(const char *abs_dir, const char *rel_dir, cbm_pkg_entries_t *entries, - int depth, char **excluded_dirs, int excluded_count) { - if (depth >= PKGMAP_WALK_MAX_DEPTH) { - cbm_log_info("pkgmap.walk", "depth_cap", rel_dir && rel_dir[0] ? rel_dir : "."); - return 0; - } - cbm_dir_t *dir = cbm_opendir(abs_dir); - if (!dir) { - return 0; - } - int parsed = 0; - cbm_dirent_t *entry; - while ((entry = cbm_readdir(dir)) != NULL) { - const char *name = entry->name; - if (name[0] == '.' && (name[1] == '\0' || (name[1] == '.' && name[2] == '\0'))) { - continue; - } - char abs_path[PKGMAP_PATH_BUF]; - char rel_path[PKGMAP_PATH_BUF]; - snprintf(abs_path, sizeof(abs_path), "%s/%s", abs_dir, name); - if (rel_dir && rel_dir[0]) { - snprintf(rel_path, sizeof(rel_path), "%s/%s", rel_dir, name); - } else { - snprintf(rel_path, sizeof(rel_path), "%s", name); - } - struct stat st; - if (pkgmap_safe_stat(abs_path, &st) != 0) { - continue; - } - if (S_ISDIR(st.st_mode)) { - if (cbm_should_skip_dir(name, CBM_MODE_FULL) || - cbm_pipeline_relpath_is_excluded(rel_path, excluded_dirs, excluded_count)) { - continue; - } -#ifdef _WIN32 - /* Don't follow Windows directory junctions — they can form - * cycles. The depth bound is the hard guarantee; this just - * avoids wasted descent. (POSIX symlinks are already skipped by - * pkgmap_safe_stat's S_ISLNK check.) */ - if (pkgmap_is_reparse_point(abs_path)) { - continue; - } -#endif - parsed += pkgmap_walk_dir(abs_path, rel_path, entries, depth + 1, excluded_dirs, - excluded_count); - continue; - } - if (!S_ISREG(st.st_mode)) { - continue; - } - if (!is_pkgmap_manifest_basename(name)) { - continue; - } - int source_len = 0; - char *source = pkgmap_read_file(abs_path, &source_len); - if (!source) { - continue; - } - if (cbm_pkgmap_try_parse(name, rel_path, source, source_len, entries)) { - parsed++; - } - free(source); - } - cbm_closedir(dir); - return parsed; -} - -/* Scan a repository for package manifest files via the filesystem - * walker above. Always-available companion to the parallel path's - * per-worker manifest parsing, which is bound to whatever `files[]` - * the discoverer produces and therefore misses ignored manifests like - * package.json. NULL-safe; returns 0 entries when repo_path is unset. - * - * Cross-platform: the walk runs on every platform via pkgmap_walk_dir, - * which is depth-bounded (PKGMAP_WALK_MAX_DEPTH) and skips symlinks / - * Windows reparse points, so it cannot hang on directory junctions. - * This is what lets bare workspace imports (e.g. "@org/pkg" declared in - * an ignored package.json) resolve on Windows as well as POSIX. */ -int cbm_pkgmap_scan_repo(const char *repo_path, cbm_pkg_entries_t *entries, char **excluded_dirs, - int excluded_count) { - if (!repo_path || !entries) { - return 0; - } - int parsed = pkgmap_walk_dir(repo_path, "", entries, 0, excluded_dirs, excluded_count); - cbm_log_info("pkgmap.scan_repo", "manifests", cbm_log_itoa(parsed)); - return parsed; -} - -static void pkgmap_free_entry(const char *key, void *value, void *userdata) { - (void)userdata; - free((void *)key); - free(value); -} - -void cbm_pkgmap_free(CBMHashTable *pkgmap) { - if (!pkgmap) { - return; - } - cbm_ht_foreach(pkgmap, pkgmap_free_entry, NULL); - cbm_ht_free(pkgmap); -} - -/* ── Resolver ──────────────────────────────────────────────────── */ - -/* Try slash-based prefix matching (Go: github.com/foo/bar/pkg/utils). - * Returns heap QN or NULL. */ -static char *resolve_slash_prefix(CBMHashTable *map, const char *module_path) { - char *buf = strdup(module_path); - if (!buf) { - return NULL; - } - for (char *slash = buf + strlen(buf) - SKIP_ONE; slash > buf; slash--) { - if (*slash != '/') { - continue; - } - *slash = '\0'; - const char *base_qn = (const char *)cbm_ht_get(map, buf); - if (!base_qn) { - continue; - } - const char *subpath = module_path + (size_t)(slash - buf) + SKIP_ONE; - char result[PKGMAP_PATH_BUF]; - snprintf(result, sizeof(result), "%s.%s", base_qn, subpath); - /* Replace / with . in the appended part */ - for (char *c = result + strlen(base_qn) + SKIP_ONE; *c; c++) { - if (*c == '/') { - *c = '.'; - } - } - free(buf); - return strdup(result); - } - free(buf); - return NULL; -} - -/* Try dot-based prefix matching (Java: com.myorg.pkg.Foo). - * Returns heap QN or NULL. */ -static char *resolve_dot_prefix(CBMHashTable *map, const char *module_path, - const char *project_name) { - char *buf = strdup(module_path); - if (!buf) { - return NULL; - } - for (char *dot = buf + strlen(buf) - SKIP_ONE; dot > buf; dot--) { - if (*dot != '.') { - continue; - } - *dot = '\0'; - const char *base_qn = (const char *)cbm_ht_get(map, buf); - if (!base_qn) { - continue; - } - const char *subpath = module_path + (size_t)(dot - buf) + SKIP_ONE; - char subpath_slashed[PKGMAP_PATH_BUF]; - snprintf(subpath_slashed, sizeof(subpath_slashed), "%s", subpath); - for (char *c = subpath_slashed; *c; c++) { - if (*c == '.') { - *c = '/'; - } - } - char result[PKGMAP_PATH_BUF]; - snprintf(result, sizeof(result), "%s/%s", base_qn, subpath_slashed); - free(buf); - return cbm_pipeline_fqn_module(project_name, result); - } - free(buf); - return NULL; -} - -/* Try backslash-based prefix matching (PHP PSR-4: App\\Controllers\\Foo). - * Returns heap QN or NULL. */ -static char *resolve_backslash_prefix(CBMHashTable *map, const char *module_path, - const char *project_name) { - char *buf = strdup(module_path); - if (!buf) { - return NULL; - } - for (char *bs = buf + strlen(buf) - SKIP_ONE; bs > buf; bs--) { - if (*bs != '\\') { - continue; - } - *bs = '\0'; - char prefix[PKGMAP_PATH_BUF]; - snprintf(prefix, sizeof(prefix), "%s\\", buf); - const char *base_dir = (const char *)cbm_ht_get(map, prefix); - if (!base_dir) { - continue; - } - const char *subpath = module_path + (size_t)(bs - buf) + SKIP_ONE; - char path_result[PKGMAP_PATH_BUF]; - snprintf(path_result, sizeof(path_result), "%s%s%s", base_dir, base_dir[0] ? "/" : "", - subpath); - for (char *c = path_result; *c; c++) { - if (*c == '\\') { - *c = '/'; - } - } - free(buf); - return cbm_pipeline_fqn_module(project_name, path_result); - } - free(buf); - return NULL; -} - -char *cbm_pipeline_resolve_module(const cbm_pipeline_ctx_t *ctx, const char *source_rel, - const char *module_path) { - if (!ctx || !module_path) { - return cbm_pipeline_fqn_module(ctx ? ctx->project_name : NULL, module_path); - } - - /* 1. Try relative import resolution (existing logic) */ - char *resolved = cbm_pipeline_resolve_relative_import(source_rel, module_path); - if (resolved) { - /* The relative resolver has already removed an explicit JS/TS file - * extension. Treat the remaining path as a module path verbatim so a - * dotted extensionless basename such as `featureX.engine` is not - * stripped a second time by cbm_pipeline_fqn_module. */ - char *qn = cbm_pipeline_fqn_folder(ctx->project_name, resolved); - free(resolved); - return qn; - } - - /* 1b. Try build-tool path aliases (tsconfig/jsconfig paths today; - * other loaders can register here later). Independent of pkgmap. */ - if (ctx->path_aliases && source_rel) { - const cbm_path_alias_map_t *amap = - cbm_path_alias_find_for_file(ctx->path_aliases, source_rel); - if (amap) { - char *aliased = cbm_path_alias_resolve(amap, module_path); - if (aliased) { - char *qn = cbm_pipeline_fqn_module(ctx->project_name, aliased); - free(aliased); - return qn; - } - } - } - - /* 2. No pkgmap → fall through immediately */ - CBMHashTable *pkgmap = ctx->pkgmap; - if (!pkgmap) { - return cbm_pipeline_fqn_module(ctx->project_name, module_path); - } - - /* 2b. Zig: a named `@import("name")` is a build.zig module, never another - * ecosystem's package of the same name. */ - if (source_rel && cbm_str_ends_with(source_rel, ".zig")) { - char key[PKGMAP_PATH_BUF]; - snprintf(key, sizeof(key), "zig:%s", module_path); - const char *zig_qn = (const char *)cbm_ht_get(pkgmap, key); - return zig_qn ? strdup(zig_qn) : cbm_pipeline_fqn_module(ctx->project_name, module_path); - } - - /* 3. Exact lookup */ - const char *mapped_qn = (const char *)cbm_ht_get(pkgmap, module_path); - if (mapped_qn) { - return strdup(mapped_qn); - } - - /* 4. Prefix matching by separator type */ - char *result = resolve_slash_prefix(pkgmap, module_path); - if (!result) { - result = resolve_dot_prefix(pkgmap, module_path, ctx->project_name); - } - if (!result) { - result = resolve_backslash_prefix(pkgmap, module_path, ctx->project_name); - } - if (result) { - return result; - } - - /* 5. Fallthrough to default resolution */ - return cbm_pipeline_fqn_module(ctx->project_name, module_path); -} - -/* ── Import-target node resolver ─────────────────────────────────── */ - -/* Return the last path segment of an import string, recognizing the separators - * used across languages: '.', '/', '\\', and "::". Returns a pointer into - * `path` (no allocation). */ -static const char *import_last_segment(const char *path) { - const char *seg = path; - for (const char *p = path; *p; p++) { - if (*p == '.' || *p == '/' || *p == '\\' || *p == ':') { - seg = p + 1; - } - } - return seg; -} - -/* Derive a representative imported symbol name from a raw module/use path, - * stripping language decorations so it can be matched against an in-graph node: - * - trailing alias " as X" (Rust/Kotlin) - * - trailing glob "::*" / ".*" (Rust/Kotlin/Java wildcard) - * - brace groups "{a, b, ...}" (Rust grouped use) → first member - * Writes the result into `out` (size `outsz`) and returns it, or NULL if none. - * For grouped/braced forms the first listed symbol is used as the representative - * (the tests only require at least one resolved IMPORTS edge per statement). */ -static const char *import_candidate_symbol(const char *module_path, char *out, size_t outsz) { - if (!module_path || !module_path[0]) { - return NULL; - } - char buf[1024]; - snprintf(buf, sizeof(buf), "%s", module_path); - - /* Brace group: `prefix::{a, b}` → take first member `a`. */ - char *brace = strchr(buf, '{'); - if (brace) { - char *first = brace + 1; - char *end = first; - while (*end && *end != ',' && *end != '}') { - end++; - } - *end = '\0'; - /* Trim whitespace. */ - while (*first == ' ' || *first == '\t') { - first++; - } - char *t = first + strlen(first); - while (t > first && (t[-1] == ' ' || t[-1] == '\t')) { - *--t = '\0'; - } - if (first[0] && strcmp(first, "self") != 0) { - snprintf(out, outsz, "%s", first); - return out; - } - /* `{self, ...}` → fall back to the path before the brace group. */ - *brace = '\0'; - } - - /* Strip trailing " as ". */ - char *as = strstr(buf, " as "); - if (as) { - *as = '\0'; - } - - /* Strip trailing glob and separator noise. */ - size_t len = strlen(buf); - while (len > 0 && (buf[len - 1] == '*' || buf[len - 1] == ':' || buf[len - 1] == '.' || - buf[len - 1] == '/' || buf[len - 1] == '\\' || buf[len - 1] == ' ')) { - buf[--len] = '\0'; - } - if (!buf[0]) { - return NULL; - } - const char *seg = import_last_segment(buf); - if (!seg || !seg[0] || strcmp(seg, "*") == 0) { - return NULL; - } - snprintf(out, outsz, "%s", seg); - return out; -} - -/* True for node labels that represent an importable definition (so a symbol-name - * fallback does not link to, e.g., a Variable or Field). */ -static bool import_targetable_label(const char *label) { - if (!label) { - return false; - } - static const char *ok[] = {"Class", "Interface", "Function", "Method", "Module", "Struct", - "Enum", "Trait", "Type", "File", NULL}; - for (const char **l = ok; *l; l++) { - if (strcmp(*l, label) == 0) { - return true; - } - } - return false; -} - -/* #1934: whether the name-guess import fallbacks — Strategy 1b (sibling file, - * whose label filter admits symbols) and Strategy 3 (symbol name) — may run - * for imports from this language. A Go import path names a package — never a - * function, method - * or field — and every correct Go import resolves in Strategy 1 (module path - * → the package's Folder node); when that misses the import is external and - * the correct result is NO edge. The fallback instead bound the last path - * segment to an arbitrary same-named project symbol (`import "os/exec"` → a - * test harness's exec() method, two imports → a Makefile target). Languages - * whose import genuinely can name a member (Python `from m import f`, Java - * `import com.example.Foo`, Rust `use crate::ops::helper`) keep it. */ -bool cbm_import_symbol_fallback_allowed(CBMLanguage lang) { - return lang != CBM_LANG_GO; -} - -/* Resolve a sibling-file import: a bare path/name (no leading "./") that names - * a file relative to the importer's directory. This covers build/markup - * grammars whose import string is a sibling filename or directory rather than a - * dotted module path: - * - SCSS `@use 'vars'` → sibling `_vars.scss` (partial underscore) - * - Just `import 'common.just'` → sibling `common.just` - * - BitBake `require mypackage.inc` → sibling `mypackage.inc` - * - Meson `subdir('lib')` → `lib/meson.build` - * - func `#include "utils.fc"` → sibling `utils.fc` - * - Pony `use "util"` → sibling `util.pony` - * Builds a path relative to source_rel's directory, then looks up the resulting - * File/Module-node QN (extension is stripped by fqn_module). Returns a borrowed - * node or NULL. Several filename conventions are tried in turn. */ -static const cbm_gbuf_node_t *resolve_sibling_file(const cbm_pipeline_ctx_t *ctx, - const char *source_rel, - const char *source_file_qn, - const char *module_path) { - if (!module_path || !module_path[0]) { - return NULL; - } - /* Directory of the importing file (empty for repo-root files). */ - char *dir = path_dirname(source_rel ? source_rel : ""); - if (!dir) { - return NULL; - } - - /* Candidate relative paths, in priority order. */ - char cands[5][PKGMAP_PATH_BUF]; - int ncand = 0; - const char *base = module_path; - /* Skip a leading "./". */ - if (base[0] == '.' && base[1] == '/') { - base += 2; - } - /* 1. Direct sibling: dir/. */ - snprintf(cands[ncand++], PKGMAP_PATH_BUF, "%s%s%s", dir, dir[0] ? "/" : "", base); - /* 2. SCSS partial: dir/[subdir/]_.scss (underscore-prefixed). */ - { - const char *slash = strrchr(base, '/'); - const char *bn = slash ? slash + 1 : base; - char *dpart = slash ? cbm_strndup(base, (size_t)(slash - base)) : strdup(""); - if (dpart && bn[0] != '_') { - snprintf(cands[ncand++], PKGMAP_PATH_BUF, "%s%s%s%s_%s.scss", dir, dir[0] ? "/" : "", - dpart[0] ? dpart : "", dpart[0] ? "/" : "", bn); - } - free(dpart); - } - /* 3. Meson subdir: dir//meson.build. */ - snprintf(cands[ncand++], PKGMAP_PATH_BUF, "%s%s%s/meson.build", dir, dir[0] ? "/" : "", base); - /* 4. Basename sibling: dir/. Covers include paths - * that carry a non-relative prefix (Hyprlang `source = ~/.config/.../x.conf`, - * absolute include paths) but reference a file sitting beside the importer. */ - { - const char *slash = strrchr(base, '/'); - if (slash && slash[1]) { - snprintf(cands[ncand++], PKGMAP_PATH_BUF, "%s%s%s", dir, dir[0] ? "/" : "", slash + 1); - } - } - - const cbm_gbuf_node_t *found = NULL; - for (int i = 0; i < ncand; i++) { - char *qn = cbm_pipeline_fqn_module(ctx->project_name, cands[i]); - if (!qn) { - continue; - } - const cbm_gbuf_node_t *n = cbm_gbuf_find_by_qn(ctx->gbuf, qn); - free(qn); - if (n && import_targetable_label(n->label) && - (!source_file_qn || !n->qualified_name || - strcmp(n->qualified_name, source_file_qn) != 0)) { - found = n; - break; - } - } - free(dir); - return found; -} - -/* ── PHP PSR-4 class imports (#1186) ──────────────────────────────── */ - -typedef enum { - PSR4_NOT_APPLICABLE = 0, /* no composer psr-4 prefix covers the import */ - PSR4_RESOLVED, /* the class file exists: *out is its File node */ - PSR4_UNRESOLVED, /* a prefix covers it but no class file exists */ -} psr4_outcome_t; - -static const char *path_leaf(const char *path) { - const char *leaf = path; - for (const char *p = path; p && *p; p++) { - if (*p == '/' || *p == '\\') { - leaf = p + 1; - } - } - return leaf; -} - -/* The File node at exactly `rel_path`, or NULL. Looked up through the name - * index (File nodes are named by basename) and matched on the full path, so - * a same-named file in another directory never qualifies. */ -static const cbm_gbuf_node_t *psr4_file_node(const cbm_gbuf_t *gbuf, const char *rel_path) { - const cbm_gbuf_node_t **hits = NULL; - int hit_count = 0; - if (cbm_gbuf_find_by_name(gbuf, path_leaf(rel_path), &hits, &hit_count) != 0 || !hits) { - return NULL; - } - for (int i = 0; i < hit_count; i++) { - const cbm_gbuf_node_t *n = hits[i]; - if (n && n->label && strcmp(n->label, "File") == 0 && n->file_path && - strcmp(n->file_path, rel_path) == 0) { - return n; - } - } - return NULL; -} - -/* Resolve a PHP class import `use A\B\C;` the way composer's PSR-4 autoloader - * does: for every autoload.psr-4 prefix covering the name, longest first, the - * class lives in exactly /.php with sub-namespaces as - * subdirectories; the first existing file wins. When a prefix covers the name - * but no such file exists the import is UNRESOLVED: it must never fall through - * to the namespace bucket, which binds it to whichever file of the namespace - * came first. `use function` / `use const` name namespace members, not class - * files, so they are not applicable. */ -static psr4_outcome_t resolve_php_psr4_class(const cbm_gbuf_t *gbuf, const char *source_file_qn, - const CBMImport *imp, const cbm_gbuf_node_t **out) { - *out = NULL; - CBMHashTable *pkgmap = cbm_pipeline_get_pkgmap(); - if (!pkgmap || imp->kind != CBM_IMPORT_KIND_DEFAULT) { - return PSR4_NOT_APPLICABLE; - } - const char *name = imp->module_path; - if (name[0] == '\\') { - name++; /* fully qualified `use \App\X;` */ - } - char key[PKGMAP_PATH_BUF]; - size_t name_len = strlen(name); - if (name_len == 0 || name_len >= sizeof(key) || !strchr(name, '\\')) { - return PSR4_NOT_APPLICABLE; - } - bool covered = false; - /* Right to left: each '\' ends a candidate prefix, longest first. */ - for (size_t cut = name_len - SKIP_ONE; cut > 0; cut--) { - if (name[cut - SKIP_ONE] != '\\') { - continue; - } - memcpy(key, name, cut); - key[cut] = '\0'; - const char *dir = (const char *)cbm_ht_get(pkgmap, key); - if (!dir) { - continue; - } - covered = true; - char rel[PKGMAP_PATH_BUF]; - int w = snprintf(rel, sizeof(rel), "%s%s%s.php", dir, dir[0] ? "/" : "", name + cut); - if (w <= 0 || (size_t)w >= sizeof(rel)) { - continue; - } - for (char *c = rel; *c; c++) { - if (*c == '\\') { - *c = '/'; - } - } - const cbm_gbuf_node_t *file = psr4_file_node(gbuf, rel); - if (file && (!source_file_qn || !file->qualified_name || - strcmp(file->qualified_name, source_file_qn) != 0)) { - *out = file; - return PSR4_RESOLVED; - } - } - return covered ? PSR4_UNRESOLVED : PSR4_NOT_APPLICABLE; -} - -static bool psr4_class_label(const char *label) { - return label && (strcmp(label, "Class") == 0 || strcmp(label, "Interface") == 0 || - strcmp(label, "Trait") == 0 || strcmp(label, "Enum") == 0); -} - -/* The QN of `member` on the class `cls` defined in `file_path`, or NULL when - * that file defines no such class or the class has no such member. */ -static const char *psr4_member_in_file(const cbm_gbuf_t *gbuf, const char *file_path, - const char *cls, const char *member) { - const cbm_gbuf_node_t **hits = NULL; - int hit_count = 0; - if (cbm_gbuf_find_by_name(gbuf, cls, &hits, &hit_count) != 0 || !hits) { - return NULL; - } - for (int i = 0; i < hit_count; i++) { - const cbm_gbuf_node_t *c = hits[i]; - if (!c || !psr4_class_label(c->label) || !c->qualified_name || !c->file_path || - strcmp(c->file_path, file_path) != 0) { - continue; - } - char qn[CBM_SZ_1K]; - int w = snprintf(qn, sizeof(qn), "%s.%s", c->qualified_name, member); - if (w <= 0 || (size_t)w >= sizeof(qn)) { - return NULL; - } - const cbm_gbuf_node_t *m = cbm_gbuf_find_by_qn(gbuf, qn); - return m ? m->qualified_name : NULL; - } - return NULL; -} - -cbm_psr4_member_t cbm_pipeline_psr4_member_qn(const cbm_gbuf_t *gbuf, const char *class_fqn, - const char *member, const char **out_qn) { - *out_qn = NULL; - if (!gbuf || !class_fqn || !member || !member[0]) { - return CBM_PSR4_MEMBER_NOT_COVERED; - } - const CBMImport imp = {.module_path = class_fqn, .kind = CBM_IMPORT_KIND_DEFAULT}; - const cbm_gbuf_node_t *file = NULL; - psr4_outcome_t placed = resolve_php_psr4_class(gbuf, NULL, &imp, &file); - if (placed == PSR4_NOT_APPLICABLE) { - return CBM_PSR4_MEMBER_NOT_COVERED; - } - /* Covered from here on: composer would load this file or none, so a - * missing file, class or member leaves the member unresolved. */ - if (placed != PSR4_RESOLVED || !file->file_path) { - return CBM_PSR4_MEMBER_UNRESOLVED; - } - const char *cls = strrchr(class_fqn, '\\'); - cls = cls ? cls + SKIP_ONE : class_fqn; - *out_qn = psr4_member_in_file(gbuf, file->file_path, cls, member); - return *out_qn ? CBM_PSR4_MEMBER_RESOLVED : CBM_PSR4_MEMBER_UNRESOLVED; -} - -const cbm_gbuf_node_t *cbm_pipeline_resolve_import_node(const cbm_pipeline_ctx_t *ctx, - const char *source_rel, - const char *source_file_qn, - const CBMImport *imp, - CBMHashTable *namespace_map) { - if (!ctx || !imp || !imp->module_path) { - return NULL; - } - - /* PHP class imports covered by a composer psr-4 prefix name exactly one - * file; when it is absent the import stays unresolved (#1186). */ - const cbm_gbuf_node_t *psr4_target = NULL; - switch (resolve_php_psr4_class(ctx->gbuf, source_file_qn, imp, &psr4_target)) { - case PSR4_RESOLVED: - return psr4_target; - case PSR4_UNRESOLVED: - return NULL; - case PSR4_NOT_APPLICABLE: - break; - } - - /* Strategy 1: module-path resolution → existing node (Python/TS/Go). - * No label filter here: directory-module languages (Go/Java packages) - * legitimately resolve straight to a Folder node -- that's the intended, - * correct import target, not a collision. The Folder-collision problem - * (#767) only shows up downstream, in Strategy 4's retry-with-truncated- - * path loop, which re-enters resolve_module with a DIFFERENT, shortened - * string that the original import never named. */ - char *target_qn = cbm_pipeline_resolve_module(ctx, source_rel, imp->module_path); - const cbm_gbuf_node_t *target = target_qn ? cbm_gbuf_find_by_qn(ctx->gbuf, target_qn) : NULL; - free(target_qn); - if (target) { - /* Python/TS from-import of a member: module_path is often - * "pkg.mod.symbol" while resolve_module lands on the Module node - * "pkg.mod". Prefer the member Function/Class when it exists — - * required for `from M import f as g` so CALLS can import_map to f - * (local_name g ≠ f). */ - if (target->label && - (strcmp(target->label, "Module") == 0 || strcmp(target->label, "File") == 0)) { - char symbuf[256]; - const char *sym = import_candidate_symbol(imp->module_path, symbuf, sizeof(symbuf)); - if (sym && sym[0] && strcmp(sym, "*") != 0) { - const char *mod_tail = import_last_segment(target->qualified_name - ? target->qualified_name - : ""); - if (!mod_tail || strcmp(mod_tail, sym) != 0) { - char member_qn[CBM_SZ_512]; - snprintf(member_qn, sizeof(member_qn), "%s.%s", target->qualified_name, sym); - const cbm_gbuf_node_t *member = cbm_gbuf_find_by_qn(ctx->gbuf, member_qn); - if (member && import_targetable_label(member->label) && - strcmp(member->label, "Module") != 0 && - strcmp(member->label, "File") != 0) { - return member; - } - } - } - } - return target; - } - - /* Name-guess fallbacks below (Strategy 1b sibling-file, Strategy 3 - * symbol-name) are gated per importing-file language — see - * cbm_import_symbol_fallback_allowed (#1934). */ - const char *src_base = source_rel ? source_rel : ""; - for (const char *pb = src_base; *pb; pb++) { - if (*pb == '/' || *pb == '\\') { - src_base = pb + SKIP_ONE; - } - } - const bool symbol_fallback_allowed = - cbm_import_symbol_fallback_allowed(cbm_language_for_filename(src_base)); - - /* Strategy 1b: sibling-file resolution for build/markup grammars whose - * import string is a sibling filename or directory (SCSS partials, Just/ - * BitBake/func includes, Meson subdir, Pony use). Its label filter admits - * symbols too, so for Go it re-creates the Strategy-3 bug one directory - * closer (`os/exec` → a same-package exec() method) — gated the same. */ - if (symbol_fallback_allowed) { - const cbm_gbuf_node_t *sib = - resolve_sibling_file(ctx, source_rel, source_file_qn, imp->module_path); - if (sib) { - return sib; - } - } - - /* Strategy 2: namespace map. `using App.Utils`, `import com.example.Foo`, - * `use App\Utils\Helper` name a NAMESPACE (or a member of it) that the - * path-based QN cannot express. Try the full module path and progressively - * shorter prefixes (dropping the trailing member segment) so both a bare - * namespace import and a member import resolve to the declaring file. */ - if (namespace_map) { - /* Normalize separators to '.' for namespace keys (PHP uses '\\'). - * Strip decorations first so `com.example.Util as U`, `crate::ops::*` - * and `App\Utils\{A, B}` reduce to a clean dotted path. */ - char norm[1024]; - snprintf(norm, sizeof(norm), "%s", imp->module_path); - char *brace = strchr(norm, '{'); - if (brace) { - *brace = '\0'; /* drop the group; the prefix is the namespace */ - } - char *as = strstr(norm, " as "); - if (as) { - *as = '\0'; - } - for (char *p = norm; *p; p++) { - if (*p == '\\' || *p == ':' || *p == '/') { - *p = '.'; - } - } - /* Strip trailing glob/separator noise. */ - size_t nl = strlen(norm); - while (nl > 0 && (norm[nl - 1] == '*' || norm[nl - 1] == '.' || norm[nl - 1] == ' ')) { - norm[--nl] = '\0'; - } - /* Collapse any "::"-induced empty segments ("a..b" → "a.b"). */ - for (;;) { - char *dd = strstr(norm, ".."); - if (!dd) { - break; - } - memmove(dd, dd + 1, strlen(dd + 1) + 1); - } - for (;;) { - /* The map value is a '\n'-delimited list of __file__ QNs declaring - * this namespace. Return the FIRST that is not the importing file, - * so a same-package wildcard import (`import com.example.*` from a - * file that is itself in com.example) resolves to a sibling — and - * deterministically, independent of file-iteration order across - * platforms (amd64 vs arm64). */ - const char *list = (const char *)cbm_ht_get(namespace_map, norm); - for (const char *seg = list; seg && *seg;) { - const char *eol = strchr(seg, '\n'); - size_t len = eol ? (size_t)(eol - seg) : strlen(seg); - char qbuf[1024]; - if (len > 0 && len < sizeof(qbuf)) { - memcpy(qbuf, seg, len); - qbuf[len] = '\0'; - const cbm_gbuf_node_t *n = cbm_gbuf_find_by_qn(ctx->gbuf, qbuf); - if (n && (!source_file_qn || strcmp(n->qualified_name, source_file_qn) != 0)) { - return n; - } - } - seg = eol ? eol + 1 : NULL; - } - char *dot = strrchr(norm, '.'); - if (!dot) { - break; - } - *dot = '\0'; - } - } - - /* Strategy 3: symbol-name fallback. Derive a representative imported - * symbol (handling alias / glob / grouped forms) and match it against an - * in-graph definition of the same simple name in another file - * (Rust `helper`, Java `Util`, Kotlin grouped, ...). - * Gated per importing-file language, like Strategy 1b above (#1934). */ - char symbuf[256]; - /* Prefer the clean candidate from the module path; the local_name may be an - * alias (Rust `as h`, Kotlin `as U`) that names no real symbol. */ - const char *seg = import_candidate_symbol(imp->module_path, symbuf, sizeof(symbuf)); - if (!seg && imp->local_name && imp->local_name[0] && strcmp(imp->local_name, "*") != 0) { - seg = imp->local_name; - } - /* Build the candidate symbol list: the derived symbol plus, when the import - * is a dotted member path like `com.example.Config.DEFAULT`, the enclosing - * type segments (`Config`). This resolves object/class-member imports to - * the declaring type when the leaf member isn't an importable node. */ - const char *cands[8]; - int ncands = 0; - if (seg && seg[0] && strcmp(seg, "*") != 0) { - cands[ncands++] = seg; - } - { - /* Strip decorations, normalize separators to '.', and collect the - * trailing path segments (last first) as fallback candidates. */ - char mp[1024]; - snprintf(mp, sizeof(mp), "%s", imp->module_path); - char *br = strchr(mp, '{'); - if (br) { - *br = '\0'; - } - char *as3 = strstr(mp, " as "); - if (as3) { - *as3 = '\0'; - } - for (char *p = mp; *p; p++) { - if (*p == '\\' || *p == ':' || *p == '/') { - *p = '.'; - } - } - /* Walk segments from the end. */ - char *end = mp + strlen(mp); - while (end > mp && ncands < 8) { - char *dot = NULL; - for (char *p = end - 1; p >= mp; p--) { - if (*p == '.') { - dot = p; - break; - } - } - const char *s = dot ? dot + 1 : mp; - if (s[0] && strcmp(s, "*") != 0) { - bool dup = false; - for (int k = 0; k < ncands; k++) { - if (strcmp(cands[k], s) == 0) { - dup = true; - break; - } - } - if (!dup) { - cands[ncands++] = s; - } - } - if (!dot) { - break; - } - *dot = '\0'; - end = dot; - } - for (int ci = 0; symbol_fallback_allowed && ci < ncands; ci++) { - const cbm_gbuf_node_t **hits = NULL; - int n = 0; - if (cbm_gbuf_find_by_name(ctx->gbuf, cands[ci], &hits, &n) == 0 && hits) { - /* Deterministic winner: hits[] is in node-registration order, - * which under parallel extraction varies run to run — taking - * the FIRST targetable hit made the same import resolve to - * different same-named nodes across runs (xfs: 360 flickering - * IMPORTS diff lines between two MT runs). Pick the candidate - * with the lexicographically smallest qualified name instead: - * stable, content-derived, identical for ST and MT. */ - const cbm_gbuf_node_t *best = NULL; - for (int i = 0; i < n; i++) { - const cbm_gbuf_node_t *cand = hits[i]; - if (!cand || !import_targetable_label(cand->label)) { - continue; - } - if (source_file_qn && cand->qualified_name && - strcmp(cand->qualified_name, source_file_qn) == 0) { - continue; /* self */ - } - if (!best || (cand->qualified_name && best->qualified_name && - strcmp(cand->qualified_name, best->qualified_name) < 0)) { - best = cand; - } - } - if (best) { - return best; - } - } - } - } - - /* Strategy 4: crate-relative module path → File/Module node. Rust glob - * `use crate::ops::*` names a module, not a symbol; strip the glob and the - * `crate::`/`self::`/`super::` prefix, convert `::`→`/`, then resolve the - * remaining path (and successive prefixes) to a Module/File node. */ - { - char mp[1024]; - snprintf(mp, sizeof(mp), "%s", imp->module_path); - char *brace = strchr(mp, '{'); - if (brace) { - *brace = '\0'; - } - char *as2 = strstr(mp, " as "); - if (as2) { - *as2 = '\0'; - } - /* Convert "::" → "/" (drop the doubled colon cleanly). */ - char clean[1024]; - size_t ci = 0; - for (const char *p = mp; *p && ci + 1 < sizeof(clean); p++) { - if (*p == ':') { - if (ci > 0 && clean[ci - 1] == '/') { - continue; /* collapse "::" → single "/" */ - } - clean[ci++] = '/'; - } else if (*p == '*' || *p == ' ') { - continue; - } else { - clean[ci++] = *p; - } - } - clean[ci] = '\0'; - /* Drop trailing slashes. */ - while (ci > 0 && clean[ci - 1] == '/') { - clean[--ci] = '\0'; - } - /* Strip a leading crate-relative prefix. */ - const char *body = clean; - static const char *prefixes[] = {"crate/", "self/", "super/", NULL}; - for (const char **pf = prefixes; *pf; pf++) { - size_t pl = strlen(*pf); - if (strncmp(body, *pf, pl) == 0) { - body += pl; - break; - } - } - if (body[0]) { - char work[1024]; - snprintf(work, sizeof(work), "%s", body); - for (;;) { - char *rqn = cbm_pipeline_resolve_module(ctx, source_rel, work); - const cbm_gbuf_node_t *n = rqn ? cbm_gbuf_find_by_qn(ctx->gbuf, rqn) : NULL; - free(rqn); - if (n && import_targetable_label(n->label) && - (!source_file_qn || !n->qualified_name || - strcmp(n->qualified_name, source_file_qn) != 0)) { - return n; - } - char *sl = strrchr(work, '/'); - if (!sl) { - break; - } - *sl = '\0'; - } - } - } - - return NULL; -} - -/* ── Namespace map ───────────────────────────────────────────────── */ - -CBMHashTable *cbm_pipeline_namespace_map_build(const char *project_name, - CBMFileResult **results, - cbm_result_spill_t *spill, - const char *const *rels, int count) { - CBMHashTable *map = NULL; - for (int i = 0; i < count; i++) { - if (!rels[i]) { - continue; - } - CBMFileResult *r = cbm_result_cache_acquire(results, spill, i); - char *key = r && r->namespace_name && r->namespace_name[0] ? strdup(r->namespace_name) - : NULL; - cbm_result_cache_release(results, i, r); - if (!key) { - continue; - } - if (!map) { - map = cbm_ht_create(CBM_SZ_64); - if (!map) { - free(key); - return NULL; - } - } - char *file_qn = cbm_pipeline_fqn_compute(project_name, rels[i], "__file__"); - if (!file_qn) { - free(key); - continue; - } - /* Normalize the namespace key to dot-separated form so it matches the - * dot-normalized lookups in cbm_pipeline_resolve_import_node (PHP uses - * '\\', some grammars '::' or '/'). */ - for (char *p = key; *p; p++) { - if (*p == '\\' || *p == ':' || *p == '/') { - *p = '.'; - } - } - /* Store ALL files declaring a namespace as a '\n'-delimited list so the - * resolver can pick a non-importer sibling (see resolve loop). The hash - * table does not copy keys, so the strdup'd key is owned by the map and - * freed in ns_map_free_entry. */ - if (!cbm_ht_has(map, key)) { - cbm_ht_set(map, key, file_qn); /* map owns key + file_qn */ - } else { - /* Append to the existing list. Re-key with the STORED key pointer - * (not our fresh strdup) so the map's key pointer never changes — - * otherwise Verstable would adopt the new key and our free(key) - * below would free the live key (use-after-free). */ - const char *stored_key = cbm_ht_get_key(map, key); - const char *cur = (const char *)cbm_ht_get(map, key); - char *combined = NULL; - if (stored_key && cur) { - size_t need = strlen(cur) + 1 + strlen(file_qn) + 1; - combined = malloc(need); - if (combined) { - snprintf(combined, need, "%s\n%s", cur, file_qn); - void *prev = cbm_ht_set(map, stored_key, combined); - free(prev); /* old value string */ - } - } - free(key); /* our fresh strdup — never stored */ - free(file_qn); /* content copied into combined */ - } - } - return map; -} - -static void ns_map_free_entry(const char *key, void *value, void *ud) { - (void)ud; - free((void *)key); /* strdup'd in cbm_pipeline_namespace_map_build */ - free(value); -} - -void cbm_pipeline_namespace_map_free(CBMHashTable *map) { - if (!map) { - return; - } - cbm_ht_foreach(map, ns_map_free_entry, NULL); - cbm_ht_free(map); -} diff --git a/src/pipeline/pass_pkgmap.zig b/src/pipeline/pass_pkgmap.zig new file mode 100644 index 00000000..8b722d90 --- /dev/null +++ b/src/pipeline/pass_pkgmap.zig @@ -0,0 +1,1966 @@ +//! pass_pkgmap.zig — Generic package/module manifest resolution. +//! +//! Scans discovered files for manifest files (package.json, go.mod, Cargo.toml, +//! pyproject.toml, composer.json, pubspec.yaml, pom.xml, build.gradle, mix.exs, +//! *.gemspec) and builds a hash table mapping bare package specifiers to resolved +//! module QNs. This enables IMPORTS edges for non-relative imports like +//! "@myorg/pkg", "github.com/foo/bar", "use my_crate::foo". +//! +//! Integration: called from parallel extract workers (per-worker local arrays) +//! and merged sequentially before registry build. + +const std = @import("std"); +const builtin = @import("builtin"); +const c = @import("c"); +const interop = @import("../foundation/interop.zig"); +const util = @import("util.zig"); + +const CStr = util.CStr; +const MutStr = util.MutStr; +const ix = util.ix; +const streq = util.streq; +const ptrDiff = util.ptrDiff; +const HashTable = c.CBMHashTable; +const PkgEntries = c.cbm_pkg_entries_t; +const PkgEntry = c.cbm_pkg_entry_t; +const Node = c.cbm_gbuf_node_t; +const Gbuf = c.cbm_gbuf_t; +const Ctx = c.cbm_pipeline_ctx_t; +const JsonVal = ?*c.yyjson_val; + +/// Read an entire file into a malloc'd buffer. Returns NULL on failure. +fn pkgmapReadFile(path: CStr, out_len: *c_int) MutStr { + const f = c.cbm_fopen(path, "rb") orelse return null; + _ = c.fseek(f, 0, c.SEEK_END); + const size = c.ftell(f); + _ = c.fseek(f, 0, c.SEEK_SET); + if (size <= 0 or size > c.CBM_SZ_1K * c.CBM_SZ_1K) { // 1MB cap for manifests + _ = c.fclose(f); + return null; + } + const buf: MutStr = @ptrCast(c.malloc(@as(usize, @intCast(size)) + 1)); + if (buf == null) { + _ = c.fclose(f); + return null; + } + const nread = c.fread(buf, 1, @intCast(size), f); + _ = c.fclose(f); + buf[nread] = 0; + out_len.* = @intCast(nread); + return buf; +} + +// ── Constants ─────────────────────────────────────────────────────── + +const PKGMAP_INIT_CAP = 16; +const PKGMAP_PATH_BUF = 1024; +const PKGMAP_LINE_BUF = 512; +const PKGMAP_HT_INIT = 64; +/// Hard ceiling on recursive directory descent. This is the +/// non-negotiable termination guarantee for the manifest walk: even +/// if the filesystem contains directory junctions / symlink cycles +/// (the documented reason the walk was once disabled on Windows), +/// descent stops at this depth so the walk can never hang. 64 is far +/// deeper than any real source tree. +const PKGMAP_WALK_MAX_DEPTH = 64; + +/// Check if the text at p (bounded by end) starts with prefix. +fn atPrefix(p: [*]const u8, end: [*]const u8, comptime prefix: []const u8) bool { + return @intFromPtr(p) + prefix.len <= @intFromPtr(end) and + std.mem.eql(u8, p[0..prefix.len], prefix); +} + +/// p < end, for the pointer scans below. +inline fn before(p: [*]const u8, end: [*]const u8) bool { + return @intFromPtr(p) < @intFromPtr(end); +} + +fn isBlank(ch: u8) bool { + return ch == ' ' or ch == '\t'; +} + +// ── Per-worker collection ─────────────────────────────────────────── + +pub export fn cbm_pkg_entries_init(e: *PkgEntries) void { + e.items = null; + e.count = 0; + e.cap = 0; +} + +fn pkgEntriesPush(e: *PkgEntries, pkg_name: MutStr, entry_rel: MutStr) void { + if (e.count >= e.cap) { + const new_cap: c_int = if (e.cap == 0) PKGMAP_INIT_CAP else e.cap * 2; + const tmp = interop.reallocN(PkgEntry, e.items, ix(new_cap)); + if (tmp == null) { + c.free(pkg_name); + c.free(entry_rel); + return; + } + e.items = tmp; + e.cap = new_cap; + } + e.items[ix(e.count)].pkg_name = pkg_name; + e.items[ix(e.count)].entry_rel = entry_rel; + e.count += 1; +} + +pub export fn cbm_pkg_entries_free(e: *PkgEntries) void { + for (0..ix(e.count)) |i| { + c.free(e.items[i].pkg_name); + c.free(e.items[i].entry_rel); + } + c.free(e.items); + e.items = null; + e.count = 0; + e.cap = 0; +} + +// ── Helpers ───────────────────────────────────────────────────────── + +/// Get the directory part of a relative path (without trailing slash). +/// Returns heap-allocated string. For "foo.json" returns "". +fn pathDirname(rel_path: CStr) MutStr { + const last: CStr = c.strrchr(rel_path, '/'); + if (last == null) return c.strdup(""); + return c.cbm_strndup(rel_path, ptrDiff(last, rel_path)); +} + +/// Strip file extension from a path. Returns heap-allocated string. +/// "src/index.ts" → "src/index", "lib/main" → "lib/main" +fn stripExtension(path: CStr) MutStr { + var i = c.strlen(path); + while (i > 0) : (i -= 1) { + if (path[i - 1] == '.') return c.cbm_strndup(path, i - 1); + if (path[i - 1] == '/') break; + } + return c.strdup(path); +} + +/// Join directory + relative entry path, normalize. +/// "packages/foo" + "src/index.ts" → "packages/foo/src/index" (stripped ext) +fn joinAndStrip(dir: CStr, entry_in: CStr) MutStr { + var entry = entry_in; + if (entry == null or entry[0] == 0) return null; + // Skip leading ./ from entry + if (entry[0] == '.' and entry[1] == '/') entry += 2; + var buf: [PKGMAP_PATH_BUF]u8 = undefined; + if (dir[0] == 0) { + _ = c.snprintf(&buf, buf.len, "%s", entry); + } else { + _ = c.snprintf(&buf, buf.len, "%s/%s", dir, entry); + } + return stripExtension(&buf); +} + +/// Find a line starting with a prefix in source. Returns pointer to first char +/// after prefix, or NULL. Handles leading whitespace. +fn findLineValue(src: [*]const u8, src_len: c_int, prefix: []const u8) ?[*]const u8 { + var p = src; + const end = src + ix(src_len); + while (before(p, end)) { + // Skip leading whitespace + while (before(p, end) and isBlank(p[0])) p += 1; + if (@intFromPtr(p) + prefix.len <= @intFromPtr(end) and std.mem.eql(u8, p[0..prefix.len], prefix)) { + return p + prefix.len; + } + // Skip to next line + while (before(p, end) and p[0] != '\n') p += 1; + if (before(p, end)) p += 1; // skip \n + } + return null; +} + +/// Extract a quoted string value from position. Handles both "..." and '...' +/// Returns heap-allocated string, or NULL. +fn extractQuoted(p_in: [*]const u8, end: [*]const u8) MutStr { + var p = p_in; + // Skip whitespace and = sign + while (before(p, end) and (isBlank(p[0]) or p[0] == '=')) p += 1; + if (!before(p, end)) return null; + const quote = p[0]; + if (quote != '"' and quote != '\'') return null; + p += 1; + const start = p; + while (before(p, end) and p[0] != quote and p[0] != '\n') p += 1; + if (!before(p, end) or p[0] != quote) return null; + return c.cbm_strndup(start, ptrDiff(p, start)); +} + +// ── Language-specific manifest parsers ────────────────────────────── + +/// Resolve JS/TS entry point from exports["."] object. +fn resolveExportsDot(dot: JsonVal) CStr { + @setEvalBranchQuota(10000); + if (c.yyjson_is_str(dot)) return c.yyjson_get_str(dot); + if (!c.yyjson_is_obj(dot)) return null; + for ([_][*:0]const u8{ "import", "default", "require" }) |key| { + const v = c.yyjson_obj_get(dot, key); + if (c.yyjson_is_str(v)) return c.yyjson_get_str(v); + } + return null; +} + +/// Resolve JS/TS entry point from package.json root. +fn resolvePkgEntry(root: JsonVal) CStr { + @setEvalBranchQuota(10000); + const exports = c.yyjson_obj_get(root, "exports"); + if (c.yyjson_is_obj(exports)) { + const e = resolveExportsDot(c.yyjson_obj_get(exports, ".")); + if (e != null) return e; + } + for ([_][*:0]const u8{ "main", "module" }) |key| { + const v = c.yyjson_obj_get(root, key); + if (c.yyjson_is_str(v)) return c.yyjson_get_str(v); + } + return "src/index.ts"; // last resort default +} + +/// JS/TS: package.json — name + entry point resolution +fn parsePackageJson(source: [*]const u8, source_len: c_int, rel_path: CStr, entries: *PkgEntries) void { + @setEvalBranchQuota(10000); + const doc = c.yyjson_read(source, ix(source_len), 0); + if (doc == null) return; + defer c.yyjson_doc_free(doc); + const root = c.yyjson_doc_get_root(doc); + if (!c.yyjson_is_obj(root)) return; + + const name_val = c.yyjson_obj_get(root, "name"); + if (!c.yyjson_is_str(name_val)) return; + const name: CStr = c.yyjson_get_str(name_val); + if (name == null or name[0] == 0) return; + + const entry = resolvePkgEntry(root); + if (entry != null) { + const dir = pathDirname(rel_path); + const resolved = joinAndStrip(dir, entry); + if (resolved != null) { + pkgEntriesPush(entries, c.strdup(name), resolved); + } + c.free(dir); + } +} + +/// Go: go.mod — module directive +fn parseGoMod(source: [*]const u8, source_len: c_int, rel_path: CStr, entries: *PkgEntries) void { + const end = source + ix(source_len); + var val = findLineValue(source, source_len, "module ") orelse return; + // Extract module path (rest of line, trimmed) + while (before(val, end) and isBlank(val[0])) val += 1; + const start = val; + while (before(val, end) and val[0] != '\n' and val[0] != '\r' and val[0] != ' ') val += 1; + if (@intFromPtr(val) <= @intFromPtr(start)) return; + const module_path = c.cbm_strndup(start, ptrDiff(val, start)); + const dir = pathDirname(rel_path); + + // The module path maps to the directory containing go.mod. + // For "." dir, use empty string. + pkgEntriesPush(entries, module_path, c.strdup(dir)); + c.free(dir); +} + +/// Extract "name" value from a TOML section. Scans from section_start until +/// the next [...] header or EOF. Returns heap string or NULL. +fn tomlExtractName(section_start: [*]const u8, end: [*]const u8) MutStr { + var p = section_start; + while (before(p, end)) { + while (before(p, end) and isBlank(p[0])) p += 1; + if (before(p, end) and p[0] == '[') break; + if (atPrefix(p, end, "name ")) return extractQuoted(p + "name ".len, end); + if (atPrefix(p, end, "name=")) return extractQuoted(p + "name=".len, end); + while (before(p, end) and p[0] != '\n') p += 1; + if (before(p, end)) p += 1; + } + return null; +} + +/// "/" for a non-empty dir, "" otherwise: the two snprintf arguments the +/// C code wrote as `dir[0] ? dir : "", dir[0] ? "/" : ""`. +const DirPrefix = struct { + dir: CStr, + sep: CStr, + + fn of(dir: CStr) DirPrefix { + return if (dir[0] != 0) .{ .dir = dir, .sep = "/" } else .{ .dir = "", .sep = "" }; + } +}; + +/// Build entry path: dir/suffix or just suffix if dir is empty. +fn buildEntryPath(rel_path: CStr, suffix: CStr) MutStr { + const dir = pathDirname(rel_path); + const prefix = DirPrefix.of(dir); + var buf: [PKGMAP_PATH_BUF]u8 = undefined; + _ = c.snprintf(&buf, buf.len, "%s%s%s", prefix.dir, prefix.sep, suffix); + c.free(dir); + return c.strdup(&buf); +} + +/// Rust: Cargo.toml — [package] name +fn parseCargoToml(source: [*]const u8, source_len: c_int, rel_path: CStr, entries: *PkgEntries) void { + const section = findLineValue(source, source_len, "[package]") orelse return; + const name = tomlExtractName(section, source + ix(source_len)); + if (name == null) return; + const entry = buildEntryPath(rel_path, "src/lib"); + pkgEntriesPush(entries, name, entry); +} + +/// Normalize Python package name: hyphens → underscores (PEP 503). +fn pyNormalizeName(name: [*:0]u8) void { + var p = name; + while (p[0] != 0) : (p += 1) { + if (p[0] == '-') p[0] = '_'; + } +} + +/// Python: pyproject.toml — [project] name +fn parsePyprojectToml(source: [*]const u8, source_len: c_int, rel_path: CStr, entries: *PkgEntries) void { + const section = findLineValue(source, source_len, "[project]") orelse return; + const name = tomlExtractName(section, source + ix(source_len)); + if (name == null) return; + pyNormalizeName(name); + + // Register src//__init__ as primary entry + var suffix: [PKGMAP_PATH_BUF]u8 = undefined; + _ = c.snprintf(&suffix, suffix.len, "src/%s/__init__", name); + const entry = buildEntryPath(rel_path, &suffix); + const name_copy: MutStr = c.strdup(name); + pkgEntriesPush(entries, name, entry); + + // Also register /__init__ as alternative (no src/ prefix) + _ = c.snprintf(&suffix, suffix.len, "%s/__init__", name_copy); + const alt_entry = buildEntryPath(rel_path, &suffix); + if (name_copy != null and alt_entry != null) { + pkgEntriesPush(entries, name_copy, alt_entry); + } else { + c.free(name_copy); + c.free(alt_entry); + } +} + +/// Extract PSR-4 entries from one composer.json autoload section ("autoload" +/// or "autoload-dev": composer registers both, so test namespaces such as +/// Tests\ map to their directory as well). +fn extractPsr4(root: JsonVal, section: [*:0]const u8, dir: CStr, entries: *PkgEntries) void { + @setEvalBranchQuota(10000); + const autoload = c.yyjson_obj_get(root, section); + if (!c.yyjson_is_obj(autoload)) return; + const psr4 = c.yyjson_obj_get(autoload, "psr-4"); + if (!c.yyjson_is_obj(psr4)) return; + var iter = c.yyjson_obj_iter_with(psr4); + while (true) { + const key = c.yyjson_obj_iter_next(&iter); + if (key == null) break; + const val = c.yyjson_obj_iter_get_val(key); + if (!c.yyjson_is_str(key) or !c.yyjson_is_str(val)) continue; + const ns_prefix: CStr = c.yyjson_get_str(key); + var ns_dir: CStr = c.yyjson_get_str(val); + while (ns_dir[0] == '.' and ns_dir[1] == '/') { + ns_dir += 2; // "./src/" names src/ + } + if (streq(ns_dir, ".")) { + ns_dir = ""; // "." names the manifest's own directory + } + var ns_entry: [PKGMAP_PATH_BUF]u8 = undefined; + if (dir[0] != 0) { + _ = c.snprintf(&ns_entry, ns_entry.len, "%s/%s", dir, ns_dir); + } else { + _ = c.snprintf(&ns_entry, ns_entry.len, "%s", ns_dir); + } + const nelen = c.strlen(&ns_entry); + if (nelen > 0 and ns_entry[nelen - 1] == '/') ns_entry[nelen - 1] = 0; + pkgEntriesPush(entries, c.strdup(ns_prefix), c.strdup(&ns_entry)); + } +} + +/// PHP: composer.json — name + PSR-4 autoload +fn parseComposerJson(source: [*]const u8, source_len: c_int, rel_path: CStr, entries: *PkgEntries) void { + @setEvalBranchQuota(10000); + const doc = c.yyjson_read(source, ix(source_len), 0); + if (doc == null) return; + defer c.yyjson_doc_free(doc); + const root = c.yyjson_doc_get_root(doc); + if (!c.yyjson_is_obj(root)) return; + + const dir = pathDirname(rel_path); + + // Register package name → directory + const name_val = c.yyjson_obj_get(root, "name"); + if (c.yyjson_is_str(name_val)) { + const name: CStr = c.yyjson_get_str(name_val); + if (name != null and name[0] != 0) { + pkgEntriesPush(entries, c.strdup(name), c.strdup(dir)); + } + } + + extractPsr4(root, "autoload", dir, entries); + extractPsr4(root, "autoload-dev", dir, entries); + + c.free(dir); +} + +/// Dart: pubspec.yaml — name +fn parsePubspecYaml(source: [*]const u8, source_len: c_int, rel_path: CStr, entries: *PkgEntries) void { + const root = c.cbm_yaml_parse(source, source_len); + if (root == null) return; + const name: CStr = c.cbm_yaml_get_str(root, "name"); + if (name != null and name[0] != 0) { + const dir = pathDirname(rel_path); + const prefix = DirPrefix.of(dir); + var entry: [PKGMAP_PATH_BUF]u8 = undefined; + _ = c.snprintf(&entry, entry.len, "%s%slib", prefix.dir, prefix.sep); + pkgEntriesPush(entries, c.strdup(name), c.strdup(&entry)); + c.free(dir); + } + c.cbm_yaml_free(root); +} + +/// Extract text content of an XML tag at position p. Returns heap string or NULL. +/// p must point to the char after the opening tag's '>'. +fn xmlTagContent(p_in: [*]const u8, end: [*]const u8) MutStr { + var p = p_in; + const s = p; + while (before(p, end) and p[0] != '<') p += 1; + if (@intFromPtr(p) <= @intFromPtr(s)) return null; + return c.cbm_strndup(s, ptrDiff(p, s)); +} + +/// Scan pom.xml for a top-level XML tag (outside ). Returns heap string or NULL. +fn pomFindTag(source: [*]const u8, end: [*]const u8, comptime tag: []const u8) MutStr { + var p = source; + var in_parent = false; + while (before(p, end)) : (p += 1) { + if (atPrefix(p, end, "")) in_parent = true; + if (atPrefix(p, end, "")) in_parent = false; + if (!in_parent and atPrefix(p, end, tag)) { + return xmlTagContent(p + tag.len, end); + } + } + return null; +} + +/// Java: pom.xml — + +fn parsePomXml(source: [*]const u8, source_len: c_int, rel_path: CStr, entries: *PkgEntries) void { + const end = source + ix(source_len); + const group_id = pomFindTag(source, end, ""); + const artifact_id = pomFindTag(source, end, ""); + + if (group_id != null and artifact_id != null) { + // Map: "com.myorg.myapp" → src/main/java directory + var pkg_name: [PKGMAP_PATH_BUF]u8 = undefined; + _ = c.snprintf(&pkg_name, pkg_name.len, "%s.%s", group_id, artifact_id); + const dir = pathDirname(rel_path); + const prefix = DirPrefix.of(dir); + var entry: [PKGMAP_PATH_BUF]u8 = undefined; + _ = c.snprintf(&entry, entry.len, "%s%ssrc/main/java", prefix.dir, prefix.sep); + pkgEntriesPush(entries, c.strdup(&pkg_name), c.strdup(&entry)); + + // Also register just the groupId for package-level imports + var grp_entry: [PKGMAP_PATH_BUF]u8 = undefined; + _ = c.snprintf(&grp_entry, grp_entry.len, "%s%ssrc/main/java", prefix.dir, prefix.sep); + pkgEntriesPush(entries, c.strdup(group_id), c.strdup(&grp_entry)); + c.free(dir); + } + + c.free(group_id); + c.free(artifact_id); +} + +/// Gradle: build.gradle / build.gradle.kts — group = '...' +fn parseBuildGradle(source: [*]const u8, source_len: c_int, rel_path: CStr, entries: *PkgEntries) void { + const end = source + ix(source_len); + // Look for group = '...' or group '...' or group = "..." + const val = findLineValue(source, source_len, "group") orelse return; + const group = extractQuoted(val, end); + if (group == null) return; + const dir = pathDirname(rel_path); + const prefix = DirPrefix.of(dir); + // Check for src/main/java or src/main/kotlin + var entry: [PKGMAP_PATH_BUF]u8 = undefined; + _ = c.snprintf(&entry, entry.len, "%s%ssrc/main/java", prefix.dir, prefix.sep); + pkgEntriesPush(entries, group, c.strdup(&entry)); + c.free(dir); +} + +/// Elixir: mix.exs — app: :name +fn parseMixExs(source: [*]const u8, source_len: c_int, rel_path: CStr, entries: *PkgEntries) void { + const end = source + ix(source_len); + // Look for app: :app_name + var val = findLineValue(source, source_len, "app:") orelse return; + while (before(val, end) and isBlank(val[0])) val += 1; + if (!before(val, end) or val[0] != ':') return; + val += 1; // skip : + const start = val; + while (before(val, end) and val[0] != ',' and val[0] != '\n' and val[0] != ' ' and val[0] != ')') { + val += 1; + } + if (@intFromPtr(val) <= @intFromPtr(start)) return; + const app_name = c.cbm_strndup(start, ptrDiff(val, start)); + const dir = pathDirname(rel_path); + const prefix = DirPrefix.of(dir); + var entry: [PKGMAP_PATH_BUF]u8 = undefined; + _ = c.snprintf(&entry, entry.len, "%s%slib/%s", prefix.dir, prefix.sep, app_name); + // Register with colon prefix as Elixir uses :atom syntax + var atom_name: [PKGMAP_PATH_BUF]u8 = undefined; + _ = c.snprintf(&atom_name, atom_name.len, "%s", app_name); + pkgEntriesPush(entries, c.strdup(&atom_name), c.strdup(&entry)); + c.free(app_name); + c.free(dir); +} + +/// Ruby: *.gemspec — spec.name = '...' +fn parseGemspec(source: [*]const u8, source_len: c_int, rel_path: CStr, entries: *PkgEntries) void { + const end = source + ix(source_len); + // Try spec.name, s.name, gem.name patterns + const pattern = ".name"; + var p = source; + while (before(p, end)) { + const found: CStr = c.strstr(p, pattern); + if (found == null or !before(found, end)) break; + const name = extractQuoted(found + pattern.len, end); + if (name != null) { + const dir = pathDirname(rel_path); + const prefix = DirPrefix.of(dir); + var entry: [PKGMAP_PATH_BUF]u8 = undefined; + _ = c.snprintf(&entry, entry.len, "%s%slib/%s", prefix.dir, prefix.sep, name); + pkgEntriesPush(entries, name, c.strdup(&entry)); + c.free(dir); + return; + } + p = found + 1; + } +} + +// Zig: build.zig — named modules for `@import("name")`. A module is declared +// with `b.addModule("name", .{ .root_source_file = b.path("src/root.zig") })`, +// or created into a variable (`const m = b.createModule(.{ .root_source_file = +// b.path(...) })`) that is registered under a name by `x.addImport("name", m)` +// or `.imports = &.{ .{ .name = "name", .module = m } }`. Keys carry a "zig:" +// prefix so a Zig module never collides with another ecosystem's package. +// +// A module of a dependency (`b.dependency("pkg", .{}).module("x")`) resolves +// when build.zig.zon declares the dependency by an in-repo `.path`: the +// dependency's build.zig `addModule("x", ...)` names its root. A dependency +// fetched by URL has no source in the repo and stays unmapped. Entries for +// this: "zigdep:|" -> the dependency's directory (build.zig.zon in +// ), "zigmod:|" -> module x of the build.zig in , and +// "zig:" -> "@||", resolved by cbm_pkgmap_build. + +const Text = ?[*]const u8; + +fn bzFind(p_in: Text, end: [*]const u8, needle: []const u8) Text { + var p = p_in orelse return null; + while (@intFromPtr(p) + needle.len <= @intFromPtr(end)) : (p += 1) { + if (std.mem.eql(u8, p[0..needle.len], needle)) return p; + } + return null; +} + +fn bzSkipWs(p_in: [*]const u8, end: [*]const u8) [*]const u8 { + var p = p_in; + while (before(p, end) and (p[0] == ' ' or p[0] == '\t' or p[0] == '\n' or p[0] == '\r')) p += 1; + return p; +} + +fn bzIdentLen(p: [*]const u8, end: [*]const u8) usize { + var n: usize = 0; + while (before(p + n, end) and (p[n] == '_' or (p[n] >= 'a' and p[n] <= 'z') or + (p[n] >= 'A' and p[n] <= 'Z') or (n > 0 and p[n] >= '0' and p[n] <= '9'))) + { + n += 1; + } + return n; +} + +/// memchr over [p, end). +fn bzMemchr(p: [*]const u8, ch: u8, end: [*]const u8) Text { + return @ptrCast(c.memchr(p, ch, ptrDiff(end, p))); +} + +/// A "..." string at p (after whitespace): heap copy, after.* past the quote. +fn bzQuoted(p_in: [*]const u8, end: [*]const u8, after: *[*]const u8) MutStr { + const p = bzSkipWs(p_in, end); + if (!before(p, end) or p[0] != '"') return null; + const close = bzMemchr(p + 1, '"', end) orelse return null; + after.* = close + 1; + return c.cbm_strndup(p + 1, ptrDiff(close, p) - 1); +} + +/// The `.root_source_file = x.path("...zig")` of the statement starting at p. +fn bzRootSource(p: [*]const u8, end: [*]const u8) MutStr { + const stop = bzMemchr(p, ';', end) orelse end; + const r = bzFind(p, stop, "root_source_file"); + const call = if (r != null) bzFind(r, stop, ".path(") else null; + var after: [*]const u8 = undefined; + var path: MutStr = if (call) |found| bzQuoted(found + ".path(".len, stop, &after) else null; + if (path != null and (!c.cbm_str_ends_with(path, ".zig") or c.strstr(path, "..") != null)) { + c.free(path); + path = null; + } + return path; +} + +/// The text after `const ident =` for the declaration of `ident`, or null. +fn bzConstValue(src: [*]const u8, end: [*]const u8, ident: []const u8) Text { + var p = bzFind(src, end, "const "); + while (p) |decl| : (p = bzFind(decl + 1, end, "const ")) { + var q = bzSkipWs(decl + "const ".len, end); + if (bzIdentLen(q, end) != ident.len or !std.mem.eql(u8, q[0..ident.len], ident)) continue; + q = bzSkipWs(q + ident.len, end); + if (before(q, end) and q[0] == '=') return q; + } + return null; +} + +/// Root source of module variable `ident` (from its `const ident = ...;`). +fn bzVarSource(src: [*]const u8, end: [*]const u8, ident: []const u8) MutStr { + const eq = bzConstValue(src, end, ident) orelse return null; + return bzRootSource(eq, end); +} + +/// The module variable at p (`m` followed by `)`/`,`/`}`): its root source. +fn bzModuleArgSource(src: [*]const u8, p_in: [*]const u8, end: [*]const u8) MutStr { + const p = bzSkipWs(p_in, end); + const len = bzIdentLen(p, end); + const q = bzSkipWs(p + len, end); + if (len == 0 or !before(q, end) or (q[0] != ')' and q[0] != ',' and q[0] != '}')) return null; + return bzVarSource(src, end, p[0..len]); +} + +fn bzPush(entries: *PkgEntries, rel_path: CStr, name: CStr, path: CStr, exported: bool) void { + var key: [PKGMAP_PATH_BUF]u8 = undefined; + _ = c.snprintf(&key, key.len, "zig:%s", name); + const dir = pathDirname(rel_path); + var mod_key: [PKGMAP_PATH_BUF]u8 = undefined; + _ = c.snprintf(&mod_key, mod_key.len, "zigmod:%s|%s", dir, name); + const mod_entry: MutStr = if (exported) joinAndStrip(dir, path) else null; + if (mod_entry != null) { + pkgEntriesPush(entries, c.strdup(&mod_key), mod_entry); + } + const entry = joinAndStrip(dir, path); + c.free(dir); + if (entry != null) { + pkgEntriesPush(entries, c.strdup(&key), entry); + } +} + +/// End of the expression at p: the `,` `)` `}` or `;` that closes it. +fn bzExprEnd(p_in: [*]const u8, end: [*]const u8) [*]const u8 { + var nest: c_int = 0; + var p = p_in; + while (before(p, end)) : (p += 1) { + if (p[0] == '"') { + p = bzMemchr(p + 1, '"', end) orelse end; + } else if (p[0] == '(' or p[0] == '{' or p[0] == '[') { + nest += 1; + } else if ((p[0] == ')' or p[0] == '}' or p[0] == ']') and nest > 0) { + nest -= 1; + } else if (nest == 0 and (p[0] == ',' or p[0] == ')' or p[0] == '}' or p[0] == ';')) { + return p; + } + } + return end; +} + +/// The expression [b, e), or a bare identifier's `const` value. +fn bzDeref(src: [*]const u8, end: [*]const u8, b: *[*]const u8, e: *[*]const u8) void { + const len = bzIdentLen(b.*, e.*); + if (len > 0 and bzSkipWs(b.* + len, e.*) == e.*) { + if (bzConstValue(src, end, b.*[0..len])) |eq| { + const vb = bzSkipWs(eq + 1, end); + b.* = vb; + e.* = bzExprEnd(vb, end); + } + } +} + +/// A module argument `dep.module("x")` / `b.dependency("pkg", ..).module("x")` +/// (or a `const` holding one): push "zig:" -> "@||". +fn bzPushDepModule(entries: *PkgEntries, rel_path: CStr, name: CStr, src: [*]const u8, p: [*]const u8, end: [*]const u8) void { + var b = bzSkipWs(p, end); + var e = bzExprEnd(b, end); + bzDeref(src, end, &b, &e); + var m: Text = null; + var q = bzFind(b, e, ".module("); + while (q) |found| : (q = bzFind(found + 1, e, ".module(")) { + m = found; + } + var after: [*]const u8 = undefined; + const mod: MutStr = if (m) |found| bzQuoted(found + ".module(".len, e, &after) else null; + var db = b; + var de: Text = m; + if (mod != null) { + var de_end = m.?; + bzDeref(src, end, &db, &de_end); + de = de_end; + } + const d: Text = if (mod != null) bzFind(db, de.?, ".dependency(") else null; + const pkg: MutStr = if (d) |found| bzQuoted(found + ".dependency(".len, de.?, &after) else null; + if (pkg != null) { + var key: [PKGMAP_PATH_BUF]u8 = undefined; + var val: [PKGMAP_PATH_BUF]u8 = undefined; + const dir = pathDirname(rel_path); + _ = c.snprintf(&key, key.len, "zig:%s", name); + _ = c.snprintf(&val, val.len, "@%s|%s|%s", dir, pkg, mod); + c.free(dir); + pkgEntriesPush(entries, c.strdup(&key), c.strdup(&val)); + } + c.free(mod); + c.free(pkg); +} + +/// build.zig.zon: `.dependencies = .{ .pkg = .{ .path = "libs/pkg" }, ... }`. +fn parseBuildZigZon(source: [*]const u8, source_len: c_int, rel_path: CStr, entries: *PkgEntries) void { + // Blank `//` comments (outside strings), so a commented-out dependency + // is not one. + const len = ix(source_len); + const text: MutStr = c.cbm_strndup(source, len); + if (text == null) return; + var in_str = false; + var i: usize = 0; + while (i < len) : (i += 1) { + if (in_str) { + in_str = !(text[i] == '"' and text[i - 1] != '\\'); + } else if (text[i] == '"') { + in_str = true; + } else if (text[i] == '/' and i + 1 < len and text[i + 1] == '/') { + while (i < len and text[i] != '\n') : (i += 1) text[i] = ' '; + } + } + const end: [*]const u8 = @as([*]const u8, text) + len; + const deps = bzFind(text, end, ".dependencies"); + const open = if (deps != null) bzFind(deps, end, ".{") else null; + const dir = pathDirname(rel_path); + if (open) |brace| { + const close = bzExprEnd(brace + 2, end); + var p: [*]const u8 = brace + 2; + while (before(p, close)) { + p = bzSkipWs(p, close); + if (!before(p, close) or p[0] != '.') break; + p += 1; + var after = p; + var name: MutStr = null; + if (p[0] == '@') { + name = bzQuoted(p + 1, close, &after); + } else { + const n = bzIdentLen(p, close); + name = if (n > 0) c.cbm_strndup(p, n) else null; + after = p + n; + } + const eq = bzSkipWs(after, close); + const body = if (before(eq, close) and eq[0] == '=') bzSkipWs(eq + 1, close) else close; + const body_end = bzExprEnd(body, close); + const path: Text = if (name != null) bzFind(body, body_end, ".path") else null; + const peq: Text = if (path) |found| bzSkipWs(found + ".path".len, body_end) else null; + var rel: MutStr = null; + if (peq) |assign| { + if (before(assign, body_end) and assign[0] == '=') { + rel = bzQuoted(assign + 1, body_end, &after); + } + } + if (rel != null) { + var rl = c.strlen(rel); + while (rl > 0 and rel[rl - 1] == '/') { + rl -= 1; + rel[rl] = 0; + } + const r: CStr = if (rel[0] == '.' and rel[1] == '/') rel + 2 else rel; + var key: [PKGMAP_PATH_BUF]u8 = undefined; + var val: [PKGMAP_PATH_BUF]u8 = undefined; + _ = c.snprintf(&key, key.len, "zigdep:%s|%s", dir, name); + if (dir[0] == 0) { + _ = c.snprintf(&val, val.len, "%s", r); + } else { + _ = c.snprintf(&val, val.len, "%s/%s", dir, r); + } + pkgEntriesPush(entries, c.strdup(&key), c.strdup(&val)); + } + c.free(name); + c.free(rel); + p = if (before(body_end, close) and body_end[0] == ',') body_end + 1 else close; + } + } + c.free(dir); + c.free(text); +} + +fn parseBuildZig(source: [*]const u8, source_len: c_int, rel_path: CStr, entries: *PkgEntries) void { + const end = source + ix(source_len); + var after: [*]const u8 = undefined; + var p = bzFind(source, end, "addModule("); + while (p) |found| : (p = bzFind(found + 1, end, "addModule(")) { + const name = bzQuoted(found + "addModule(".len, end, &after); + const path: MutStr = if (name != null) bzRootSource(after, end) else null; + if (path != null) bzPush(entries, rel_path, name, path, true); + c.free(name); + c.free(path); + } + p = bzFind(source, end, "addImport("); + while (p) |found| : (p = bzFind(found + 1, end, "addImport(")) { + const name = bzQuoted(found + "addImport(".len, end, &after); + const comma: Text = if (name != null) bzSkipWs(after, end) else null; + const has_arg = if (comma) |sep| before(sep, end) and sep[0] == ',' else false; + const path: MutStr = if (has_arg) bzModuleArgSource(source, comma.? + 1, end) else null; + if (path != null) { + bzPush(entries, rel_path, name, path, false); + } else if (has_arg) { + bzPushDepModule(entries, rel_path, name, source, comma.? + 1, end); + } + c.free(name); + c.free(path); + } + p = bzFind(source, end, ".name"); + while (p) |found| : (p = bzFind(found + 1, end, ".name")) { + const eq = bzSkipWs(found + ".name".len, end); + const name: MutStr = if (before(eq, end) and eq[0] == '=') bzQuoted(eq + 1, end, &after) else null; + const close: Text = if (name != null) bzMemchr(after, '}', end) else null; + const mod: Text = if (close) |brace| bzFind(after, brace, ".module") else null; + const meq: Text = if (mod) |m| bzSkipWs(m + ".module".len, close.?) else null; + const has_arg = if (meq) |assign| before(assign, close.?) and assign[0] == '=' else false; + const path: MutStr = if (has_arg) bzModuleArgSource(source, meq.? + 1, end) else null; + if (path != null) { + bzPush(entries, rel_path, name, path, false); + } else if (has_arg) { + bzPushDepModule(entries, rel_path, name, source, meq.? + 1, end); + } + c.free(name); + c.free(path); + } +} + +// ── Public: manifest detection + parsing ──────────────────────────── + +pub export fn cbm_pkgmap_try_parse(basename: CStr, rel_path: CStr, source_c: CStr, source_len: c_int, entries: *PkgEntries) bool { + if (basename == null or source_c == null or source_len <= 0) return false; + const source: [*]const u8 = source_c; + + const Parser = *const fn ([*]const u8, c_int, CStr, *PkgEntries) void; + const by_name = [_]struct { [*:0]const u8, Parser }{ + .{ "package.json", parsePackageJson }, + .{ "go.mod", parseGoMod }, + .{ "Cargo.toml", parseCargoToml }, + .{ "pyproject.toml", parsePyprojectToml }, + .{ "composer.json", parseComposerJson }, + .{ "pubspec.yaml", parsePubspecYaml }, + .{ "pom.xml", parsePomXml }, + .{ "build.gradle", parseBuildGradle }, + .{ "build.gradle.kts", parseBuildGradle }, + .{ "mix.exs", parseMixExs }, + }; + for (by_name) |entry| { + if (streq(basename, entry[0])) { + entry[1](source, source_len, rel_path, entries); + return true; + } + } + if (c.cbm_str_ends_with(basename, ".gemspec")) { + parseGemspec(source, source_len, rel_path, entries); + return true; + } + if (streq(basename, "build.zig")) { + parseBuildZig(source, source_len, rel_path, entries); + return true; + } + if (streq(basename, "build.zig.zon")) { + parseBuildZigZon(source, source_len, rel_path, entries); + return true; + } + return false; +} + +// ── Merge: per-worker entries → hash table ────────────────────────── + +/// A composer.json `autoload.psr-4` key: a namespace prefix ending in '\\'. +fn pkgmapIsPsr4Prefix(pkg_name: CStr) bool { + const n: usize = if (pkg_name != null) c.strlen(pkg_name) else 0; + return n > 0 and pkg_name[n - 1] == '\\'; +} + +/// The entry_rel recorded for key, or NULL. +fn pkgmapRawEntry(worker_entries: [*c]PkgEntries, worker_count: c_int, key: CStr) CStr { + for (0..ix(worker_count)) |w| { + for (0..ix(worker_entries[w].count)) |i| { + if (streq(worker_entries[w].items[i].pkg_name, key)) { + return worker_entries[w].items[i].entry_rel; + } + } + } + return null; +} + +/// Zig "zig:" -> "@||": module x of the build.zig in the +/// directory build.zig.zon in gives dependency pkg. Returns the number +/// of entries added. +fn pkgmapResolveZigDeps(map: ?*HashTable, worker_entries: [*c]PkgEntries, worker_count: c_int) c_int { + var added: c_int = 0; + for (0..ix(worker_count)) |w| { + for (0..ix(worker_entries[w].count)) |i| { + const name: CStr = worker_entries[w].items[i].pkg_name; + const v: CStr = worker_entries[w].items[i].entry_rel; + const bar1: CStr = if (v[0] == '@') c.strchr(v, '|') else null; + const bar2: CStr = if (bar1 != null) c.strchr(bar1 + 1, '|') else null; + if (bar2 == null or c.cbm_ht_has(map, name)) continue; + var key: [PKGMAP_PATH_BUF]u8 = undefined; + _ = c.snprintf(&key, key.len, "zigdep:%.*s", @as(c_int, @intCast(ptrDiff(bar2, v) - 1)), v + 1); + const dep_dir = pkgmapRawEntry(worker_entries, worker_count, &key); + if (dep_dir == null) continue; + _ = c.snprintf(&key, key.len, "zigmod:%s|%s", dep_dir, bar2 + 1); + const qn: CStr = @ptrCast(c.cbm_ht_get(map, &key)); + if (qn != null) { + _ = c.cbm_ht_set(map, c.strdup(name), c.strdup(qn)); + added += 1; + } + } + } + return added; +} + +pub export fn cbm_pkgmap_build(worker_entries: [*c]PkgEntries, worker_count: c_int, project_name: CStr) ?*HashTable { + // Count total entries + var total: c_int = 0; + for (0..ix(worker_count)) |w| total += worker_entries[w].count; + if (total == 0) return null; + + const map = c.cbm_ht_create(PKGMAP_HT_INIT); + var merged: c_int = 0; + + for (0..ix(worker_count)) |w| { + const we = util.at(worker_entries, w); + for (0..ix(we.count)) |i| { + const item = util.at(we.items, i); + // Zig dependency entries are resolved below. + if (item.entry_rel[0] == '@' or c.strncmp(item.pkg_name, "zigdep:", "zigdep:".len) == 0) { + continue; + } + // Convert entry_rel to QN: project.dir.parts. A PSR-4 namespace + // prefix (key ends in '\\') keeps its repo-relative DIRECTORY + // instead: its readers map `Prefix\Sub\Class` onto a file path + // (/Sub/Class.php), which a dotted QN cannot express (#1186). + const entry_rel: CStr = item.entry_rel; + const qn: MutStr = if (pkgmapIsPsr4Prefix(item.pkg_name)) + c.cbm_strndup(entry_rel, c.strlen(entry_rel)) + else + c.cbm_pipeline_fqn_module(project_name, entry_rel); + if (qn == null) continue; + + // Check for duplicate — first wins + if (c.cbm_ht_has(map, item.pkg_name)) { + c.free(qn); + continue; + } + + // Transfer ownership: key = strdup'd pkg_name, value = qn + const key: MutStr = c.strdup(item.pkg_name); + _ = c.cbm_ht_set(map, key, qn); + merged += 1; + } + } + + merged += pkgmapResolveZigDeps(map, worker_entries, worker_count); + if (merged == 0) { + c.cbm_ht_free(map); + return null; + } + interop.logInfo("pkgmap.build", .{ @as(CStr, "entries"), @as(CStr, c.cbm_log_itoa(merged)) }); + return map; +} + +/// Returns true if basename is a package manifest we know how to parse. +/// Used by the filesystem walker; cbm_pkgmap_try_parse is the source of +/// truth for which basenames produce entries. +fn isPkgmapManifestBasename(basename: CStr) bool { + if (basename == null) return false; + const names = [_][*:0]const u8{ + "package.json", "go.mod", "Cargo.toml", "pyproject.toml", + "composer.json", "pubspec.yaml", "pom.xml", "build.gradle", + "build.gradle.kts", "mix.exs", "build.zig", "build.zig.zon", + }; + for (names) |name| { + if (streq(basename, name)) return true; + } + return c.cbm_str_ends_with(basename, ".gemspec"); +} + +const is_windows = builtin.os.tag == .windows; + +/// Stat a path, skipping symlinks. Returns 0 on success, -1 to skip. +/// On POSIX, lstat + S_ISLNK avoids following symlink cycles. On Windows +/// the UTF-8-safe wide stat is used (cbm_stat_utf8: _wstat64 by wide path, +/// filling st_mode, st_size and st_mtime); reparse points (junctions/symlinks) +/// are detected separately by pkgmapIsReparsePoint below before we descend. +fn pkgmapSafeStat(abs_path: CStr, st: *c.struct_stat) c_int { + if (is_windows) { + return if (c.cbm_stat_utf8(abs_path, st) != 0) c.CBM_NOT_FOUND else 0; + } + if (c.lstat(abs_path, st) != 0) return c.CBM_NOT_FOUND; + if ((st.st_mode & c.S_IFMT) == c.S_IFLNK) return c.CBM_NOT_FOUND; + return 0; +} + +/// True if abs_path is a Windows reparse point (directory junction or +/// symlink). Following these is the documented cause of the historic CI +/// hang, so we skip them before recursing. On POSIX this is a no-op +/// (symlinks are already filtered by pkgmapSafeStat's S_ISLNK check). +/// Note: the PKGMAP_WALK_MAX_DEPTH bound below is the hard termination +/// guarantee; this check is a best-effort early skip on top of it. +/// +/// The C code asked GetFileAttributesW for FILE_ATTRIBUTE_REPARSE_POINT. +/// cbm_stat_no_follow asks the same question on Windows: it fails for a +/// reparse point, and the path is already known to stat successfully. +fn pkgmapIsReparsePoint(abs_path: CStr) bool { + if (!is_windows) return false; + var st: c.struct_stat = undefined; + return c.cbm_stat_no_follow(abs_path, &st) != 0; +} + +/// Recursive filesystem walker that finds and parses package manifest +/// files independently of the main discovery filter. The main discovery +/// filter intentionally hides package.json / composer.json etc. from +/// code indexing (they're config, not source), but pass_pkgmap still +/// needs to read them to resolve workspace imports. Skips directories +/// matched by the shared cbm_should_skip_dir helper so we don't walk +/// node_modules, .git, build, etc. Returns the number of manifests +/// parsed, accumulated across the whole walk. +/// +/// Cross-platform: uses the portable cbm_opendir/cbm_readdir/cbm_closedir +/// API (same as src/discover) and the symlink-skipping pkgmapSafeStat. +/// Termination is guaranteed by the PKGMAP_WALK_MAX_DEPTH recursion bound — +/// even directory junctions / symlink cycles cannot make it hang. On Windows +/// we additionally skip reparse points before descending as a best-effort +/// early-out. +fn pkgmapWalkDir(abs_dir: CStr, rel_dir: CStr, entries: *PkgEntries, depth: c_int, excluded_dirs: [*c]MutStr, excluded_count: c_int) c_int { + if (depth >= PKGMAP_WALK_MAX_DEPTH) { + interop.logInfo("pkgmap.walk", .{ + @as(CStr, "depth_cap"), if (rel_dir != null and rel_dir[0] != 0) rel_dir else @as(CStr, "."), + }); + return 0; + } + const dir = c.cbm_opendir(abs_dir) orelse return 0; + var parsed: c_int = 0; + while (true) { + const entry = c.cbm_readdir(dir) orelse break; + const name: CStr = &entry.*.name; + if (name[0] == '.' and (name[1] == 0 or (name[1] == '.' and name[2] == 0))) continue; + var abs_path: [PKGMAP_PATH_BUF]u8 = undefined; + var rel_path: [PKGMAP_PATH_BUF]u8 = undefined; + _ = c.snprintf(&abs_path, abs_path.len, "%s/%s", abs_dir, name); + if (rel_dir != null and rel_dir[0] != 0) { + _ = c.snprintf(&rel_path, rel_path.len, "%s/%s", rel_dir, name); + } else { + _ = c.snprintf(&rel_path, rel_path.len, "%s", name); + } + var st: c.struct_stat = undefined; + if (pkgmapSafeStat(&abs_path, &st) != 0) continue; + if (util.isDirMode(&st)) { + if (c.cbm_should_skip_dir(name, c.CBM_MODE_FULL) or + c.cbm_pipeline_relpath_is_excluded(&rel_path, excluded_dirs, excluded_count) != 0) + { + continue; + } + // Don't follow Windows directory junctions — they can form + // cycles. The depth bound is the hard guarantee; this just + // avoids wasted descent. (POSIX symlinks are already skipped by + // pkgmapSafeStat's S_ISLNK check.) + if (pkgmapIsReparsePoint(&abs_path)) continue; + parsed += pkgmapWalkDir(&abs_path, &rel_path, entries, depth + 1, excluded_dirs, excluded_count); + continue; + } + if ((st.st_mode & c.S_IFMT) != c.S_IFREG) continue; + if (!isPkgmapManifestBasename(name)) continue; + var source_len: c_int = 0; + const source = pkgmapReadFile(&abs_path, &source_len); + if (source == null) continue; + if (cbm_pkgmap_try_parse(name, &rel_path, source, source_len, entries)) parsed += 1; + c.free(source); + } + _ = c.cbm_closedir(dir); + return parsed; +} + +/// Scan a repository for package manifest files via the filesystem +/// walker above. Always-available companion to the parallel path's +/// per-worker manifest parsing, which is bound to whatever `files[]` +/// the discoverer produces and therefore misses ignored manifests like +/// package.json. NULL-safe; returns 0 entries when repo_path is unset. +/// +/// Cross-platform: the walk runs on every platform via pkgmapWalkDir, +/// which is depth-bounded (PKGMAP_WALK_MAX_DEPTH) and skips symlinks / +/// Windows reparse points, so it cannot hang on directory junctions. +/// This is what lets bare workspace imports (e.g. "@org/pkg" declared in +/// an ignored package.json) resolve on Windows as well as POSIX. +pub export fn cbm_pkgmap_scan_repo(repo_path: CStr, entries_c: ?*PkgEntries, excluded_dirs: [*c]MutStr, excluded_count: c_int) c_int { + const entries = entries_c orelse return 0; + if (repo_path == null) return 0; + const parsed = pkgmapWalkDir(repo_path, "", entries, 0, excluded_dirs, excluded_count); + interop.logInfo("pkgmap.scan_repo", .{ @as(CStr, "manifests"), @as(CStr, c.cbm_log_itoa(parsed)) }); + return parsed; +} + +fn pkgmapFreeEntry(key: CStr, value: ?*anyopaque, userdata: ?*anyopaque) callconv(.c) void { + _ = userdata; + c.free(@constCast(key)); + c.free(value); +} + +pub export fn cbm_pkgmap_free(pkgmap_c: ?*HashTable) void { + const pkgmap = pkgmap_c orelse return; + c.cbm_ht_foreach(pkgmap, pkgmapFreeEntry, null); + c.cbm_ht_free(pkgmap); +} + +// ── Resolver ──────────────────────────────────────────────────────── + +/// Try slash-based prefix matching (Go: github.com/foo/bar/pkg/utils). +/// Returns heap QN or NULL. +fn resolveSlashPrefix(map: ?*HashTable, module_path: CStr) MutStr { + const buf: MutStr = c.strdup(module_path); + if (buf == null) return null; + var i = c.strlen(buf); + // for (slash = buf + strlen(buf) - 1; slash > buf; slash--) + while (i > 1) { + i -= 1; + if (buf[i] != '/') continue; + buf[i] = 0; + const base_qn: CStr = @ptrCast(c.cbm_ht_get(map, buf)); + if (base_qn == null) continue; + const subpath: CStr = module_path + i + 1; + var result: [PKGMAP_PATH_BUF]u8 = undefined; + _ = c.snprintf(&result, result.len, "%s.%s", base_qn, subpath); + // Replace / with . in the appended part + var j = c.strlen(base_qn) + 1; + while (result[j] != 0) : (j += 1) { + if (result[j] == '/') result[j] = '.'; + } + c.free(buf); + return c.strdup(&result); + } + c.free(buf); + return null; +} + +/// Try dot-based prefix matching (Java: com.myorg.pkg.Foo). +/// Returns heap QN or NULL. +fn resolveDotPrefix(map: ?*HashTable, module_path: CStr, project_name: CStr) MutStr { + const buf: MutStr = c.strdup(module_path); + if (buf == null) return null; + var i = c.strlen(buf); + while (i > 1) { + i -= 1; + if (buf[i] != '.') continue; + buf[i] = 0; + const base_qn: CStr = @ptrCast(c.cbm_ht_get(map, buf)); + if (base_qn == null) continue; + const subpath: CStr = module_path + i + 1; + var subpath_slashed: [PKGMAP_PATH_BUF]u8 = undefined; + _ = c.snprintf(&subpath_slashed, subpath_slashed.len, "%s", subpath); + var j: usize = 0; + while (subpath_slashed[j] != 0) : (j += 1) { + if (subpath_slashed[j] == '.') subpath_slashed[j] = '/'; + } + var result: [PKGMAP_PATH_BUF]u8 = undefined; + _ = c.snprintf(&result, result.len, "%s/%s", base_qn, @as(CStr, &subpath_slashed)); + c.free(buf); + return c.cbm_pipeline_fqn_module(project_name, &result); + } + c.free(buf); + return null; +} + +/// Try backslash-based prefix matching (PHP PSR-4: App\\Controllers\\Foo). +/// Returns heap QN or NULL. +fn resolveBackslashPrefix(map: ?*HashTable, module_path: CStr, project_name: CStr) MutStr { + const buf: MutStr = c.strdup(module_path); + if (buf == null) return null; + var i = c.strlen(buf); + while (i > 1) { + i -= 1; + if (buf[i] != '\\') continue; + buf[i] = 0; + var prefix: [PKGMAP_PATH_BUF]u8 = undefined; + _ = c.snprintf(&prefix, prefix.len, "%s\\", buf); + const base_dir: CStr = @ptrCast(c.cbm_ht_get(map, &prefix)); + if (base_dir == null) continue; + const subpath: CStr = module_path + i + 1; + var path_result: [PKGMAP_PATH_BUF]u8 = undefined; + _ = c.snprintf( + &path_result, + path_result.len, + "%s%s%s", + base_dir, + @as(CStr, if (base_dir[0] != 0) "/" else ""), + subpath, + ); + var j: usize = 0; + while (path_result[j] != 0) : (j += 1) { + if (path_result[j] == '\\') path_result[j] = '/'; + } + c.free(buf); + return c.cbm_pipeline_fqn_module(project_name, &path_result); + } + c.free(buf); + return null; +} + +pub export fn cbm_pipeline_resolve_module(ctx: ?*const Ctx, source_rel: CStr, module_path: CStr) MutStr { + const cx = ctx orelse return c.cbm_pipeline_fqn_module(null, module_path); + if (module_path == null) return c.cbm_pipeline_fqn_module(cx.project_name, module_path); + + // 1. Try relative import resolution (existing logic) + const resolved: MutStr = c.cbm_pipeline_resolve_relative_import(source_rel, module_path); + if (resolved != null) { + // The relative resolver has already removed an explicit JS/TS file + // extension. Treat the remaining path as a module path verbatim so a + // dotted extensionless basename such as `featureX.engine` is not + // stripped a second time by cbm_pipeline_fqn_module. + const qn = c.cbm_pipeline_fqn_folder(cx.project_name, resolved); + c.free(resolved); + return qn; + } + + // 1b. Try build-tool path aliases (tsconfig/jsconfig paths today; + // other loaders can register here later). Independent of pkgmap. + if (cx.path_aliases != null and source_rel != null) { + const amap = c.cbm_path_alias_find_for_file(cx.path_aliases, source_rel); + if (amap != null) { + const aliased: MutStr = c.cbm_path_alias_resolve(amap, module_path); + if (aliased != null) { + const qn = c.cbm_pipeline_fqn_module(cx.project_name, aliased); + c.free(aliased); + return qn; + } + } + } + + // 2. No pkgmap → fall through immediately + const pkgmap = cx.pkgmap; + if (pkgmap == null) return c.cbm_pipeline_fqn_module(cx.project_name, module_path); + + // 2b. Zig: a named `@import("name")` is a build.zig module, never another + // ecosystem's package of the same name. + if (source_rel != null and c.cbm_str_ends_with(source_rel, ".zig")) { + var key: [PKGMAP_PATH_BUF]u8 = undefined; + _ = c.snprintf(&key, key.len, "zig:%s", module_path); + const zig_qn: CStr = @ptrCast(c.cbm_ht_get(pkgmap, &key)); + return if (zig_qn != null) + c.strdup(zig_qn) + else + c.cbm_pipeline_fqn_module(cx.project_name, module_path); + } + + // 3. Exact lookup + const mapped_qn: CStr = @ptrCast(c.cbm_ht_get(pkgmap, module_path)); + if (mapped_qn != null) return c.strdup(mapped_qn); + + // 4. Prefix matching by separator type + var result = resolveSlashPrefix(pkgmap, module_path); + if (result == null) result = resolveDotPrefix(pkgmap, module_path, cx.project_name); + if (result == null) result = resolveBackslashPrefix(pkgmap, module_path, cx.project_name); + if (result != null) return result; + + // 5. Fallthrough to default resolution + return c.cbm_pipeline_fqn_module(cx.project_name, module_path); +} + +// ── Import-target node resolver ───────────────────────────────────── + +/// Return the last path segment of an import string, recognizing the separators +/// used across languages: '.', '/', '\\', and "::". Returns a pointer into +/// `path` (no allocation). +fn importLastSegment(path: CStr) CStr { + var seg = path; + var p = path; + while (p[0] != 0) : (p += 1) { + if (p[0] == '.' or p[0] == '/' or p[0] == '\\' or p[0] == ':') seg = p + 1; + } + return seg; +} + +/// Derive a representative imported symbol name from a raw module/use path, +/// stripping language decorations so it can be matched against an in-graph node: +/// - trailing alias " as X" (Rust/Kotlin) +/// - trailing glob "::*" / ".*" (Rust/Kotlin/Java wildcard) +/// - brace groups "{a, b, ...}" (Rust grouped use) → first member +/// Writes the result into `out` (size `outsz`) and returns it, or NULL if none. +/// For grouped/braced forms the first listed symbol is used as the representative +/// (the tests only require at least one resolved IMPORTS edge per statement). +fn importCandidateSymbol(module_path: CStr, out: MutStr, outsz: usize) CStr { + if (module_path == null or module_path[0] == 0) return null; + var buf: [1024]u8 = undefined; + _ = c.snprintf(&buf, buf.len, "%s", module_path); + + // Brace group: `prefix::{a, b}` → take first member `a`. + const brace: MutStr = c.strchr(&buf, '{'); + if (brace != null) { + var first: MutStr = brace + 1; + var end: MutStr = first; + while (end[0] != 0 and end[0] != ',' and end[0] != '}') end += 1; + end[0] = 0; + // Trim whitespace. + while (isBlank(first[0])) first += 1; + var t: MutStr = first + c.strlen(first); + while (@intFromPtr(t) > @intFromPtr(first) and isBlank((t - 1)[0])) { + t -= 1; + t[0] = 0; + } + if (first[0] != 0 and !streq(first, "self")) { + _ = c.snprintf(out, outsz, "%s", first); + return out; + } + // `{self, ...}` → fall back to the path before the brace group. + brace[0] = 0; + } + + // Strip trailing " as ". + const as: MutStr = c.strstr(&buf, " as "); + if (as != null) as[0] = 0; + + // Strip trailing glob and separator noise. + var len = c.strlen(&buf); + while (len > 0 and (buf[len - 1] == '*' or buf[len - 1] == ':' or buf[len - 1] == '.' or + buf[len - 1] == '/' or buf[len - 1] == '\\' or buf[len - 1] == ' ')) + { + len -= 1; + buf[len] = 0; + } + if (buf[0] == 0) return null; + const seg = importLastSegment(&buf); + if (seg == null or seg[0] == 0 or streq(seg, "*")) return null; + _ = c.snprintf(out, outsz, "%s", seg); + return out; +} + +/// True for node labels that represent an importable definition (so a symbol-name +/// fallback does not link to, e.g., a Variable or Field). +fn importTargetableLabel(label: CStr) bool { + if (label == null) return false; + const ok = [_][*:0]const u8{ + "Class", "Interface", "Function", "Method", "Module", + "Struct", "Enum", "Trait", "Type", "File", + }; + for (ok) |l| { + if (streq(l, label)) return true; + } + return false; +} + +/// #1934: whether the name-guess import fallbacks — Strategy 1b (sibling file, +/// whose label filter admits symbols) and Strategy 3 (symbol name) — may run +/// for imports from this language. A Go import path names a package — never a +/// function, method +/// or field — and every correct Go import resolves in Strategy 1 (module path +/// → the package's Folder node); when that misses the import is external and +/// the correct result is NO edge. The fallback instead bound the last path +/// segment to an arbitrary same-named project symbol (`import "os/exec"` → a +/// test harness's exec() method, two imports → a Makefile target). Languages +/// whose import genuinely can name a member (Python `from m import f`, Java +/// `import com.example.Foo`, Rust `use crate::ops::helper`) keep it. +pub export fn cbm_import_symbol_fallback_allowed(lang: c.CBMLanguage) bool { + return lang != c.CBM_LANG_GO; +} + +/// "/" after a non-empty directory, "" after an empty one. +fn dirSep(dir: CStr) CStr { + return if (dir[0] != 0) "/" else ""; +} + +/// True when `n` may be an import target of the file `source_file_qn`: it is +/// not that file itself. +fn notSelf(n: *const Node, source_file_qn: CStr) bool { + return source_file_qn == null or n.qualified_name == null or + !streq(n.qualified_name, source_file_qn); +} + +/// Resolve a sibling-file import: a bare path/name (no leading "./") that names +/// a file relative to the importer's directory. This covers build/markup +/// grammars whose import string is a sibling filename or directory rather than a +/// dotted module path: +/// - SCSS `@use 'vars'` → sibling `_vars.scss` (partial underscore) +/// - Just `import 'common.just'` → sibling `common.just` +/// - BitBake `require mypackage.inc` → sibling `mypackage.inc` +/// - Meson `subdir('lib')` → `lib/meson.build` +/// - func `#include "utils.fc"` → sibling `utils.fc` +/// - Pony `use "util"` → sibling `util.pony` +/// Builds a path relative to source_rel's directory, then looks up the resulting +/// File/Module-node QN (extension is stripped by fqn_module). Returns a borrowed +/// node or NULL. Several filename conventions are tried in turn. +fn resolveSiblingFile(ctx: *const Ctx, source_rel: CStr, source_file_qn: CStr, module_path: CStr) ?*const Node { + if (module_path == null or module_path[0] == 0) return null; + // Directory of the importing file (empty for repo-root files). + const dir: MutStr = pathDirname(if (source_rel != null) source_rel else ""); + if (dir == null) return null; + + // Candidate relative paths, in priority order. + var cands: [5][PKGMAP_PATH_BUF]u8 = undefined; + var ncand: usize = 0; + var base = module_path; + // Skip a leading "./". + if (base[0] == '.' and base[1] == '/') base += 2; + // 1. Direct sibling: dir/. + _ = c.snprintf(&cands[ncand], PKGMAP_PATH_BUF, "%s%s%s", dir, dirSep(dir), base); + ncand += 1; + // 2. SCSS partial: dir/[subdir/]_.scss (underscore-prefixed). + { + const slash: CStr = c.strrchr(base, '/'); + const bn: CStr = if (slash != null) slash + 1 else base; + const dpart: MutStr = if (slash != null) + c.cbm_strndup(base, ptrDiff(slash, base)) + else + c.strdup(""); + if (dpart != null and bn[0] != '_') { + _ = c.snprintf( + &cands[ncand], + PKGMAP_PATH_BUF, + "%s%s%s%s_%s.scss", + dir, + dirSep(dir), + @as(CStr, if (dpart[0] != 0) dpart else ""), + dirSep(dpart), + bn, + ); + ncand += 1; + } + c.free(dpart); + } + // 3. Meson subdir: dir//meson.build. + _ = c.snprintf(&cands[ncand], PKGMAP_PATH_BUF, "%s%s%s/meson.build", dir, dirSep(dir), base); + ncand += 1; + // 4. Basename sibling: dir/. Covers include paths + // that carry a non-relative prefix (Hyprlang `source = ~/.config/.../x.conf`, + // absolute include paths) but reference a file sitting beside the importer. + { + const slash: CStr = c.strrchr(base, '/'); + if (slash != null and slash[1] != 0) { + _ = c.snprintf(&cands[ncand], PKGMAP_PATH_BUF, "%s%s%s", dir, dirSep(dir), slash + 1); + ncand += 1; + } + } + + var found: ?*const Node = null; + for (cands[0..ncand]) |*cand| { + const qn: MutStr = c.cbm_pipeline_fqn_module(ctx.project_name, cand); + if (qn == null) continue; + const n: ?*const Node = c.cbm_gbuf_find_by_qn(ctx.gbuf, qn); + c.free(qn); + if (n) |node| { + if (importTargetableLabel(node.label) and notSelf(node, source_file_qn)) { + found = node; + break; + } + } + } + c.free(dir); + return found; +} + +// ── PHP PSR-4 class imports (#1186) ───────────────────────────────── + +const Psr4Outcome = enum { + /// no composer psr-4 prefix covers the import + not_applicable, + /// the class file exists: *out is its File node + resolved, + /// a prefix covers it but no class file exists + unresolved, +}; + +fn pathLeaf(path: CStr) CStr { + var leaf = path; + var p = path; + while (p != null and p[0] != 0) : (p += 1) { + if (p[0] == '/' or p[0] == '\\') leaf = p + 1; + } + return leaf; +} + +/// The File node at exactly `rel_path`, or NULL. Looked up through the name +/// index (File nodes are named by basename) and matched on the full path, so +/// a same-named file in another directory never qualifies. +fn psr4FileNode(gbuf: ?*const Gbuf, rel_path: CStr) ?*const Node { + var hits: [*c][*c]const Node = null; + var hit_count: c_int = 0; + if (c.cbm_gbuf_find_by_name(gbuf, pathLeaf(rel_path), &hits, &hit_count) != 0 or hits == null) { + return null; + } + for (0..ix(hit_count)) |i| { + const n: *const Node = hits[i] orelse continue; + if (n.label != null and streq(n.label, "File") and n.file_path != null and + streq(n.file_path, rel_path)) + { + return n; + } + } + return null; +} + +/// Resolve a PHP class import `use A\B\C;` the way composer's PSR-4 autoloader +/// does: for every autoload.psr-4 prefix covering the name, longest first, the +/// class lives in exactly /.php with sub-namespaces as +/// subdirectories; the first existing file wins. When a prefix covers the name +/// but no such file exists the import is UNRESOLVED: it must never fall through +/// to the namespace bucket, which binds it to whichever file of the namespace +/// came first. `use function` / `use const` name namespace members, not class +/// files, so they are not applicable. +fn resolvePhpPsr4Class(gbuf: ?*const Gbuf, source_file_qn: CStr, imp: *const c.CBMImport, out: *?*const Node) Psr4Outcome { + out.* = null; + const pkgmap = c.cbm_pipeline_get_pkgmap(); + if (pkgmap == null or imp.kind != c.CBM_IMPORT_KIND_DEFAULT) return .not_applicable; + var name: CStr = imp.module_path; + if (name[0] == '\\') name += 1; // fully qualified `use \App\X;` + var key: [PKGMAP_PATH_BUF]u8 = undefined; + const name_len = c.strlen(name); + if (name_len == 0 or name_len >= key.len or c.strchr(name, '\\') == null) { + return .not_applicable; + } + var covered = false; + // Right to left: each '\' ends a candidate prefix, longest first. + var cut = name_len - 1; + while (cut > 0) : (cut -= 1) { + if (name[cut - 1] != '\\') continue; + @memcpy(key[0..cut], name[0..cut]); + key[cut] = 0; + const dir: CStr = @ptrCast(c.cbm_ht_get(pkgmap, &key)); + if (dir == null) continue; + covered = true; + var rel: [PKGMAP_PATH_BUF]u8 = undefined; + const w = c.snprintf(&rel, rel.len, "%s%s%s.php", dir, dirSep(dir), name + cut); + if (w <= 0 or ix(w) >= rel.len) continue; + var j: usize = 0; + while (rel[j] != 0) : (j += 1) { + if (rel[j] == '\\') rel[j] = '/'; + } + if (psr4FileNode(gbuf, &rel)) |file| { + if (notSelf(file, source_file_qn)) { + out.* = file; + return .resolved; + } + } + } + return if (covered) .unresolved else .not_applicable; +} + +fn psr4ClassLabel(label: CStr) bool { + return label != null and (streq(label, "Class") or streq(label, "Interface") or + streq(label, "Trait") or streq(label, "Enum")); +} + +/// The QN of `member` on the class `cls` defined in `file_path`, or NULL when +/// that file defines no such class or the class has no such member. +fn psr4MemberInFile(gbuf: ?*const Gbuf, file_path: CStr, cls: CStr, member: CStr) CStr { + var hits: [*c][*c]const Node = null; + var hit_count: c_int = 0; + if (c.cbm_gbuf_find_by_name(gbuf, cls, &hits, &hit_count) != 0 or hits == null) return null; + for (0..ix(hit_count)) |i| { + const cn: *const Node = hits[i] orelse continue; + if (!psr4ClassLabel(cn.label) or cn.qualified_name == null or cn.file_path == null or + !streq(cn.file_path, file_path)) + { + continue; + } + var qn: [c.CBM_SZ_1K]u8 = undefined; + const w = c.snprintf(&qn, qn.len, "%s.%s", cn.qualified_name, member); + if (w <= 0 or ix(w) >= qn.len) return null; + const m: ?*const Node = c.cbm_gbuf_find_by_qn(gbuf, &qn); + return if (m) |node| node.qualified_name else null; + } + return null; +} + +pub export fn cbm_pipeline_psr4_member_qn(gbuf: ?*const Gbuf, class_fqn: CStr, member: CStr, out_qn: *CStr) c.cbm_psr4_member_t { + out_qn.* = null; + if (gbuf == null or class_fqn == null or member == null or member[0] == 0) { + return c.CBM_PSR4_MEMBER_NOT_COVERED; + } + var imp = std.mem.zeroes(c.CBMImport); + imp.module_path = class_fqn; + imp.kind = c.CBM_IMPORT_KIND_DEFAULT; + var file: ?*const Node = null; + const placed = resolvePhpPsr4Class(gbuf, null, &imp, &file); + if (placed == .not_applicable) return c.CBM_PSR4_MEMBER_NOT_COVERED; + // Covered from here on: composer would load this file or none, so a + // missing file, class or member leaves the member unresolved. + if (placed != .resolved or file.?.file_path == null) return c.CBM_PSR4_MEMBER_UNRESOLVED; + var cls: CStr = c.strrchr(class_fqn, '\\'); + cls = if (cls != null) cls + 1 else class_fqn; + out_qn.* = psr4MemberInFile(gbuf, file.?.file_path, cls, member); + return if (out_qn.* != null) c.CBM_PSR4_MEMBER_RESOLVED else c.CBM_PSR4_MEMBER_UNRESOLVED; +} + +/// Copies `module_path` into `buf` and cuts it at a brace group and at a +/// trailing " as ". +fn copyUndecorated(buf: *[1024]u8, module_path: CStr) void { + _ = c.snprintf(buf, buf.len, "%s", module_path); + const brace: MutStr = c.strchr(buf, '{'); + if (brace != null) brace[0] = 0; + const as: MutStr = c.strstr(buf, " as "); + if (as != null) as[0] = 0; +} + +/// Turns the '\\', ':' and '/' separators of a NUL-terminated path into '.'. +fn dotSeparators(buf: [*]u8) void { + var i: usize = 0; + while (buf[i] != 0) : (i += 1) { + if (buf[i] == '\\' or buf[i] == ':' or buf[i] == '/') buf[i] = '.'; + } +} + +pub export fn cbm_pipeline_resolve_import_node( + ctx_c: ?*const Ctx, + source_rel: CStr, + source_file_qn: CStr, + imp_c: ?*const c.CBMImport, + namespace_map: ?*HashTable, +) ?*const Node { + const ctx = ctx_c orelse return null; + const imp = imp_c orelse return null; + if (imp.module_path == null) return null; + + // PHP class imports covered by a composer psr-4 prefix name exactly one + // file; when it is absent the import stays unresolved (#1186). + var psr4_target: ?*const Node = null; + switch (resolvePhpPsr4Class(ctx.gbuf, source_file_qn, imp, &psr4_target)) { + .resolved => return psr4_target, + .unresolved => return null, + .not_applicable => {}, + } + + // Strategy 1: module-path resolution → existing node (Python/TS/Go). + // No label filter here: directory-module languages (Go/Java packages) + // legitimately resolve straight to a Folder node -- that's the intended, + // correct import target, not a collision. The Folder-collision problem + // (#767) only shows up downstream, in Strategy 4's retry-with-truncated- + // path loop, which re-enters resolve_module with a DIFFERENT, shortened + // string that the original import never named. + const target_qn = cbm_pipeline_resolve_module(ctx, source_rel, imp.module_path); + const target_opt: ?*const Node = + if (target_qn != null) c.cbm_gbuf_find_by_qn(ctx.gbuf, target_qn) else null; + c.free(target_qn); + if (target_opt) |target| { + // Python/TS from-import of a member: module_path is often + // "pkg.mod.symbol" while resolve_module lands on the Module node + // "pkg.mod". Prefer the member Function/Class when it exists — + // required for `from M import f as g` so CALLS can import_map to f + // (local_name g ≠ f). + if (target.label != null and (streq(target.label, "Module") or streq(target.label, "File"))) { + var symbuf: [256]u8 = undefined; + const sym = importCandidateSymbol(imp.module_path, &symbuf, symbuf.len); + if (sym != null and sym[0] != 0 and !streq(sym, "*")) { + const mod_tail = importLastSegment( + if (target.qualified_name != null) target.qualified_name else "", + ); + if (mod_tail == null or !streq(mod_tail, sym)) { + var member_qn: [c.CBM_SZ_512]u8 = undefined; + _ = c.snprintf(&member_qn, member_qn.len, "%s.%s", target.qualified_name, sym); + const member: ?*const Node = c.cbm_gbuf_find_by_qn(ctx.gbuf, &member_qn); + if (member) |m| { + if (importTargetableLabel(m.label) and !streq(m.label, "Module") and + !streq(m.label, "File")) + { + return m; + } + } + } + } + } + return target; + } + + // Name-guess fallbacks below (Strategy 1b sibling-file, Strategy 3 + // symbol-name) are gated per importing-file language — see + // cbm_import_symbol_fallback_allowed (#1934). + var src_base: CStr = if (source_rel != null) source_rel else ""; + { + var pb = src_base; + while (pb[0] != 0) : (pb += 1) { + if (pb[0] == '/' or pb[0] == '\\') src_base = pb + 1; + } + } + const symbol_fallback_allowed = + cbm_import_symbol_fallback_allowed(c.cbm_language_for_filename(src_base)); + + // Strategy 1b: sibling-file resolution for build/markup grammars whose + // import string is a sibling filename or directory (SCSS partials, Just/ + // BitBake/func includes, Meson subdir, Pony use). Its label filter admits + // symbols too, so for Go it re-creates the Strategy-3 bug one directory + // closer (`os/exec` → a same-package exec() method) — gated the same. + if (symbol_fallback_allowed) { + if (resolveSiblingFile(ctx, source_rel, source_file_qn, imp.module_path)) |sib| return sib; + } + + // Strategy 2: namespace map. `using App.Utils`, `import com.example.Foo`, + // `use App\Utils\Helper` name a NAMESPACE (or a member of it) that the + // path-based QN cannot express. Try the full module path and progressively + // shorter prefixes (dropping the trailing member segment) so both a bare + // namespace import and a member import resolve to the declaring file. + if (namespace_map != null) { + // Normalize separators to '.' for namespace keys (PHP uses '\\'). + // Strip decorations first so `com.example.Util as U`, `crate::ops::*` + // and `App\Utils\{A, B}` reduce to a clean dotted path. + var norm: [1024]u8 = undefined; + copyUndecorated(&norm, imp.module_path); + dotSeparators(&norm); + // Strip trailing glob/separator noise. + var nl = c.strlen(&norm); + while (nl > 0 and (norm[nl - 1] == '*' or norm[nl - 1] == '.' or norm[nl - 1] == ' ')) { + nl -= 1; + norm[nl] = 0; + } + // Collapse any "::"-induced empty segments ("a..b" → "a.b"). + while (true) { + const dd: MutStr = c.strstr(&norm, ".."); + if (dd == null) break; + _ = c.memmove(dd, dd + 1, c.strlen(dd + 1) + 1); + } + while (true) { + // The map value is a '\n'-delimited list of __file__ QNs declaring + // this namespace. Return the FIRST that is not the importing file, + // so a same-package wildcard import (`import com.example.*` from a + // file that is itself in com.example) resolves to a sibling — and + // deterministically, independent of file-iteration order across + // platforms (amd64 vs arm64). + var seg: CStr = @ptrCast(c.cbm_ht_get(namespace_map, &norm)); + while (seg != null and seg[0] != 0) { + const eol: CStr = c.strchr(seg, '\n'); + const len = if (eol != null) ptrDiff(eol, seg) else c.strlen(seg); + var qbuf: [1024]u8 = undefined; + if (len > 0 and len < qbuf.len) { + @memcpy(qbuf[0..len], seg[0..len]); + qbuf[len] = 0; + const n: ?*const Node = c.cbm_gbuf_find_by_qn(ctx.gbuf, &qbuf); + if (n) |node| { + if (source_file_qn == null or !streq(node.qualified_name, source_file_qn)) { + return node; + } + } + } + seg = if (eol != null) eol + 1 else null; + } + const dot: MutStr = c.strrchr(&norm, '.'); + if (dot == null) break; + dot[0] = 0; + } + } + + // Strategy 3: symbol-name fallback. Derive a representative imported + // symbol (handling alias / glob / grouped forms) and match it against an + // in-graph definition of the same simple name in another file + // (Rust `helper`, Java `Util`, Kotlin grouped, ...). + // Gated per importing-file language, like Strategy 1b above (#1934). + var symbuf: [256]u8 = undefined; + // Prefer the clean candidate from the module path; the local_name may be an + // alias (Rust `as h`, Kotlin `as U`) that names no real symbol. + var seg = importCandidateSymbol(imp.module_path, &symbuf, symbuf.len); + if (seg == null and imp.local_name != null and imp.local_name[0] != 0 and + !streq(imp.local_name, "*")) + { + seg = imp.local_name; + } + // Build the candidate symbol list: the derived symbol plus, when the import + // is a dotted member path like `com.example.Config.DEFAULT`, the enclosing + // type segments (`Config`). This resolves object/class-member imports to + // the declaring type when the leaf member isn't an importable node. + var cands: [8]CStr = undefined; + var ncands: usize = 0; + if (seg != null and seg[0] != 0 and !streq(seg, "*")) { + cands[ncands] = seg; + ncands += 1; + } + { + // Strip decorations, normalize separators to '.', and collect the + // trailing path segments (last first) as fallback candidates. + var mp: [1024]u8 = undefined; + copyUndecorated(&mp, imp.module_path); + dotSeparators(&mp); + // Walk segments from the end. + var end = c.strlen(&mp); + while (end > 0 and ncands < cands.len) { + var dot: ?usize = null; + var p = end; + while (p > 0) { + p -= 1; + if (mp[p] == '.') { + dot = p; + break; + } + } + const s: CStr = if (dot) |d| @as(CStr, &mp) + d + 1 else ∓ + if (s[0] != 0 and !streq(s, "*")) { + var dup = false; + for (cands[0..ncands]) |cand| { + if (streq(cand, s)) { + dup = true; + break; + } + } + if (!dup) { + cands[ncands] = s; + ncands += 1; + } + } + const d = dot orelse break; + mp[d] = 0; + end = d; + } + if (symbol_fallback_allowed) for (cands[0..ncands]) |name| { + var hits: [*c][*c]const Node = null; + var n: c_int = 0; + if (c.cbm_gbuf_find_by_name(ctx.gbuf, name, &hits, &n) == 0 and hits != null) { + // Deterministic winner: hits[] is in node-registration order, + // which under parallel extraction varies run to run — taking + // the FIRST targetable hit made the same import resolve to + // different same-named nodes across runs (xfs: 360 flickering + // IMPORTS diff lines between two MT runs). Pick the candidate + // with the lexicographically smallest qualified name instead: + // stable, content-derived, identical for ST and MT. + var best: ?*const Node = null; + for (0..ix(n)) |i| { + const cand: *const Node = hits[i] orelse continue; + if (!importTargetableLabel(cand.label)) continue; + if (source_file_qn != null and cand.qualified_name != null and + streq(cand.qualified_name, source_file_qn)) + { + continue; // self + } + if (best == null or (cand.qualified_name != null and + best.?.qualified_name != null and + c.strcmp(cand.qualified_name, best.?.qualified_name) < 0)) + { + best = cand; + } + } + if (best) |b| return b; + } + }; + } + + // Strategy 4: crate-relative module path → File/Module node. Rust glob + // `use crate::ops::*` names a module, not a symbol; strip the glob and the + // `crate::`/`self::`/`super::` prefix, convert `::`→`/`, then resolve the + // remaining path (and successive prefixes) to a Module/File node. + { + var mp: [1024]u8 = undefined; + copyUndecorated(&mp, imp.module_path); + // Convert "::" → "/" (drop the doubled colon cleanly). + var clean: [1024]u8 = undefined; + var ci: usize = 0; + var pi: usize = 0; + while (mp[pi] != 0 and ci + 1 < clean.len) : (pi += 1) { + if (mp[pi] == ':') { + if (ci > 0 and clean[ci - 1] == '/') continue; // collapse "::" → single "/" + clean[ci] = '/'; + ci += 1; + } else if (mp[pi] == '*' or mp[pi] == ' ') { + continue; + } else { + clean[ci] = mp[pi]; + ci += 1; + } + } + clean[ci] = 0; + // Drop trailing slashes. + while (ci > 0 and clean[ci - 1] == '/') { + ci -= 1; + clean[ci] = 0; + } + // Strip a leading crate-relative prefix. + var body: CStr = &clean; + for ([_][]const u8{ "crate/", "self/", "super/" }) |pf| { + if (c.strncmp(body, pf.ptr, pf.len) == 0) { + body += pf.len; + break; + } + } + if (body[0] != 0) { + var work: [1024]u8 = undefined; + _ = c.snprintf(&work, work.len, "%s", body); + while (true) { + const rqn = cbm_pipeline_resolve_module(ctx, source_rel, &work); + const n: ?*const Node = if (rqn != null) c.cbm_gbuf_find_by_qn(ctx.gbuf, rqn) else null; + c.free(rqn); + if (n) |node| { + if (importTargetableLabel(node.label) and notSelf(node, source_file_qn)) { + return node; + } + } + const sl: MutStr = c.strrchr(&work, '/'); + if (sl == null) break; + sl[0] = 0; + } + } + } + + return null; +} + +// ── Namespace map ─────────────────────────────────────────────────── + +pub export fn cbm_pipeline_namespace_map_build( + project_name: CStr, + results: [*c][*c]c.CBMFileResult, + spill: ?*c.cbm_result_spill_t, + rels: [*c]const CStr, + count: c_int, +) ?*HashTable { + var map: ?*HashTable = null; + for (0..ix(count)) |i| { + if (rels[i] == null) continue; + const r: ?*c.CBMFileResult = c.cbm_result_cache_acquire(results, spill, @intCast(i)); + const key: MutStr = if (r != null and r.?.namespace_name != null and r.?.namespace_name[0] != 0) + c.strdup(r.?.namespace_name) + else + null; + c.cbm_result_cache_release(results, @intCast(i), r); + if (key == null) continue; + if (map == null) { + map = c.cbm_ht_create(c.CBM_SZ_64); + if (map == null) { + c.free(key); + return null; + } + } + const file_qn: MutStr = c.cbm_pipeline_fqn_compute(project_name, rels[i], "__file__"); + if (file_qn == null) { + c.free(key); + continue; + } + // Normalize the namespace key to dot-separated form so it matches the + // dot-normalized lookups in cbm_pipeline_resolve_import_node (PHP uses + // '\\', some grammars '::' or '/'). + dotSeparators(key); + // Store ALL files declaring a namespace as a '\n'-delimited list so the + // resolver can pick a non-importer sibling (see resolve loop). The hash + // table does not copy keys, so the strdup'd key is owned by the map and + // freed in nsMapFreeEntry. + if (!c.cbm_ht_has(map, key)) { + _ = c.cbm_ht_set(map, key, file_qn); // map owns key + file_qn + } else { + // Append to the existing list. Re-key with the STORED key pointer + // (not our fresh strdup) so the map's key pointer never changes — + // otherwise Verstable would adopt the new key and our free(key) + // below would free the live key (use-after-free). + const stored_key: CStr = c.cbm_ht_get_key(map, key); + const cur: CStr = @ptrCast(c.cbm_ht_get(map, key)); + if (stored_key != null and cur != null) { + const need = c.strlen(cur) + 1 + c.strlen(file_qn) + 1; + const combined: MutStr = @ptrCast(c.malloc(need)); + if (combined != null) { + _ = c.snprintf(combined, need, "%s\n%s", cur, file_qn); + const prev = c.cbm_ht_set(map, stored_key, combined); + c.free(prev); // old value string + } + } + c.free(key); // our fresh strdup — never stored + c.free(file_qn); // content copied into combined + } + } + return map; +} + +fn nsMapFreeEntry(key: CStr, value: ?*anyopaque, ud: ?*anyopaque) callconv(.c) void { + _ = ud; + c.free(@constCast(key)); // strdup'd in cbm_pipeline_namespace_map_build + c.free(value); +} + +pub export fn cbm_pipeline_namespace_map_free(map: ?*HashTable) void { + if (map == null) return; + c.cbm_ht_foreach(map, nsMapFreeEntry, null); + c.cbm_ht_free(map); +} diff --git a/src/pipeline/pass_route_nodes.c b/src/pipeline/pass_route_nodes.c deleted file mode 100644 index e395bc3f..00000000 --- a/src/pipeline/pass_route_nodes.c +++ /dev/null @@ -1,1285 +0,0 @@ -/* - * pass_route_nodes.c — Create Route nodes for HTTP_CALLS/ASYNC_CALLS edges. - * - * After parallel resolve merges edges into the main gbuf, HTTP_CALLS and - * ASYNC_CALLS edges have url_path/method/broker in properties but point to - * the library function (e.g., requests.get). This pass: - * - * 1. Scans all HTTP_CALLS/ASYNC_CALLS edges - * 2. Extracts url_path from edge properties - * 3. Creates Route nodes with deterministic QNs (__route__METHOD__/path) - * 4. Re-targets edges from library function → Route node - * - * Route nodes are the rendezvous point for cross-service communication: - * Service A: checkout() → HTTP_CALLS → Route("POST /api/orders") - * Service B: create_order() → HANDLES → Route("POST /api/orders") - */ -#include "foundation/constants.h" -#include "foundation/str_util.h" // cbm_json_escape -#include // validate assembled DATA_FLOWS props - -enum { - RN_MAX_SCHEME = 12, - RN_MIN_SCHEME = 3, - RN_STRIP_PASSES = 2, - RN_SCHEME_SKIP = 3, - RN_ARGS_SKIP = 7, - RN_PROPS_MARGIN = 100, - RN_CALLEE_SKIP = 14, /* strlen('"callee_args":[') - 1 */ -}; - -#define SLEN(s) (sizeof(s) - 1) -#include "pipeline/pipeline_internal.h" -#include -#include "graph_buffer/graph_buffer.h" -#include "foundation/log.h" -#include "service_patterns.h" /* cbm_service_pattern_is_http_route_literal */ - -#include -#include -#include - -/* True for characters that may appear in a ":name" route parameter. */ -static inline bool is_route_ident_char(char c) { - return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_'; -} - -/* Canonicalize route-path parameter placeholders to a single "{}" token so that - * client call sites and server handlers rendezvous on the same Route QN - * regardless of framework syntax. Each parameter token collapses to "{}": - * - * :name Express / React-Router / Rails / typical JS API clients - * {name} Axum / Spring / OpenAPI / ASP.NET - * Flask / Rocket (incl. typed "") - * ${...} JS template interpolation captured into the path - * - * Parameter names are intentionally discarded so the same logical endpoint - * matches across services that name the path variable differently. Static path - * text is copied verbatim; the result never exceeds the input length. */ -const char *cbm_route_canon_path(const char *in, char *out, size_t out_sz) { - if (out == NULL || out_sz == 0) { - /* WHY: cppcheck value-flow follows a caller that passes an as-yet - * uninitialized output buffer (e.g. `char cpath[256];`) and flags this - * `return out` as uninitvar. False positive: we only return the pointer - * here and never read the buffer; returning it (NULL, or a buffer too - * small to write into) is the intended early-out contract. */ - // cppcheck-suppress uninitvar - return out; - } - if (in == NULL) { - out[0] = '\0'; - return out; - } - const size_t last = out_sz - 1; - size_t oi = 0; - size_t i = 0; - while (in[i] != '\0' && oi < last) { - char c = in[i]; - bool at_seg_start = (oi == 0) || (out[oi - 1] == '/'); - bool is_param = false; - - if (c == ':' && at_seg_start && is_route_ident_char(in[i + 1])) { - i++; - while (in[i] != '\0' && is_route_ident_char(in[i])) { - i++; - } - is_param = true; - } else if (c == '{') { - i++; - while (in[i] != '\0' && in[i] != '}' && in[i] != '/') { - i++; - } - if (in[i] == '}') { - i++; - } - is_param = true; - } else if (c == '<') { - i++; - while (in[i] != '\0' && in[i] != '>' && in[i] != '/') { - i++; - } - if (in[i] == '>') { - i++; - } - is_param = true; - } else if (c == '$' && in[i + 1] == '{') { - i += 2; - while (in[i] != '\0' && in[i] != '}' && in[i] != '/') { - i++; - } - if (in[i] == '}') { - i++; - } - is_param = true; - } - - if (is_param) { - if (oi + 2 > last) { - break; - } - out[oi++] = '{'; - out[oi++] = '}'; - continue; - } - out[oi++] = c; - i++; - } - out[oi] = '\0'; - return out; -} - -/* Extract a JSON string value by key from properties. - * Returns pointer into buf (caller provides buffer). NULL if not found. */ -static const char *json_extract(const char *json, const char *key, char *buf, int bufsz) { - if (!json || !key) { - return NULL; - } - /* Build "key":" pattern */ - char pattern[CBM_SZ_128]; - snprintf(pattern, sizeof(pattern), "\"%s\":\"", key); - const char *start = strstr(json, pattern); - if (!start) { - return NULL; - } - start += strlen(pattern); - const char *end = strchr(start, '"'); - if (!end || end == start) { - return NULL; - } - int len = (int)(end - start); - if (len >= bufsz) { - len = bufsz - SKIP_ONE; - } - memcpy(buf, start, (size_t)len); - buf[len] = '\0'; - return buf; -} - -/* Visitor context for edge scanning */ -typedef struct { - cbm_gbuf_t *gb; - int created; -} route_ctx_t; - -static void route_edge_visitor(const cbm_gbuf_edge_t *edge, void *userdata) { - route_ctx_t *ctx = (route_ctx_t *)userdata; - - /* Only process HTTP_CALLS and ASYNC_CALLS */ - if (strcmp(edge->type, "HTTP_CALLS") != 0 && strcmp(edge->type, "ASYNC_CALLS") != 0) { - return; - } - - /* Extract url_path from properties */ - char url_buf[CBM_SZ_512]; - const char *url = json_extract(edge->properties_json, "url_path", url_buf, sizeof(url_buf)); - if (!url || !url[0]) { - return; - } - char callee_buf[CBM_SZ_256]; - const char *callee = - json_extract(edge->properties_json, "callee", callee_buf, sizeof(callee_buf)); - if (strcmp(edge->type, "HTTP_CALLS") == 0 && - !cbm_service_pattern_is_http_route_literal(url, callee)) { - return; - } - - /* Extract method or broker */ - char method_buf[CBM_SZ_16]; - char broker_buf[CBM_SZ_64]; - const char *method = - json_extract(edge->properties_json, "method", method_buf, sizeof(method_buf)); - const char *broker = - json_extract(edge->properties_json, "broker", broker_buf, sizeof(broker_buf)); - - /* Build Route QN */ - char route_qn[CBM_ROUTE_QN_SIZE]; - if (strcmp(edge->type, "HTTP_CALLS") == 0) { - char cpath[CBM_SZ_256]; - snprintf(route_qn, sizeof(route_qn), "__route__%s__%s", method ? method : "ANY", - cbm_route_canon_path(url, cpath, sizeof(cpath))); - } else { - snprintf(route_qn, sizeof(route_qn), "__route__%s__%s", broker ? broker : "async", url); - } - - /* Build properties for Route node */ - char route_props[CBM_SZ_256]; - if (method) { - snprintf(route_props, sizeof(route_props), "{\"method\":\"%s\"}", method); - } else if (broker) { - snprintf(route_props, sizeof(route_props), "{\"broker\":\"%s\"}", broker); - } else { - snprintf(route_props, sizeof(route_props), "{}"); - } - - /* Create or find Route node (deduped by QN) */ - cbm_gbuf_upsert_node(ctx->gb, "Route", url, route_qn, "", 0, 0, route_props); - ctx->created++; - - /* Note: we do NOT re-target the edge here because modifying edges during - * iteration is unsafe. The edge stays pointing to the library function. - * The URL-in-args detection in pass_parallel will create Route→handler HANDLES - * edge separately. The caller→library edge - * with url_path in properties is sufficient for query_graph to find - * the Route via: caller → HTTP_CALLS(url_path="/api/x") + Route("/api/x"). */ -} - -/* Extract URL path from full URL: "https://host/path/" → "/path/" */ -static const char *url_path(const char *url) { - if (!url) { - return NULL; - } - const char *scheme_end = strstr(url, "://"); - if (!scheme_end) { - return url; /* Already a path */ - } - const char *path = strchr(scheme_end + RN_SCHEME_SKIP, '/'); - return path ? path : "/"; -} - -/* Check if a trailing dash-segment is a hash/project-number (strippable). - * Returns true if the segment is short alphanumeric but not a meaningful word. */ -static bool is_hash_segment(const char *seg, size_t slen) { - if (slen > RN_MAX_SCHEME || slen == 0) { - return false; - } - int has_letter = 0; - for (size_t si = 0; si < slen; si++) { - if (!((seg[si] >= '0' && seg[si] <= '9') || (seg[si] >= 'a' && seg[si] <= 'z'))) { - return false; - } - if (seg[si] >= 'a' && seg[si] <= 'z') { - has_letter = SKIP_ONE; - } - } - /* Don't strip meaningful words (>=3 chars with letters like "api", "endpoint") */ - return !(has_letter && slen >= RN_MIN_SCHEME); -} - -/* Strip up to 2 trailing hash/project-number segments from a Cloud Run hostname. */ -static void strip_cloud_run_suffixes(char *hostname) { - for (int strip = 0; strip < RN_STRIP_PASSES; strip++) { - char *last_dash = strrchr(hostname, '-'); - if (!last_dash) { - break; - } - if (!is_hash_segment(last_dash + SKIP_ONE, strlen(last_dash + SKIP_ONE))) { - break; - } - *last_dash = '\0'; - } -} - -/* Extract service name from Cloud Run URL hostname. - * "my-svc-ab12cd34ef-uc.a.run.app/path" → "my-svc" */ -static const char *extract_service_name(const char *url, char *buf, int bufsz) { - if (!url) { - return NULL; - } - const char *scheme_end = strstr(url, "://"); - if (!scheme_end) { - return NULL; - } - const char *host_start = scheme_end + RN_SCHEME_SKIP; - const char *end = host_start; - while (*end && *end != '.' && *end != '/') { - end++; - } - int hlen = (int)(end - host_start); - if (hlen <= 0 || hlen >= bufsz) { - return NULL; - } - - char tmp[CBM_SZ_256]; - if (hlen >= (int)sizeof(tmp)) { - return NULL; - } - memcpy(tmp, host_start, (size_t)hlen); - tmp[hlen] = '\0'; - - strip_cloud_run_suffixes(tmp); - - snprintf(buf, (size_t)bufsz, "%s", tmp); - return buf; -} - -/* Phase 2: Match infra Route URLs to handler Route nodes by URL path + service name. */ -/* Check if a Route QN indicates an infra/async/broker Route (not a handler). */ -static bool is_broker_route(const char *qn) { - static const char *const prefixes[] = { - "__route__infra__", "__route__pubsub__", "__route__cloud_tasks__", - "__route__async__", "__route__cloud_scheduler__", "__route__kafka__", - "__route__sqs__"}; - if (!qn) { - return false; - } - for (int i = 0; i < (int)(sizeof(prefixes) / sizeof(prefixes[0])); i++) { - if (strstr(qn, prefixes[i]) == qn) { - return true; - } - } - return false; -} - -/* Length of a URL path without its query/fragment and trailing '/' ("/" stays 1). */ -static size_t route_path_len(const char *path) { - size_t len = strcspn(path, "?#"); - while (len > SKIP_ONE && path[len - SKIP_ONE] == '/') { - len--; - } - return len; -} - -/* True when `prefix` is `path` or a whole-segment prefix of it: - * /api/v1 covers /api/v1/items but not /api/v10. */ -static bool route_path_covers(const char *prefix, size_t prefix_len, const char *path, - size_t path_len) { - return prefix_len <= path_len && strncmp(prefix, path, prefix_len) == 0 && - (prefix_len == path_len || path[prefix_len] == '/'); -} - -/* Try to match a single infra Route to a handler Route and create HANDLES bridge. - * Picks the best match, not the first: an exact path beats a segment-prefix - * match, then the longer handler path, then the smaller QN. Route list order - * follows parallel extraction and changes between runs, so first-match was not - * reproducible. Returns 1 if matched, 0 otherwise. */ -static int match_one_infra_route(cbm_gbuf_t *gb, const cbm_gbuf_node_t *infra, - const char *infra_path, const char *svc_name, - const cbm_gbuf_node_t **all_routes, int route_count) { - size_t infra_len = route_path_len(infra_path); - bool infra_is_root = infra_len <= SKIP_ONE; - const cbm_gbuf_node_t *best = NULL; - bool best_exact = false; - size_t best_len = 0; - for (int j = 0; j < route_count; j++) { - const cbm_gbuf_node_t *handler_route = all_routes[j]; - if (is_broker_route(handler_route->qualified_name) || !handler_route->qualified_name) { - continue; - } - int file_matches = (handler_route->file_path != NULL && - strstr(handler_route->file_path, svc_name) != NULL); - int is_prefix_route = - strncmp(handler_route->qualified_name, "__route__ANY__", SLEN("__route__ANY__")) == 0; - if (!file_matches && !is_prefix_route) { - continue; - } - const char *handler_name = handler_route->name; - const char *handler_path = handler_name ? strchr(handler_name, '/') : NULL; - if (!handler_path) { - continue; - } - size_t handler_len = route_path_len(handler_path); - bool handler_is_root = handler_len <= SKIP_ONE; - - /* A root handler stands only for its own service's root URL; a URL - * without a path names no endpoint beyond that root. */ - bool exact; - if (handler_is_root || infra_is_root) { - if (!(handler_is_root && infra_is_root && file_matches)) { - continue; - } - exact = true; - } else { - exact = handler_len == infra_len && - strncmp(handler_path, infra_path, handler_len) == 0; - if (!exact && - !route_path_covers(handler_path, handler_len, infra_path, infra_len) && - !route_path_covers(infra_path, infra_len, handler_path, handler_len)) { - continue; - } - } - bool better = !best || (exact != best_exact ? exact - : handler_len != best_len - ? handler_len > best_len - : strcmp(handler_route->qualified_name, - best->qualified_name) < 0); - if (better) { - best = handler_route; - best_exact = exact; - best_len = handler_len; - } - } - if (!best) { - return 0; - } - const cbm_gbuf_edge_t **fn_handles = NULL; - int fn_hcount = 0; - cbm_gbuf_find_edges_by_target_type(gb, best->id, "HANDLES", &fn_handles, &fn_hcount); - for (int fh = 0; fh < fn_hcount; fh++) { - cbm_gbuf_insert_edge(gb, fn_handles[fh]->source_id, infra->id, "HANDLES", - "{\"source\":\"infra_match\"}"); - } - return SKIP_ONE; -} - -/* Phase 2: Match infra Route URLs to handler Route nodes by URL path + service name. */ -static void match_infra_routes(cbm_gbuf_t *gb) { - const cbm_gbuf_node_t **all_routes = NULL; - int route_count = 0; - if (cbm_gbuf_find_by_label(gb, "Route", &all_routes, &route_count) != 0 || route_count == 0) { - return; - } - - int matched = 0; - - for (int i = 0; i < route_count; i++) { - const cbm_gbuf_node_t *infra = all_routes[i]; - if (!infra->qualified_name || - strncmp(infra->qualified_name, "__route__infra__", SLEN("__route__infra__")) != 0) { - continue; - } - if (!infra->name || !strstr(infra->name, "://")) { - continue; - } - - const char *infra_path = url_path(infra->name); - char svc_buf[CBM_SZ_128]; - const char *svc_name = extract_service_name(infra->name, svc_buf, sizeof(svc_buf)); - if (!infra_path || !svc_name) { - continue; - } - - matched += match_one_infra_route(gb, infra, infra_path, svc_name, all_routes, route_count); - } - - if (matched > 0) { - char buf[CBM_SZ_16]; - snprintf(buf, sizeof(buf), "%d", matched); - cbm_log_info("pass.route_match", "infra_matched", buf); - } -} - -/* Phase 2a: Ensure all functions with route_path properties have Route+HANDLES edges. - * During incremental indexing, only changed files get Route nodes from extraction. - * This pass scans ALL Function/Method nodes and creates missing Route+HANDLES. */ -/* Extract a JSON string property value into buf. Returns true if found. */ -static bool extract_json_prop(const char *json, const char *key, char *buf, int bufsz) { - if (!json) { - return false; - } - char pattern[CBM_SZ_64]; - snprintf(pattern, sizeof(pattern), "\"%s\":\"", key); - const char *p = strstr(json, pattern); - if (!p) { - return false; - } - p += strlen(pattern); - const char *end = strchr(p, '"'); - if (!end || end <= p) { - return false; - } - int len = (int)(end - p); - if (len >= bufsz) { - return false; - } - memcpy(buf, p, (size_t)len); - buf[len] = '\0'; - return true; -} - -/* Process a single Function/Method node: create Route+HANDLES if it has route_path. - * Returns 1 if a new HANDLES edge was created, 0 otherwise. */ -static int ensure_one_decorator_route(cbm_gbuf_t *gb, const cbm_gbuf_node_t *func) { - if (!func->properties_json) { - return 0; - } - - char path[CBM_SZ_256]; - if (!extract_json_prop(func->properties_json, "route_path", path, sizeof(path))) { - return 0; - } - if (path[0] != '/') { - return 0; - } - - char method[CBM_SZ_16] = "ANY"; - extract_json_prop(func->properties_json, "route_method", method, sizeof(method)); - - char route_qn[CBM_ROUTE_QN_SIZE]; - char cpath[CBM_SZ_256]; - snprintf(route_qn, sizeof(route_qn), "__route__%s__%s", method, - cbm_route_canon_path(path, cpath, sizeof(cpath))); - const cbm_gbuf_node_t *existing = cbm_gbuf_find_by_qn(gb, route_qn); - - char rprops[CBM_SZ_256]; - snprintf(rprops, sizeof(rprops), "{\"method\":\"%s\",\"source\":\"decorator\"}", method); - int64_t route_id = cbm_gbuf_upsert_node(gb, "Route", path, route_qn, - func->file_path ? func->file_path : "", 0, 0, rprops); - - if (existing) { - const cbm_gbuf_edge_t **existing_handles = NULL; - int eh_count = 0; - cbm_gbuf_find_edges_by_target_type(gb, route_id, "HANDLES", &existing_handles, &eh_count); - for (int eh = 0; eh < eh_count; eh++) { - if (existing_handles[eh]->source_id == func->id) { - return 0; - } - } - } - - char hprops[CBM_SZ_512]; - snprintf(hprops, sizeof(hprops), "{\"handler\":\"%s\"}", - func->qualified_name ? func->qualified_name : ""); - cbm_gbuf_insert_edge(gb, func->id, route_id, "HANDLES", hprops); - return SKIP_ONE; -} - -/* Phase 2a: Ensure all functions with route_path properties have Route+HANDLES edges. */ -static void ensure_decorator_routes(cbm_gbuf_t *gb) { - const char *labels[] = {"Function", "Method"}; - int created = 0; - - for (int li = 0; li < RN_STRIP_PASSES; li++) { - const cbm_gbuf_node_t **nodes = NULL; - int count = 0; - if (cbm_gbuf_find_by_label(gb, labels[li], &nodes, &count) != 0) { - continue; - } - for (int i = 0; i < count; i++) { - created += ensure_one_decorator_route(gb, nodes[i]); - } - } - - if (created > 0) { - char buf[CBM_SZ_16]; - snprintf(buf, sizeof(buf), "%d", created); - cbm_log_info("pass.ensure_decorator_routes", "created", buf); - } -} - -/* Phase 2b: Connect prefix Routes to decorator handler Functions. - * For each prefix Route (__route__ANY__/path), find the CALLS edge leading to it - * (from the registering file), derive the service directory, then find decorator - * Routes in that directory tree and create HANDLES from their handler Functions - * to the prefix Route. This bridges include_router → decorator → handler. */ -/* Bridge decorator handler Functions to a prefix Route. Returns number connected. */ -static int bridge_funcs_to_prefix(cbm_gbuf_t *gb, const cbm_gbuf_node_t *prefix_route, - const char *registrar_path, int dir_len, - const char *prefix_segs) { - const cbm_gbuf_node_t **funcs = NULL; - int func_count = 0; - cbm_gbuf_find_by_label(gb, "Function", &funcs, &func_count); - - int connected = 0; - for (int fi = 0; fi < func_count; fi++) { - const cbm_gbuf_node_t *func = funcs[fi]; - if (!func->file_path) { - continue; - } - if (strncmp(func->file_path, registrar_path, (size_t)dir_len) != 0) { - continue; - } - if (!func->properties_json || !strstr(func->properties_json, "\"route_path\"")) { - continue; - } - if (prefix_segs && prefix_segs[0] && !strstr(func->file_path, prefix_segs)) { - continue; - } - cbm_gbuf_insert_edge(gb, func->id, prefix_route->id, "HANDLES", - "{\"source\":\"prefix_decorator_bridge\"}"); - connected++; - } - return connected; -} - -/* Phase 2b: Connect prefix Routes to decorator handler Functions. */ -static void connect_prefix_to_decorators(cbm_gbuf_t *gb) { - const cbm_gbuf_node_t **routes = NULL; - int route_count = 0; - if (cbm_gbuf_find_by_label(gb, "Route", &routes, &route_count) != 0) { - return; - } - - int connected = 0; - - for (int ri = 0; ri < route_count; ri++) { - const cbm_gbuf_node_t *prefix_route = routes[ri]; - if (!prefix_route->qualified_name || - strncmp(prefix_route->qualified_name, "__route__ANY__/", SLEN("__route__ANY__/")) != - 0) { - continue; - } - - /* The root prefix "/" has no segment to scope handlers by, so bridging - * it would claim every route handler under the registrar's directory. */ - const char *prefix_path = prefix_route->name; - const char *prefix_segs = - (prefix_path && prefix_path[0] == '/') ? prefix_path + SKIP_ONE : prefix_path; - if (!prefix_segs || !prefix_segs[0]) { - continue; - } - - const cbm_gbuf_edge_t **calls_in = NULL; - int calls_count = 0; - cbm_gbuf_find_edges_by_target_type(gb, prefix_route->id, "CALLS", &calls_in, &calls_count); - - /* Every registering file bridges its own directory. Taking only the - * first CALLS edge depended on parallel merge order. Repeated bridges - * dedup in the gbuf. */ - for (int ci = 0; ci < calls_count; ci++) { - const cbm_gbuf_node_t *registrar = cbm_gbuf_find_by_id(gb, calls_in[ci]->source_id); - if (!registrar || !registrar->file_path) { - continue; - } - const char *last_slash = strrchr(registrar->file_path, '/'); - if (!last_slash) { - continue; - } - int dir_len = (int)(last_slash - registrar->file_path) + SKIP_ONE; - connected += bridge_funcs_to_prefix(gb, prefix_route, registrar->file_path, dir_len, - prefix_segs); - } - } - - if (connected > 0) { - char buf[CBM_SZ_16]; - snprintf(buf, sizeof(buf), "%d", connected); - cbm_log_info("pass.prefix_bridge", "connected", buf); - } -} - -/* Phase 3: Create DATA_FLOWS edges by linking callers through Route to handlers. - * For each HTTP_CALLS/ASYNC_CALLS edge (caller → Route), find the HANDLES edge - * (handler → Route) and create DATA_FLOWS (caller → handler) with route context. */ -/* Check if a direct CALLS edge already exists between two nodes */ -static int has_direct_call(const cbm_gbuf_t *gb, int64_t source, int64_t target) { - const cbm_gbuf_edge_t **edges = NULL; - int count = 0; - cbm_gbuf_find_edges_by_source_type(gb, source, "CALLS", &edges, &count); - for (int i = 0; i < count; i++) { - if (edges[i]->target_id == target) { - return SKIP_ONE; - } - } - return 0; -} - -/* Extract param_names from a node's properties_json. - * Returns a comma-separated string in buf, or empty string. */ -static void extract_param_names(const cbm_gbuf_node_t *node, char *buf, int bufsize) { - buf[0] = '\0'; - if (!node || !node->properties_json) { - return; - } - const char *p = strstr(node->properties_json, "\"param_names\":"); - if (!p) { - return; - } - p = strchr(p, '['); - if (!p) { - return; - } - p++; /* skip '[' */ - const char *end = strchr(p, ']'); - if (!end || end <= p) { - return; - } - int len = (int)(end - p); - if (len >= bufsize) { - len = bufsize - SKIP_ONE; - } - memcpy(buf, p, (size_t)len); - buf[len] = '\0'; -} - -/* Extract the "args" JSON fragment from an edge's properties. - * Returns pointer into the properties string (not copied). */ -static const char *find_args_in_props(const char *props) { - if (!props) { - return NULL; - } - const char *p = strstr(props, "\"args\":["); - if (!p) { - return NULL; - } - return p + RN_ARGS_SKIP; /* skip "args":[ , points to first { or ] */ -} - -/* Append handler_params and caller_args to DATA_FLOWS props. Closes with '}'. */ -static void finish_data_flow_props(char *props, size_t propsz, size_t pos, - const char *handler_params, const char *args_json) { - if (handler_params[0]) { - int w = snprintf(props + pos, propsz - pos, ",\"handler_params\":[%s]", handler_params); - if (w > 0) { - pos += (size_t)w; - } - } - if (args_json) { - int w = snprintf(props + pos, propsz - pos, ",\"caller_args\":[%.*s", 400, args_json); - if (w > 0) { - pos += (size_t)w; - char *close = strchr(props + (pos - (size_t)w) + RN_CALLEE_SKIP, ']'); - if (close && close < props + propsz - PAIR_LEN) { - pos = (size_t)(close - props) + SKIP_ONE; - } - } - } - if (pos < propsz - SKIP_ONE) { - props[pos] = '}'; - props[pos + SKIP_ONE] = '\0'; - } -} - -typedef struct { - int64_t source_id; - const char *props; - const char *edge_type; -} caller_edge_ref_t; - -static bool http_call_edge_has_valid_route(const cbm_gbuf_edge_t *edge) { - char url_buf[CBM_SZ_512]; - const char *url = json_extract(edge->properties_json, "url_path", url_buf, sizeof(url_buf)); - char callee_buf[CBM_SZ_256]; - const char *callee = - json_extract(edge->properties_json, "callee", callee_buf, sizeof(callee_buf)); - return cbm_service_pattern_is_http_route_literal(url, callee); -} - -/* Try to create a DATA_FLOWS edge between caller and handler via a route. - * Returns: 1=created, 0=skipped (self/duplicate), -1=skipped (has direct call). */ -static int try_create_data_flow(cbm_gbuf_t *gb, int64_t caller_id, int64_t handler_id, - const cbm_gbuf_node_t *route, const char *edge_type, - const char *caller_props, bool via_infra) { - if (caller_id == handler_id) { - return 0; - } - if (has_direct_call(gb, caller_id, handler_id)) { - return CBM_NOT_FOUND; - } - - const char *args_json = find_args_in_props(caller_props); - const cbm_gbuf_node_t *handler_node = cbm_gbuf_find_by_id(gb, handler_id); - char handler_params[CBM_SZ_512]; - extract_param_names(handler_node, handler_params, sizeof(handler_params)); - - /* Route names/QNs are sliced source text (URL strings, decorator args) and - * can contain quotes — escape them or the edge properties JSON is - * malformed (aborts json_extract consumers incl. integrity_check). */ - char esc_rname[CBM_SZ_256]; - char esc_rqn[CBM_SZ_512]; - cbm_json_escape(esc_rname, sizeof(esc_rname), route->name ? route->name : ""); - cbm_json_escape(esc_rqn, sizeof(esc_rqn), route->qualified_name ? route->qualified_name : ""); - char props[CBM_SZ_2K]; - int n; - if (via_infra) { - n = snprintf(props, sizeof(props), - "{\"via\":\"%s\",\"route\":\"%s\",\"edge_type\":\"%s\",\"via_infra\":true", - esc_rname, esc_rqn, edge_type); - } else { - n = snprintf(props, sizeof(props), "{\"via\":\"%s\",\"route\":\"%s\",\"edge_type\":\"%s\"", - esc_rname, esc_rqn, edge_type); - } - - if (n > 0 && (size_t)n < sizeof(props) - RN_PROPS_MARGIN) { - finish_data_flow_props(props, sizeof(props), (size_t)n, handler_params, args_json); - } - /* handler_params/args_json are sliced text fragments; truncation or stray - * quotes can leave the assembled JSON malformed, which aborts every - * json_extract consumer downstream. Validate and fall back to the minimal - * envelope rather than persisting invalid properties. */ - { - yyjson_doc *vdoc = yyjson_read(props, strlen(props), 0); - if (!vdoc) { - snprintf(props, sizeof(props), "{\"via\":\"%s\",\"edge_type\":\"%s\"}", esc_rname, - edge_type); - } else { - yyjson_doc_free(vdoc); - } - } - cbm_gbuf_insert_edge(gb, caller_id, handler_id, "DATA_FLOWS", props); - return SKIP_ONE; -} - -/* Collect extra handler IDs reachable via INFRA_MAPS chain from a route. */ -static int collect_infra_handlers(cbm_gbuf_t *gb, int64_t route_id, int64_t *out, int max_out) { - const cbm_gbuf_edge_t **infra_edges = NULL; - int infra_count = 0; - cbm_gbuf_find_edges_by_source_type(gb, route_id, "INFRA_MAPS", &infra_edges, &infra_count); - - int n = 0; - for (int ie = 0; ie < infra_count; ie++) { - const cbm_gbuf_edge_t **ep_handles = NULL; - int ep_hcount = 0; - cbm_gbuf_find_edges_by_target_type(gb, infra_edges[ie]->target_id, "HANDLES", &ep_handles, - &ep_hcount); - for (int eh = 0; eh < ep_hcount && n < max_out; eh++) { - out[n++] = ep_handles[eh]->source_id; - } - } - return n; -} - -/* Collect caller edges (HTTP_CALLS + ASYNC_CALLS) targeting a route. */ -static int collect_caller_edges(cbm_gbuf_t *gb, int64_t route_id, caller_edge_ref_t *out, - int max_out) { - int n = 0; - const cbm_gbuf_edge_t **http_edges = NULL; - int http_count = 0; - cbm_gbuf_find_edges_by_target_type(gb, route_id, "HTTP_CALLS", &http_edges, &http_count); - for (int i = 0; i < http_count && n < max_out; i++) { - if (!http_call_edge_has_valid_route(http_edges[i])) { - continue; - } - out[n].source_id = http_edges[i]->source_id; - out[n].props = http_edges[i]->properties_json; - out[n].edge_type = "HTTP_CALLS"; - n++; - } - const cbm_gbuf_edge_t **async_edges = NULL; - int async_count = 0; - cbm_gbuf_find_edges_by_target_type(gb, route_id, "ASYNC_CALLS", &async_edges, &async_count); - for (int i = 0; i < async_count && n < max_out; i++) { - out[n].source_id = async_edges[i]->source_id; - out[n].props = async_edges[i]->properties_json; - out[n].edge_type = "ASYNC_CALLS"; - n++; - } - return n; -} - -/* Create DATA_FLOWS between callers and handlers for one route node. */ -static void create_route_data_flows(cbm_gbuf_t *gb, const cbm_gbuf_node_t *route, - const caller_edge_ref_t *callers, int n_callers, int *flows, - int *skipped) { - const cbm_gbuf_edge_t **handles_edges = NULL; - int handles_count = 0; - cbm_gbuf_find_edges_by_target_type(gb, route->id, "HANDLES", &handles_edges, &handles_count); - - int64_t extra_handlers[CBM_SZ_32]; - int n_extra = collect_infra_handlers(gb, route->id, extra_handlers, CBM_SZ_32); - - for (int ci = 0; ci < n_callers; ci++) { - for (int hi = 0; hi < handles_count; hi++) { - int rc = try_create_data_flow(gb, callers[ci].source_id, handles_edges[hi]->source_id, - route, callers[ci].edge_type, callers[ci].props, false); - if (rc > 0) { - (*flows)++; - } else if (rc < 0) { - (*skipped)++; - } - } - for (int xi = 0; xi < n_extra; xi++) { - int rc = try_create_data_flow(gb, callers[ci].source_id, extra_handlers[xi], route, - callers[ci].edge_type, callers[ci].props, true); - if (rc > 0) { - (*flows)++; - } else if (rc < 0) { - (*skipped)++; - } - } - } -} - -/* Find the enclosing proto service for a function by file + line range. */ -static const cbm_gbuf_node_t *find_enclosing_service(const cbm_gbuf_node_t *fn, - const cbm_gbuf_node_t **services, - int svc_count) { - for (int s = 0; s < svc_count; s++) { - if (strcmp(fn->file_path, services[s]->file_path) != 0) { - continue; - } - if (fn->start_line >= services[s]->start_line && fn->end_line <= services[s]->end_line) { - return services[s]; - } - } - return NULL; -} - -/* Phase 4: Create __grpc__ Route nodes from protobuf service definitions. */ -static void create_grpc_routes(cbm_gbuf_t *gb) { - const cbm_gbuf_node_t **classes = NULL; - int class_count = 0; - if (cbm_gbuf_find_by_label(gb, "Class", &classes, &class_count) != 0 || class_count == 0) { - return; - } - - /* Every .proto class, uncapped: a fixed cap kept whichever classes the - * parallel workers happened to emit first, so routes varied by run. */ - const cbm_gbuf_node_t **services = malloc((size_t)class_count * sizeof(*services)); - if (!services) { - return; - } - int svc_count = 0; - for (int i = 0; i < class_count; i++) { - if (classes[i]->file_path && strstr(classes[i]->file_path, ".proto")) { - services[svc_count++] = classes[i]; - } - } - const cbm_gbuf_node_t **funcs = NULL; - int func_count = 0; - if (svc_count == 0 || - cbm_gbuf_find_by_label(gb, "Function", &funcs, &func_count) != 0 || func_count == 0) { - free(services); - return; - } - - int grpc_routes = 0; - for (int f = 0; f < func_count; f++) { - const cbm_gbuf_node_t *fn = funcs[f]; - if (!fn->file_path || !strstr(fn->file_path, ".proto") || !fn->name) { - continue; - } - const cbm_gbuf_node_t *svc = find_enclosing_service(fn, services, svc_count); - if (!svc) { - continue; - } - char route_qn[CBM_ROUTE_QN_SIZE]; - snprintf(route_qn, sizeof(route_qn), "__grpc__%s/%s", svc->name, fn->name); - - char props[CBM_SZ_128]; - snprintf(props, sizeof(props), "{\"source\":\"proto\",\"service\":\"%s\"}", svc->name); - - int64_t route_id = cbm_gbuf_upsert_node(gb, "Route", fn->name, route_qn, fn->file_path, - fn->start_line, fn->end_line, props); - cbm_gbuf_insert_edge(gb, fn->id, route_id, "HANDLES", "{\"via\":\"proto_rpc\"}"); - grpc_routes++; - } - free(services); - if (grpc_routes > 0) { - char buf[CBM_SZ_16]; - snprintf(buf, sizeof(buf), "%d", grpc_routes); - cbm_log_info("pass.route_nodes.grpc", "routes", buf); - } -} - -/* Phase 3: Create DATA_FLOWS edges by linking callers through Route to handlers. */ -static void create_data_flows(cbm_gbuf_t *gb) { - const cbm_gbuf_node_t **routes = NULL; - int route_count = 0; - if (cbm_gbuf_find_by_label(gb, "Route", &routes, &route_count) != 0 || route_count == 0) { - return; - } - - int flows = 0; - int skipped = 0; - - for (int ri = 0; ri < route_count; ri++) { - caller_edge_ref_t caller_edges[CBM_SZ_64]; - int n_callers = collect_caller_edges(gb, routes[ri]->id, caller_edges, CBM_SZ_64); - create_route_data_flows(gb, routes[ri], caller_edges, n_callers, &flows, &skipped); - } - - if (flows > 0 || skipped > 0) { - char buf1[CBM_SZ_16]; - char buf2[CBM_SZ_16]; - snprintf(buf1, sizeof(buf1), "%d", flows); - snprintf(buf2, sizeof(buf2), "%d", skipped); - cbm_log_info("pass.data_flows", "created", buf1, "skipped_has_call", buf2); - } -} - -/* ── SvelteKit filesystem-based route extractor ────────────────────── - * - * SvelteKit derives REST endpoints and SSR loaders from the filesystem - * layout under `src/routes/`. No `app.get(...)` style call exists for - * the call resolver to pick up, so we walk File nodes and synthesise Route - * nodes + HANDLES edges directly here. - * - * Recognised filenames (TypeScript and JavaScript variants): - * - * +server.{ts,js} → REST endpoints, exports GET/POST/... - * +page.server.{ts,js} → SSR page server module: load + actions - * +layout.server.{ts,js} → SSR layout server module: load - * - * Route path is derived from the directory chain under `routes/`: - * - * apps/x/src/routes/foo/+server.ts → /foo - * apps/x/src/routes/api/items/+server.ts → /api/items - * apps/x/src/routes/(grp)/foo/+server.ts → /foo (group stripped) - * apps/x/src/routes/[slug]/+server.ts → /:slug (param rewrite) - * apps/x/src/routes/+server.ts → / (root) - */ - -enum { - SKR_PATH_BUF = 1024, - SKR_NAME_BUF = 64, -}; - -/* Detect the SvelteKit kind of a file path. Returns 1 for +server, - * 2 for +page.server, 3 for +layout.server; 0 if not a SvelteKit - * server-side route file. */ -static int sveltekit_file_kind(const char *file_path) { - if (!file_path) { - return 0; - } - /* The basename must match one of the three patterns. We also require - * a "/routes/" segment somewhere in the path so we don't snag files - * in unrelated directories that happen to be named "+server.ts". */ - if (!strstr(file_path, "/routes/")) { - return 0; - } - const char *slash = strrchr(file_path, '/'); - const char *base = slash ? slash + 1 : file_path; - if (strcmp(base, "+server.ts") == 0 || strcmp(base, "+server.js") == 0) { - return 1; - } - if (strcmp(base, "+page.server.ts") == 0 || strcmp(base, "+page.server.js") == 0) { - return 2; - } - if (strcmp(base, "+layout.server.ts") == 0 || strcmp(base, "+layout.server.js") == 0) { - return 3; - } - return 0; -} - -/* Compute the URL route path for a SvelteKit file path. Writes into - * `out` (caller-provided buffer) and returns the buffer or NULL on - * error. The leading "/" is always present; "/" is used for the root - * route. Group segments wrapped in parentheses are stripped. Dynamic - * params `[slug]` become `:slug`, and rest params `[...slug]` become - * `*slug` so the result is recognisable as a path pattern. */ -static const char *sveltekit_route_path(const char *file_path, char *out, int outsz) { - if (!file_path || !out || outsz <= 1) { - return NULL; - } - const char *routes_seg = strstr(file_path, "/routes/"); - if (!routes_seg) { - return NULL; - } - /* Walk segment-by-segment between "/routes/" and the trailing "+...". */ - const char *p = routes_seg + strlen("/routes/"); - const char *last_slash = strrchr(file_path, '/'); - if (!last_slash || last_slash < p) { - /* File is directly under /routes/ — root route. */ - out[0] = '/'; - out[1] = '\0'; - return out; - } - /* p points at first char after /routes/, last_slash points at the - * slash before the +...filename. Iterate segments between them. */ - int pos = 0; - out[pos] = '\0'; - while (p < last_slash) { - const char *seg_end = strchr(p, '/'); - if (!seg_end || seg_end > last_slash) { - seg_end = last_slash; - } - size_t seg_len = (size_t)(seg_end - p); - if (seg_len > 0) { - /* Group `(name)` — skip entirely. */ - if (p[0] == '(' && p[seg_len - 1] == ')') { - p = seg_end + 1; - continue; - } - /* Emit "/" then rewritten segment. */ - if (pos + 1 < outsz - 1) { - out[pos++] = '/'; - } - /* Dynamic param: `[slug]` → `:slug`, `[...slug]` → `*slug`. */ - if (seg_len >= 2 && p[0] == '[' && p[seg_len - 1] == ']') { - const char *inner = p + 1; - size_t inner_len = seg_len - 2; - if (inner_len >= 3 && strncmp(inner, "...", 3) == 0) { - if (pos < outsz - 1) { - out[pos++] = '*'; - } - inner += 3; - inner_len -= 3; - } else { - if (pos < outsz - 1) { - out[pos++] = ':'; - } - } - /* Strip an optional `=matcher` suffix after the param name. */ - size_t copy_len = inner_len; - for (size_t i = 0; i < inner_len; i++) { - if (inner[i] == '=') { - copy_len = i; - break; - } - } - if ((int)copy_len > outsz - 1 - pos) { - copy_len = (size_t)(outsz - 1 - pos); - } - memcpy(out + pos, inner, copy_len); - pos += (int)copy_len; - } else { - size_t copy_len = seg_len; - if ((int)copy_len > outsz - 1 - pos) { - copy_len = (size_t)(outsz - 1 - pos); - } - memcpy(out + pos, p, copy_len); - pos += (int)copy_len; - } - out[pos] = '\0'; - } - p = seg_end + 1; - } - if (pos == 0) { - out[pos++] = '/'; - out[pos] = '\0'; - } - return out; -} - -/* Returns the HTTP method string a function name maps to for a +server - * file, or NULL if the function isn't a SvelteKit REST handler. The set - * mirrors SvelteKit's documented HTTP verb exports. */ -static const char *sveltekit_server_method(const char *name) { - if (!name) { - return NULL; - } - static const char *const verbs[] = { - "GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD", - }; - for (size_t i = 0; i < sizeof(verbs) / sizeof(verbs[0]); i++) { - if (strcmp(name, verbs[i]) == 0) { - return verbs[i]; - } - } - /* `fallback` catches any verb not explicitly exported. */ - if (strcmp(name, "fallback") == 0) { - return "ANY"; - } - return NULL; -} - -typedef struct { - cbm_gbuf_t *gb; - int routes_created; - int handles_created; - int files_seen; -} sveltekit_ctx_t; - -/* Process one File node: if it matches a SvelteKit server-side file, - * synthesise a Route node and HANDLES edges from any handler functions - * (or actions Variable) defined in the file. */ -static void sveltekit_file_visitor(const cbm_gbuf_node_t *node, void *userdata) { - sveltekit_ctx_t *ctx = (sveltekit_ctx_t *)userdata; - if (!node || !node->label || strcmp(node->label, "File") != 0) { - return; - } - int kind = sveltekit_file_kind(node->file_path); - if (kind == 0) { - return; - } - ctx->files_seen++; - - char route_path[SKR_PATH_BUF]; - if (!sveltekit_route_path(node->file_path, route_path, sizeof(route_path))) { - return; - } - - /* Find every DEFINES edge from this file to a Function or Variable. */ - const cbm_gbuf_edge_t **edges = NULL; - int edge_count = 0; - if (cbm_gbuf_find_edges_by_source_type(ctx->gb, node->id, "DEFINES", &edges, &edge_count) != - 0 || - edge_count == 0) { - return; - } - - for (int i = 0; i < edge_count; i++) { - const cbm_gbuf_node_t *child = cbm_gbuf_find_by_id(ctx->gb, edges[i]->target_id); - if (!child || !child->name || !child->label) { - continue; - } - bool is_fn = strcmp(child->label, "Function") == 0; - bool is_var = strcmp(child->label, "Variable") == 0; - if (!is_fn && !is_var) { - continue; - } - - const char *method = NULL; - bool is_actions = false; - if (kind == 1) { - method = sveltekit_server_method(child->name); - } else if (kind == 2) { - if (is_fn && strcmp(child->name, "load") == 0) { - method = "LOAD"; - } else if (strcmp(child->name, "actions") == 0) { - /* SvelteKit form actions: emit a single Route at this - * path with method=ACTIONS; finer-grained action names - * aren't exposed as top-level identifiers. */ - method = "ACTIONS"; - is_actions = is_var; - } - } else if (kind == 3) { - if (is_fn && strcmp(child->name, "load") == 0) { - method = "LOAD"; - } - } - if (!method) { - continue; - } - - char route_qn[CBM_ROUTE_QN_SIZE]; - char cpath[CBM_SZ_256]; - snprintf(route_qn, sizeof(route_qn), "__route__%s__%s", method, - cbm_route_canon_path(route_path, cpath, sizeof(cpath))); - char route_props[CBM_SZ_256]; - snprintf(route_props, sizeof(route_props), - "{\"method\":\"%s\",\"framework\":\"sveltekit\"}", method); - int64_t route_id = - cbm_gbuf_upsert_node(ctx->gb, "Route", route_path, route_qn, "", 0, 0, route_props); - if (route_id == 0) { - continue; - } - ctx->routes_created++; - - char hprops[CBM_SZ_256]; - snprintf(hprops, sizeof(hprops), "{\"handler\":\"%s\",\"framework\":\"sveltekit\"%s}", - child->qualified_name ? child->qualified_name : child->name, - is_actions ? ",\"via\":\"actions_object\"" : ""); - cbm_gbuf_insert_edge(ctx->gb, child->id, route_id, "HANDLES", hprops); - ctx->handles_created++; - } -} - -/* Public entry point: scan all File nodes for SvelteKit server modules - * and synthesise Route + HANDLES nodes from the filesystem convention. */ -static void create_sveltekit_routes(cbm_gbuf_t *gb) { - if (!gb) { - return; - } - sveltekit_ctx_t ctx = {.gb = gb, .routes_created = 0, .handles_created = 0, .files_seen = 0}; - cbm_gbuf_foreach_node(gb, sveltekit_file_visitor, &ctx); - if (ctx.files_seen > 0) { - char b1[CBM_SZ_16]; - char b2[CBM_SZ_16]; - char b3[CBM_SZ_16]; - snprintf(b1, sizeof(b1), "%d", ctx.files_seen); - snprintf(b2, sizeof(b2), "%d", ctx.routes_created); - snprintf(b3, sizeof(b3), "%d", ctx.handles_created); - cbm_log_info("pass.sveltekit_routes", "files", b1, "routes", b2, "handles", b3); - } -} - -void cbm_pipeline_create_route_nodes(cbm_gbuf_t *gb) { - if (!gb) { - return; - } - - route_ctx_t ctx = {.gb = gb, .created = 0}; - cbm_gbuf_foreach_edge(gb, route_edge_visitor, &ctx); - - if (ctx.created > 0) { - char buf[CBM_SZ_16]; - snprintf(buf, sizeof(buf), "%d", ctx.created); - cbm_log_info("pass.route_nodes", "created", buf); - } - - /* Phase 2a: ensure all functions with route_path have Route+HANDLES. - * Handles incremental mode where unchanged files don't re-extract. */ - ensure_decorator_routes(gb); - - /* Phase 2b: connect prefix Routes to decorator handler Functions. - * Must run BEFORE match_infra_routes so infra matching can find - * HANDLES edges on prefix Routes for the bridge. */ - connect_prefix_to_decorators(gb); - - /* Phase 2b: match infra Routes to handler Routes by URL path */ - match_infra_routes(gb); - - /* Phase 3: create DATA_FLOWS edges through Routes */ - create_data_flows(gb); - - /* Phase 4: create gRPC Route nodes from protobuf service definitions. - * Scans Class nodes from .proto files, follows DEFINES_METHOD edges - * to find rpc methods, creates __grpc__ServiceName/MethodName Route nodes. */ - create_grpc_routes(gb); - - /* Phase 5: filesystem-based SvelteKit routes (+server / +page.server / - * +layout.server) — no call-site equivalent for the call resolver to - * pick up, so we walk File nodes directly here. */ - create_sveltekit_routes(gb); -} diff --git a/src/pipeline/pass_route_nodes.zig b/src/pipeline/pass_route_nodes.zig new file mode 100644 index 00000000..d1c19d5d --- /dev/null +++ b/src/pipeline/pass_route_nodes.zig @@ -0,0 +1,1152 @@ +//! pass_route_nodes.zig — Create Route nodes for HTTP_CALLS/ASYNC_CALLS edges. +//! +//! After parallel resolve merges edges into the main gbuf, HTTP_CALLS and +//! ASYNC_CALLS edges have url_path/method/broker in properties but point to +//! the library function (e.g., requests.get). This pass: +//! +//! 1. Scans all HTTP_CALLS/ASYNC_CALLS edges +//! 2. Extracts url_path from edge properties +//! 3. Creates Route nodes with deterministic QNs (__route__METHOD__/path) +//! 4. Re-targets edges from library function → Route node +//! +//! Route nodes are the rendezvous point for cross-service communication: +//! Service A: checkout() → HTTP_CALLS → Route("POST /api/orders") +//! Service B: create_order() → HANDLES → Route("POST /api/orders") + +const std = @import("std"); +const c = @import("c"); +const interop = @import("../foundation/interop.zig"); +const util = @import("util.zig"); + +const CStr = util.CStr; +const MutStr = util.MutStr; +const ix = util.ix; +const streq = util.streq; +const Gbuf = c.cbm_gbuf_t; +const Node = c.cbm_gbuf_node_t; +const Edge = c.cbm_gbuf_edge_t; +const NodeList = [*c][*c]const Node; +const EdgeList = [*c][*c]const Edge; + +const RN_MAX_SCHEME = 12; +const RN_MIN_SCHEME = 3; +const RN_STRIP_PASSES = 2; +const RN_SCHEME_SKIP = 3; +const RN_ARGS_SKIP = 7; +const RN_PROPS_MARGIN = 100; +const RN_CALLEE_SKIP = 14; // strlen('"callee_args":[') - 1 +const ROUTE_QN_SIZE = c.CBM_ROUTE_QN_SIZE; + +fn hasPrefix(s: CStr, comptime prefix: [:0]const u8) bool { + return c.strncmp(s, prefix, prefix.len) == 0; +} + +/// Logs one integer counter under `key`. +fn logCount(msg: [*:0]const u8, key: [*:0]const u8, value: c_int) void { + var buf: [c.CBM_SZ_16]u8 = undefined; + _ = c.snprintf(&buf, buf.len, "%d", value); + interop.logInfo(msg, .{ @as(CStr, key), @as(CStr, &buf) }); +} + +/// True for characters that may appear in a ":name" route parameter. +inline fn isRouteIdentChar(ch: u8) bool { + return (ch >= 'a' and ch <= 'z') or (ch >= 'A' and ch <= 'Z') or (ch >= '0' and ch <= '9') or + ch == '_'; +} + +/// Canonicalize route-path parameter placeholders to a single "{}" token so that +/// client call sites and server handlers rendezvous on the same Route QN +/// regardless of framework syntax. Each parameter token collapses to "{}": +/// +/// :name Express / React-Router / Rails / typical JS API clients +/// {name} Axum / Spring / OpenAPI / ASP.NET +/// Flask / Rocket (incl. typed "") +/// ${...} JS template interpolation captured into the path +/// +/// Parameter names are intentionally discarded so the same logical endpoint +/// matches across services that name the path variable differently. Static path +/// text is copied verbatim; the result never exceeds the input length. +pub export fn cbm_route_canon_path(in: CStr, out: MutStr, out_sz: usize) CStr { + if (out == null or out_sz == 0) return out; + if (in == null) { + out[0] = 0; + return out; + } + const last = out_sz - 1; + var oi: usize = 0; + var i: usize = 0; + while (in[i] != 0 and oi < last) { + const ch = in[i]; + const at_seg_start = oi == 0 or out[oi - 1] == '/'; + var is_param = false; + + if (ch == ':' and at_seg_start and isRouteIdentChar(in[i + 1])) { + i += 1; + while (in[i] != 0 and isRouteIdentChar(in[i])) i += 1; + is_param = true; + } else if (ch == '{') { + i += 1; + while (in[i] != 0 and in[i] != '}' and in[i] != '/') i += 1; + if (in[i] == '}') i += 1; + is_param = true; + } else if (ch == '<') { + i += 1; + while (in[i] != 0 and in[i] != '>' and in[i] != '/') i += 1; + if (in[i] == '>') i += 1; + is_param = true; + } else if (ch == '$' and in[i + 1] == '{') { + i += 2; + while (in[i] != 0 and in[i] != '}' and in[i] != '/') i += 1; + if (in[i] == '}') i += 1; + is_param = true; + } + + if (is_param) { + if (oi + 2 > last) break; + out[oi] = '{'; + out[oi + 1] = '}'; + oi += 2; + continue; + } + out[oi] = ch; + oi += 1; + i += 1; + } + out[oi] = 0; + return out; +} + +/// Extract a JSON string value by key from properties. +/// Returns pointer into buf (caller provides buffer). NULL if not found. +fn jsonExtract(json: CStr, key: CStr, buf: []u8) CStr { + if (json == null or key == null) return null; + // Build "key":" pattern + var pattern: [c.CBM_SZ_128]u8 = undefined; + _ = c.snprintf(&pattern, pattern.len, "\"%s\":\"", key); + var start: CStr = c.strstr(json, &pattern); + if (start == null) return null; + start += c.strlen(&pattern); + const end: CStr = c.strchr(start, '"'); + if (end == null or end == start) return null; + var len = util.ptrDiff(end, start); + if (len >= buf.len) len = buf.len - 1; + @memcpy(buf[0..len], start[0..len]); + buf[len] = 0; + return buf.ptr; +} + +/// Visitor context for edge scanning +const RouteCtx = struct { + gb: *Gbuf, + created: c_int, +}; + +fn routeEdgeVisitor(edge: [*c]const Edge, userdata: ?*anyopaque) callconv(.c) void { + const ctx: *RouteCtx = @ptrCast(@alignCast(userdata)); + + // Only process HTTP_CALLS and ASYNC_CALLS + const is_http = streq(edge.*.type, "HTTP_CALLS"); + if (!is_http and !streq(edge.*.type, "ASYNC_CALLS")) return; + + // Extract url_path from properties + var url_buf: [c.CBM_SZ_512]u8 = undefined; + const url = jsonExtract(edge.*.properties_json, "url_path", &url_buf); + if (url == null or url[0] == 0) return; + var callee_buf: [c.CBM_SZ_256]u8 = undefined; + const callee = jsonExtract(edge.*.properties_json, "callee", &callee_buf); + if (is_http and !c.cbm_service_pattern_is_http_route_literal(url, callee)) return; + + // Extract method or broker + var method_buf: [c.CBM_SZ_16]u8 = undefined; + var broker_buf: [c.CBM_SZ_64]u8 = undefined; + const method = jsonExtract(edge.*.properties_json, "method", &method_buf); + const broker = jsonExtract(edge.*.properties_json, "broker", &broker_buf); + + // Build Route QN + var route_qn: [ROUTE_QN_SIZE]u8 = undefined; + if (is_http) { + var cpath: [c.CBM_SZ_256]u8 = undefined; + _ = c.snprintf( + &route_qn, + route_qn.len, + "__route__%s__%s", + if (method != null) method else @as(CStr, "ANY"), + cbm_route_canon_path(url, &cpath, cpath.len), + ); + } else { + _ = c.snprintf( + &route_qn, + route_qn.len, + "__route__%s__%s", + if (broker != null) broker else @as(CStr, "async"), + url, + ); + } + + // Build properties for Route node + var route_props: [c.CBM_SZ_256]u8 = undefined; + if (method != null) { + _ = c.snprintf(&route_props, route_props.len, "{\"method\":\"%s\"}", method); + } else if (broker != null) { + _ = c.snprintf(&route_props, route_props.len, "{\"broker\":\"%s\"}", broker); + } else { + _ = c.snprintf(&route_props, route_props.len, "{}"); + } + + // Create or find Route node (deduped by QN) + _ = c.cbm_gbuf_upsert_node(ctx.gb, "Route", url, &route_qn, "", 0, 0, &route_props); + ctx.created += 1; + + // Note: we do NOT re-target the edge here because modifying edges during + // iteration is unsafe. The edge stays pointing to the library function. + // The URL-in-args detection in pass_parallel will create Route→handler HANDLES + // edge separately. The caller→library edge + // with url_path in properties is sufficient for query_graph to find + // the Route via: caller → HTTP_CALLS(url_path="/api/x") + Route("/api/x"). +} + +/// Extract URL path from full URL: "https://host/path/" → "/path/" +fn urlPath(url: CStr) CStr { + if (url == null) return null; + const scheme_end: CStr = c.strstr(url, "://"); + if (scheme_end == null) return url; // Already a path + const path: CStr = c.strchr(scheme_end + RN_SCHEME_SKIP, '/'); + return if (path != null) path else "/"; +} + +/// Check if a trailing dash-segment is a hash/project-number (strippable). +/// Returns true if the segment is short alphanumeric but not a meaningful word. +fn isHashSegment(seg: []const u8) bool { + if (seg.len > RN_MAX_SCHEME or seg.len == 0) return false; + var has_letter = false; + for (seg) |ch| { + if (!((ch >= '0' and ch <= '9') or (ch >= 'a' and ch <= 'z'))) return false; + if (ch >= 'a' and ch <= 'z') has_letter = true; + } + // Don't strip meaningful words (>=3 chars with letters like "api", "endpoint") + return !(has_letter and seg.len >= RN_MIN_SCHEME); +} + +/// Strip up to 2 trailing hash/project-number segments from a Cloud Run hostname. +fn stripCloudRunSuffixes(hostname: [*:0]u8) void { + for (0..RN_STRIP_PASSES) |_| { + const last_dash: MutStr = c.strrchr(hostname, '-'); + if (last_dash == null) break; + const seg: [*]const u8 = last_dash + 1; + if (!isHashSegment(seg[0..c.strlen(seg)])) break; + last_dash[0] = 0; + } +} + +/// Extract service name from Cloud Run URL hostname. +/// "my-svc-ab12cd34ef-uc.a.run.app/path" → "my-svc" +fn extractServiceName(url: CStr, buf: []u8) CStr { + if (url == null) return null; + const scheme_end: CStr = c.strstr(url, "://"); + if (scheme_end == null) return null; + const host_start: [*]const u8 = scheme_end + RN_SCHEME_SKIP; + var end = host_start; + while (end[0] != 0 and end[0] != '.' and end[0] != '/') end += 1; + const hlen = util.ptrDiff(end, host_start); + if (hlen == 0 or hlen >= buf.len) return null; + + var tmp: [c.CBM_SZ_256:0]u8 = undefined; + if (hlen >= c.CBM_SZ_256) return null; + @memcpy(tmp[0..hlen], host_start[0..hlen]); + tmp[hlen] = 0; + + stripCloudRunSuffixes(&tmp); + + _ = c.snprintf(buf.ptr, buf.len, "%s", @as(CStr, &tmp)); + return buf.ptr; +} + +/// Check if a Route QN indicates an infra/async/broker Route (not a handler). +fn isBrokerRoute(qn: CStr) bool { + const prefixes = [_][*:0]const u8{ + "__route__infra__", "__route__pubsub__", "__route__cloud_tasks__", + "__route__async__", "__route__cloud_scheduler__", "__route__kafka__", + "__route__sqs__", + }; + if (qn == null) return false; + for (prefixes) |prefix| { + if (c.strstr(qn, prefix) == qn) return true; + } + return false; +} + +/// Length of a URL path without its query/fragment and trailing '/' ("/" stays 1). +fn routePathLen(path: CStr) usize { + var len = c.strcspn(path, "?#"); + while (len > 1 and path[len - 1] == '/') len -= 1; + return len; +} + +/// True when `prefix` is `path` or a whole-segment prefix of it: +/// /api/v1 covers /api/v1/items but not /api/v10. +fn routePathCovers(prefix: CStr, prefix_len: usize, path: CStr, path_len: usize) bool { + return prefix_len <= path_len and c.strncmp(prefix, path, prefix_len) == 0 and + (prefix_len == path_len or path[prefix_len] == '/'); +} + +/// Try to match a single infra Route to a handler Route and create HANDLES bridge. +/// Picks the best match, not the first: an exact path beats a segment-prefix +/// match, then the longer handler path, then the smaller QN. Route list order +/// follows parallel extraction and changes between runs, so first-match was not +/// reproducible. Returns 1 if matched, 0 otherwise. +fn matchOneInfraRoute( + gb: *Gbuf, + infra: *const Node, + infra_path: CStr, + svc_name: CStr, + all_routes: NodeList, + route_count: c_int, +) c_int { + const infra_len = routePathLen(infra_path); + const infra_is_root = infra_len <= 1; + var best: ?*const Node = null; + var best_exact = false; + var best_len: usize = 0; + for (0..ix(route_count)) |j| { + const handler_route: *const Node = all_routes[j]; + if (isBrokerRoute(handler_route.qualified_name) or handler_route.qualified_name == null) { + continue; + } + const file_matches = handler_route.file_path != null and + c.strstr(handler_route.file_path, svc_name) != null; + const is_prefix_route = hasPrefix(handler_route.qualified_name, "__route__ANY__"); + if (!file_matches and !is_prefix_route) continue; + const handler_name: CStr = handler_route.name; + const handler_path: CStr = if (handler_name != null) c.strchr(handler_name, '/') else null; + if (handler_path == null) continue; + const handler_len = routePathLen(handler_path); + const handler_is_root = handler_len <= 1; + + // A root handler stands only for its own service's root URL; a URL + // without a path names no endpoint beyond that root. + var exact: bool = undefined; + if (handler_is_root or infra_is_root) { + if (!(handler_is_root and infra_is_root and file_matches)) continue; + exact = true; + } else { + exact = handler_len == infra_len and + c.strncmp(handler_path, infra_path, handler_len) == 0; + if (!exact and + !routePathCovers(handler_path, handler_len, infra_path, infra_len) and + !routePathCovers(infra_path, infra_len, handler_path, handler_len)) + { + continue; + } + } + const better = if (best) |b| + (if (exact != best_exact) + exact + else if (handler_len != best_len) + handler_len > best_len + else + c.strcmp(handler_route.qualified_name, b.qualified_name) < 0) + else + true; + if (better) { + best = handler_route; + best_exact = exact; + best_len = handler_len; + } + } + const winner = best orelse return 0; + var fn_handles: EdgeList = null; + var fn_hcount: c_int = 0; + _ = c.cbm_gbuf_find_edges_by_target_type(gb, winner.id, "HANDLES", &fn_handles, &fn_hcount); + for (0..ix(fn_hcount)) |fh| { + _ = c.cbm_gbuf_insert_edge(gb, fn_handles[fh].*.source_id, infra.id, "HANDLES", "{\"source\":\"infra_match\"}"); + } + return 1; +} + +/// Phase 2: Match infra Route URLs to handler Route nodes by URL path + service name. +fn matchInfraRoutes(gb: *Gbuf) void { + var all_routes: NodeList = null; + var route_count: c_int = 0; + if (c.cbm_gbuf_find_by_label(gb, "Route", &all_routes, &route_count) != 0 or route_count == 0) { + return; + } + + var matched: c_int = 0; + + for (0..ix(route_count)) |i| { + const infra: *const Node = all_routes[i]; + if (infra.qualified_name == null or !hasPrefix(infra.qualified_name, "__route__infra__")) { + continue; + } + if (infra.name == null or c.strstr(infra.name, "://") == null) continue; + + const infra_path = urlPath(infra.name); + var svc_buf: [c.CBM_SZ_128]u8 = undefined; + const svc_name = extractServiceName(infra.name, &svc_buf); + if (infra_path == null or svc_name == null) continue; + + matched += matchOneInfraRoute(gb, infra, infra_path, svc_name, all_routes, route_count); + } + + if (matched > 0) logCount("pass.route_match", "infra_matched", matched); +} + +// Phase 2a: Ensure all functions with route_path properties have Route+HANDLES edges. +// During incremental indexing, only changed files get Route nodes from extraction. +// This pass scans ALL Function/Method nodes and creates missing Route+HANDLES. + +/// Extract a JSON string property value into buf. Returns true if found. +fn extractJsonProp(json: CStr, key: CStr, buf: []u8) bool { + if (json == null) return false; + var pattern: [c.CBM_SZ_64]u8 = undefined; + _ = c.snprintf(&pattern, pattern.len, "\"%s\":\"", key); + var p: CStr = c.strstr(json, &pattern); + if (p == null) return false; + p += c.strlen(&pattern); + const end: CStr = c.strchr(p, '"'); + if (end == null or @intFromPtr(end) <= @intFromPtr(p)) return false; + const len = util.ptrDiff(end, p); + if (len >= buf.len) return false; + @memcpy(buf[0..len], p[0..len]); + buf[len] = 0; + return true; +} + +/// Process a single Function/Method node: create Route+HANDLES if it has route_path. +/// Returns 1 if a new HANDLES edge was created, 0 otherwise. +fn ensureOneDecoratorRoute(gb: *Gbuf, func: *const Node) c_int { + if (func.properties_json == null) return 0; + + var path: [c.CBM_SZ_256]u8 = undefined; + if (!extractJsonProp(func.properties_json, "route_path", &path)) return 0; + if (path[0] != '/') return 0; + + var method: [c.CBM_SZ_16]u8 = @splat(0); + method[0..3].* = "ANY".*; + _ = extractJsonProp(func.properties_json, "route_method", &method); + + var route_qn: [ROUTE_QN_SIZE]u8 = undefined; + var cpath: [c.CBM_SZ_256]u8 = undefined; + _ = c.snprintf( + &route_qn, + route_qn.len, + "__route__%s__%s", + @as(CStr, &method), + cbm_route_canon_path(&path, &cpath, cpath.len), + ); + const existing: ?*const Node = c.cbm_gbuf_find_by_qn(gb, &route_qn); + + var rprops: [c.CBM_SZ_256]u8 = undefined; + _ = c.snprintf(&rprops, rprops.len, "{\"method\":\"%s\",\"source\":\"decorator\"}", @as(CStr, &method)); + const route_id = c.cbm_gbuf_upsert_node(gb, "Route", &path, &route_qn, if (func.file_path != null) func.file_path else "", 0, 0, &rprops); + + if (existing != null) { + var existing_handles: EdgeList = null; + var eh_count: c_int = 0; + _ = c.cbm_gbuf_find_edges_by_target_type(gb, route_id, "HANDLES", &existing_handles, &eh_count); + for (0..ix(eh_count)) |eh| { + if (existing_handles[eh].*.source_id == func.id) return 0; + } + } + + var hprops: [c.CBM_SZ_512]u8 = undefined; + _ = c.snprintf( + &hprops, + hprops.len, + "{\"handler\":\"%s\"}", + if (func.qualified_name != null) func.qualified_name else @as(CStr, ""), + ); + _ = c.cbm_gbuf_insert_edge(gb, func.id, route_id, "HANDLES", &hprops); + return 1; +} + +/// Phase 2a: Ensure all functions with route_path properties have Route+HANDLES edges. +fn ensureDecoratorRoutes(gb: *Gbuf) void { + var created: c_int = 0; + + for ([_][*:0]const u8{ "Function", "Method" }) |label| { + var nodes: NodeList = null; + var count: c_int = 0; + if (c.cbm_gbuf_find_by_label(gb, label, &nodes, &count) != 0) continue; + for (0..ix(count)) |i| { + created += ensureOneDecoratorRoute(gb, nodes[i]); + } + } + + if (created > 0) logCount("pass.ensure_decorator_routes", "created", created); +} + +// Phase 2b: Connect prefix Routes to decorator handler Functions. +// For each prefix Route (__route__ANY__/path), find the CALLS edge leading to it +// (from the registering file), derive the service directory, then find decorator +// Routes in that directory tree and create HANDLES from their handler Functions +// to the prefix Route. This bridges include_router → decorator → handler. + +/// Bridge decorator handler Functions to a prefix Route. Returns number connected. +fn bridgeFuncsToPrefix( + gb: *Gbuf, + prefix_route: *const Node, + registrar_path: CStr, + dir_len: usize, + prefix_segs: CStr, +) c_int { + var funcs: NodeList = null; + var func_count: c_int = 0; + _ = c.cbm_gbuf_find_by_label(gb, "Function", &funcs, &func_count); + + var connected: c_int = 0; + for (0..ix(func_count)) |fi| { + const func: *const Node = funcs[fi]; + if (func.file_path == null) continue; + if (c.strncmp(func.file_path, registrar_path, dir_len) != 0) continue; + if (func.properties_json == null or c.strstr(func.properties_json, "\"route_path\"") == null) { + continue; + } + if (prefix_segs != null and prefix_segs[0] != 0 and c.strstr(func.file_path, prefix_segs) == null) { + continue; + } + _ = c.cbm_gbuf_insert_edge(gb, func.id, prefix_route.id, "HANDLES", "{\"source\":\"prefix_decorator_bridge\"}"); + connected += 1; + } + return connected; +} + +/// Phase 2b: Connect prefix Routes to decorator handler Functions. +fn connectPrefixToDecorators(gb: *Gbuf) void { + var routes: NodeList = null; + var route_count: c_int = 0; + if (c.cbm_gbuf_find_by_label(gb, "Route", &routes, &route_count) != 0) return; + + var connected: c_int = 0; + + for (0..ix(route_count)) |ri| { + const prefix_route: *const Node = routes[ri]; + if (prefix_route.qualified_name == null or + !hasPrefix(prefix_route.qualified_name, "__route__ANY__/")) + { + continue; + } + + // The root prefix "/" has no segment to scope handlers by, so bridging + // it would claim every route handler under the registrar's directory. + const prefix_path: CStr = prefix_route.name; + const prefix_segs: CStr = if (prefix_path != null and prefix_path[0] == '/') prefix_path + 1 else prefix_path; + if (prefix_segs == null or prefix_segs[0] == 0) continue; + + var calls_in: EdgeList = null; + var calls_count: c_int = 0; + _ = c.cbm_gbuf_find_edges_by_target_type(gb, prefix_route.id, "CALLS", &calls_in, &calls_count); + + // Every registering file bridges its own directory. Taking only the + // first CALLS edge depended on parallel merge order. Repeated bridges + // dedup in the gbuf. + for (0..ix(calls_count)) |ci| { + const registrar: *const Node = @as(?*const Node, c.cbm_gbuf_find_by_id(gb, calls_in[ci].*.source_id)) orelse continue; + if (registrar.file_path == null) continue; + const last_slash: CStr = c.strrchr(registrar.file_path, '/'); + if (last_slash == null) continue; + const dir_len = util.ptrDiff(last_slash, registrar.file_path) + 1; + connected += bridgeFuncsToPrefix(gb, prefix_route, registrar.file_path, dir_len, prefix_segs); + } + } + + if (connected > 0) logCount("pass.prefix_bridge", "connected", connected); +} + +// Phase 3: Create DATA_FLOWS edges by linking callers through Route to handlers. +// For each HTTP_CALLS/ASYNC_CALLS edge (caller → Route), find the HANDLES edge +// (handler → Route) and create DATA_FLOWS (caller → handler) with route context. + +/// Check if a direct CALLS edge already exists between two nodes +fn hasDirectCall(gb: *const Gbuf, source: i64, target: i64) bool { + var edges: EdgeList = null; + var count: c_int = 0; + _ = c.cbm_gbuf_find_edges_by_source_type(gb, source, "CALLS", &edges, &count); + for (0..ix(count)) |i| { + if (edges[i].*.target_id == target) return true; + } + return false; +} + +/// Extract param_names from a node's properties_json. +/// Writes a comma-separated string into buf, or the empty string. +fn extractParamNames(node: ?*const Node, buf: []u8) void { + buf[0] = 0; + const n = node orelse return; + if (n.properties_json == null) return; + var p: CStr = c.strstr(n.properties_json, "\"param_names\":"); + if (p == null) return; + p = c.strchr(p, '['); + if (p == null) return; + p += 1; // skip '[' + const end: CStr = c.strchr(p, ']'); + if (end == null or @intFromPtr(end) <= @intFromPtr(p)) return; + var len = util.ptrDiff(end, p); + if (len >= buf.len) len = buf.len - 1; + @memcpy(buf[0..len], p[0..len]); + buf[len] = 0; +} + +/// Extract the "args" JSON fragment from an edge's properties. +/// Returns pointer into the properties string (not copied). +fn findArgsInProps(props: CStr) CStr { + if (props == null) return null; + const p: CStr = c.strstr(props, "\"args\":["); + if (p == null) return null; + return p + RN_ARGS_SKIP; // skip "args":[ , points to first { or ] +} + +/// Append handler_params and caller_args to DATA_FLOWS props. Closes with '}'. +fn finishDataFlowProps(props: [*]u8, propsz: usize, pos_in: usize, handler_params: [*:0]const u8, args_json: CStr) void { + var pos = pos_in; + if (handler_params[0] != 0) { + const w = c.snprintf(props + pos, propsz -% pos, ",\"handler_params\":[%s]", handler_params); + if (w > 0) pos += @intCast(w); + } + if (args_json != null) { + const w = c.snprintf(props + pos, propsz -% pos, ",\"caller_args\":[%.*s", @as(c_int, 400), args_json); + if (w > 0) { + pos += @intCast(w); + const close: CStr = c.strchr(props + (pos - @as(usize, @intCast(w))) + RN_CALLEE_SKIP, ']'); + if (close != null and @intFromPtr(close) < @intFromPtr(props + propsz - 2)) { + pos = util.ptrDiff(close, props) + 1; + } + } + } + if (pos < propsz - 1) { + props[pos] = '}'; + props[pos + 1] = 0; + } +} + +const CallerEdgeRef = struct { + source_id: i64, + props: CStr, + edge_type: CStr, +}; + +fn httpCallEdgeHasValidRoute(edge: *const Edge) bool { + var url_buf: [c.CBM_SZ_512]u8 = undefined; + const url = jsonExtract(edge.properties_json, "url_path", &url_buf); + var callee_buf: [c.CBM_SZ_256]u8 = undefined; + const callee = jsonExtract(edge.properties_json, "callee", &callee_buf); + return c.cbm_service_pattern_is_http_route_literal(url, callee); +} + +/// Try to create a DATA_FLOWS edge between caller and handler via a route. +/// Returns: 1=created, 0=skipped (self/duplicate), -1=skipped (has direct call). +fn tryCreateDataFlow( + gb: *Gbuf, + caller_id: i64, + handler_id: i64, + route: *const Node, + edge_type: CStr, + caller_props: CStr, + via_infra: bool, +) c_int { + @setEvalBranchQuota(10000); + if (caller_id == handler_id) return 0; + if (hasDirectCall(gb, caller_id, handler_id)) return c.CBM_NOT_FOUND; + + const args_json = findArgsInProps(caller_props); + const handler_node: ?*const Node = c.cbm_gbuf_find_by_id(gb, handler_id); + var handler_params: [c.CBM_SZ_512:0]u8 = undefined; + extractParamNames(handler_node, &handler_params); + + // Route names/QNs are sliced source text (URL strings, decorator args) and + // can contain quotes — escape them or the edge properties JSON is + // malformed (aborts json_extract consumers incl. integrity_check). + var esc_rname: [c.CBM_SZ_256]u8 = undefined; + var esc_rqn: [c.CBM_SZ_512]u8 = undefined; + _ = c.cbm_json_escape(&esc_rname, esc_rname.len, if (route.name != null) route.name else ""); + _ = c.cbm_json_escape(&esc_rqn, esc_rqn.len, if (route.qualified_name != null) route.qualified_name else ""); + var props: [c.CBM_SZ_2K]u8 = undefined; + const n = if (via_infra) + c.snprintf( + &props, + props.len, + "{\"via\":\"%s\",\"route\":\"%s\",\"edge_type\":\"%s\",\"via_infra\":true", + @as(CStr, &esc_rname), + @as(CStr, &esc_rqn), + edge_type, + ) + else + c.snprintf( + &props, + props.len, + "{\"via\":\"%s\",\"route\":\"%s\",\"edge_type\":\"%s\"", + @as(CStr, &esc_rname), + @as(CStr, &esc_rqn), + edge_type, + ); + + if (n > 0 and @as(usize, @intCast(n)) < props.len - RN_PROPS_MARGIN) { + finishDataFlowProps(&props, props.len, @intCast(n), &handler_params, args_json); + } + // handler_params/args_json are sliced text fragments; truncation or stray + // quotes can leave the assembled JSON malformed, which aborts every + // json_extract consumer downstream. Validate and fall back to the minimal + // envelope rather than persisting invalid properties. + { + const vdoc = c.yyjson_read(&props, c.strlen(&props), 0); + if (vdoc == null) { + _ = c.snprintf(&props, props.len, "{\"via\":\"%s\",\"edge_type\":\"%s\"}", @as(CStr, &esc_rname), edge_type); + } else { + c.yyjson_doc_free(vdoc); + } + } + _ = c.cbm_gbuf_insert_edge(gb, caller_id, handler_id, "DATA_FLOWS", &props); + return 1; +} + +/// Collect extra handler IDs reachable via INFRA_MAPS chain from a route. +fn collectInfraHandlers(gb: *Gbuf, route_id: i64, out: []i64) usize { + var infra_edges: EdgeList = null; + var infra_count: c_int = 0; + _ = c.cbm_gbuf_find_edges_by_source_type(gb, route_id, "INFRA_MAPS", &infra_edges, &infra_count); + + var n: usize = 0; + for (0..ix(infra_count)) |ie| { + var ep_handles: EdgeList = null; + var ep_hcount: c_int = 0; + _ = c.cbm_gbuf_find_edges_by_target_type(gb, infra_edges[ie].*.target_id, "HANDLES", &ep_handles, &ep_hcount); + var eh: usize = 0; + while (eh < ep_hcount and n < out.len) : (eh += 1) { + out[n] = ep_handles[eh].*.source_id; + n += 1; + } + } + return n; +} + +/// Collect caller edges (HTTP_CALLS + ASYNC_CALLS) targeting a route. +fn collectCallerEdges(gb: *Gbuf, route_id: i64, out: []CallerEdgeRef) usize { + var n: usize = 0; + var http_edges: EdgeList = null; + var http_count: c_int = 0; + _ = c.cbm_gbuf_find_edges_by_target_type(gb, route_id, "HTTP_CALLS", &http_edges, &http_count); + var i: usize = 0; + while (i < http_count and n < out.len) : (i += 1) { + if (!httpCallEdgeHasValidRoute(http_edges[i])) continue; + out[n] = .{ + .source_id = http_edges[i].*.source_id, + .props = http_edges[i].*.properties_json, + .edge_type = "HTTP_CALLS", + }; + n += 1; + } + var async_edges: EdgeList = null; + var async_count: c_int = 0; + _ = c.cbm_gbuf_find_edges_by_target_type(gb, route_id, "ASYNC_CALLS", &async_edges, &async_count); + i = 0; + while (i < async_count and n < out.len) : (i += 1) { + out[n] = .{ + .source_id = async_edges[i].*.source_id, + .props = async_edges[i].*.properties_json, + .edge_type = "ASYNC_CALLS", + }; + n += 1; + } + return n; +} + +/// Create DATA_FLOWS between callers and handlers for one route node. +fn createRouteDataFlows(gb: *Gbuf, route: *const Node, callers: []const CallerEdgeRef, flows: *c_int, skipped: *c_int) void { + var handles_edges: EdgeList = null; + var handles_count: c_int = 0; + _ = c.cbm_gbuf_find_edges_by_target_type(gb, route.id, "HANDLES", &handles_edges, &handles_count); + + var extra_handlers: [c.CBM_SZ_32]i64 = undefined; + const n_extra = collectInfraHandlers(gb, route.id, &extra_handlers); + + for (callers) |caller| { + for (0..ix(handles_count)) |hi| { + const rc = tryCreateDataFlow(gb, caller.source_id, handles_edges[hi].*.source_id, route, caller.edge_type, caller.props, false); + if (rc > 0) { + flows.* += 1; + } else if (rc < 0) { + skipped.* += 1; + } + } + for (extra_handlers[0..n_extra]) |handler_id| { + const rc = tryCreateDataFlow(gb, caller.source_id, handler_id, route, caller.edge_type, caller.props, true); + if (rc > 0) { + flows.* += 1; + } else if (rc < 0) { + skipped.* += 1; + } + } + } +} + +/// Find the enclosing proto service for a function by file + line range. +fn findEnclosingService(func: *const Node, services: []const [*c]const Node) ?*const Node { + for (services) |service_c| { + const service: *const Node = service_c; + if (!streq(func.file_path, service.file_path)) continue; + if (func.start_line >= service.start_line and func.end_line <= service.end_line) { + return service; + } + } + return null; +} + +/// Phase 4: Create __grpc__ Route nodes from protobuf service definitions. +fn createGrpcRoutes(gb: *Gbuf) void { + var classes: NodeList = null; + var class_count: c_int = 0; + if (c.cbm_gbuf_find_by_label(gb, "Class", &classes, &class_count) != 0 or class_count == 0) { + return; + } + + // Every .proto class, uncapped: a fixed cap kept whichever classes the + // parallel workers happened to emit first, so routes varied by run. + const services = interop.mallocN([*c]const Node, ix(class_count)); + if (services == null) return; + var svc_count: usize = 0; + for (0..ix(class_count)) |i| { + if (classes[i].*.file_path != null and c.strstr(classes[i].*.file_path, ".proto") != null) { + services[svc_count] = classes[i]; + svc_count += 1; + } + } + var funcs: NodeList = null; + var func_count: c_int = 0; + if (svc_count == 0 or + c.cbm_gbuf_find_by_label(gb, "Function", &funcs, &func_count) != 0 or func_count == 0) + { + c.free(@ptrCast(services)); + return; + } + + var grpc_routes: c_int = 0; + for (0..ix(func_count)) |f| { + const func: *const Node = funcs[f]; + if (func.file_path == null or c.strstr(func.file_path, ".proto") == null or func.name == null) { + continue; + } + const svc = findEnclosingService(func, services[0..svc_count]) orelse continue; + var route_qn: [ROUTE_QN_SIZE]u8 = undefined; + _ = c.snprintf(&route_qn, route_qn.len, "__grpc__%s/%s", svc.name, func.name); + + var props: [c.CBM_SZ_128]u8 = undefined; + _ = c.snprintf(&props, props.len, "{\"source\":\"proto\",\"service\":\"%s\"}", svc.name); + + const route_id = c.cbm_gbuf_upsert_node(gb, "Route", func.name, &route_qn, func.file_path, func.start_line, func.end_line, &props); + _ = c.cbm_gbuf_insert_edge(gb, func.id, route_id, "HANDLES", "{\"via\":\"proto_rpc\"}"); + grpc_routes += 1; + } + c.free(@ptrCast(services)); + if (grpc_routes > 0) logCount("pass.route_nodes.grpc", "routes", grpc_routes); +} + +/// Phase 3: Create DATA_FLOWS edges by linking callers through Route to handlers. +fn createDataFlows(gb: *Gbuf) void { + var routes: NodeList = null; + var route_count: c_int = 0; + if (c.cbm_gbuf_find_by_label(gb, "Route", &routes, &route_count) != 0 or route_count == 0) { + return; + } + + var flows: c_int = 0; + var skipped: c_int = 0; + + for (0..ix(route_count)) |ri| { + var caller_edges: [c.CBM_SZ_64]CallerEdgeRef = undefined; + const n_callers = collectCallerEdges(gb, routes[ri].*.id, &caller_edges); + createRouteDataFlows(gb, routes[ri], caller_edges[0..n_callers], &flows, &skipped); + } + + if (flows > 0 or skipped > 0) { + var buf1: [c.CBM_SZ_16]u8 = undefined; + var buf2: [c.CBM_SZ_16]u8 = undefined; + _ = c.snprintf(&buf1, buf1.len, "%d", flows); + _ = c.snprintf(&buf2, buf2.len, "%d", skipped); + interop.logInfo("pass.data_flows", .{ + @as(CStr, "created"), @as(CStr, &buf1), @as(CStr, "skipped_has_call"), @as(CStr, &buf2), + }); + } +} + +// ── SvelteKit filesystem-based route extractor ────────────────────── +// +// SvelteKit derives REST endpoints and SSR loaders from the filesystem +// layout under `src/routes/`. No `app.get(...)` style call exists for +// the call resolver to pick up, so we walk File nodes and synthesise Route +// nodes + HANDLES edges directly here. +// +// Recognised filenames (TypeScript and JavaScript variants): +// +// +server.{ts,js} → REST endpoints, exports GET/POST/... +// +page.server.{ts,js} → SSR page server module: load + actions +// +layout.server.{ts,js} → SSR layout server module: load +// +// Route path is derived from the directory chain under `routes/`: +// +// apps/x/src/routes/foo/+server.ts → /foo +// apps/x/src/routes/api/items/+server.ts → /api/items +// apps/x/src/routes/(grp)/foo/+server.ts → /foo (group stripped) +// apps/x/src/routes/[slug]/+server.ts → /:slug (param rewrite) +// apps/x/src/routes/+server.ts → / (root) + +const SKR_PATH_BUF = 1024; + +const SveltekitKind = enum { none, server, page_server, layout_server }; + +/// Detect the SvelteKit kind of a file path: +server, +page.server, +/// +layout.server, or none when it is not a SvelteKit server-side route file. +fn sveltekitFileKind(file_path: CStr) SveltekitKind { + if (file_path == null) return .none; + // The basename must match one of the three patterns. We also require + // a "/routes/" segment somewhere in the path so we don't snag files + // in unrelated directories that happen to be named "+server.ts". + if (c.strstr(file_path, "/routes/") == null) return .none; + const slash: CStr = c.strrchr(file_path, '/'); + const base: CStr = if (slash != null) slash + 1 else file_path; + if (streq(base, "+server.ts") or streq(base, "+server.js")) return .server; + if (streq(base, "+page.server.ts") or streq(base, "+page.server.js")) return .page_server; + if (streq(base, "+layout.server.ts") or streq(base, "+layout.server.js")) return .layout_server; + return .none; +} + +/// Compute the URL route path for a SvelteKit file path. Writes into +/// `out` (caller-provided buffer) and returns the buffer or NULL on +/// error. The leading "/" is always present; "/" is used for the root +/// route. Group segments wrapped in parentheses are stripped. Dynamic +/// params `[slug]` become `:slug`, and rest params `[...slug]` become +/// `*slug` so the result is recognisable as a path pattern. +fn sveltekitRoutePath(file_path: CStr, out: []u8) CStr { + if (file_path == null or out.len <= 1) return null; + const routes_seg: CStr = c.strstr(file_path, "/routes/"); + if (routes_seg == null) return null; + // Walk segment-by-segment between "/routes/" and the trailing "+...". + var p: [*]const u8 = routes_seg + "/routes/".len; + const last_slash: CStr = c.strrchr(file_path, '/'); + if (last_slash == null or @intFromPtr(last_slash) < @intFromPtr(p)) { + // File is directly under /routes/ — root route. + out[0] = '/'; + out[1] = 0; + return out.ptr; + } + // p points at first char after /routes/, last_slash points at the + // slash before the +...filename. Iterate segments between them. + const limit = out.len - 1; + var pos: usize = 0; + out[pos] = 0; + while (@intFromPtr(p) < @intFromPtr(last_slash)) { + var seg_end: [*]const u8 = last_slash; + const next_slash: CStr = c.strchr(p, '/'); + if (next_slash != null and @intFromPtr(next_slash) <= @intFromPtr(last_slash)) { + seg_end = next_slash; + } + const seg_len = util.ptrDiff(seg_end, p); + if (seg_len > 0) { + // Group `(name)` — skip entirely. + if (p[0] == '(' and p[seg_len - 1] == ')') { + p = seg_end + 1; + continue; + } + // Emit "/" then rewritten segment. + if (pos + 1 < limit) { + out[pos] = '/'; + pos += 1; + } + // Dynamic param: `[slug]` → `:slug`, `[...slug]` → `*slug`. + if (seg_len >= 2 and p[0] == '[' and p[seg_len - 1] == ']') { + var inner: [*]const u8 = p + 1; + var inner_len = seg_len - 2; + if (inner_len >= 3 and c.strncmp(inner, "...", 3) == 0) { + if (pos < limit) { + out[pos] = '*'; + pos += 1; + } + inner += 3; + inner_len -= 3; + } else { + if (pos < limit) { + out[pos] = ':'; + pos += 1; + } + } + // Strip an optional `=matcher` suffix after the param name. + var copy_len = inner_len; + for (inner[0..inner_len], 0..) |ch, i| { + if (ch == '=') { + copy_len = i; + break; + } + } + if (copy_len > limit - pos) copy_len = limit - pos; + @memcpy(out[pos..][0..copy_len], inner[0..copy_len]); + pos += copy_len; + } else { + var copy_len = seg_len; + if (copy_len > limit - pos) copy_len = limit - pos; + @memcpy(out[pos..][0..copy_len], p[0..copy_len]); + pos += copy_len; + } + out[pos] = 0; + } + p = seg_end + 1; + } + if (pos == 0) { + out[0] = '/'; + out[1] = 0; + } + return out.ptr; +} + +/// Returns the HTTP method string a function name maps to for a +server +/// file, or NULL if the function isn't a SvelteKit REST handler. The set +/// mirrors SvelteKit's documented HTTP verb exports. +fn sveltekitServerMethod(name: CStr) CStr { + if (name == null) return null; + const verbs = [_][*:0]const u8{ "GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD" }; + for (verbs) |verb| { + if (streq(name, verb)) return verb; + } + // `fallback` catches any verb not explicitly exported. + if (streq(name, "fallback")) return "ANY"; + return null; +} + +const SveltekitCtx = struct { + gb: *Gbuf, + routes_created: c_int = 0, + handles_created: c_int = 0, + files_seen: c_int = 0, +}; + +/// Process one File node: if it matches a SvelteKit server-side file, +/// synthesise a Route node and HANDLES edges from any handler functions +/// (or actions Variable) defined in the file. +fn sveltekitFileVisitor(node_c: [*c]const Node, userdata: ?*anyopaque) callconv(.c) void { + const ctx: *SveltekitCtx = @ptrCast(@alignCast(userdata)); + const node: *const Node = @as(?*const Node, node_c) orelse return; + if (node.label == null or !streq(node.label, "File")) return; + const kind = sveltekitFileKind(node.file_path); + if (kind == .none) return; + ctx.files_seen += 1; + + var route_path: [SKR_PATH_BUF]u8 = undefined; + if (sveltekitRoutePath(node.file_path, &route_path) == null) return; + + // Find every DEFINES edge from this file to a Function or Variable. + var edges: EdgeList = null; + var edge_count: c_int = 0; + if (c.cbm_gbuf_find_edges_by_source_type(ctx.gb, node.id, "DEFINES", &edges, &edge_count) != 0 or + edge_count == 0) + { + return; + } + + for (0..ix(edge_count)) |i| { + const child: *const Node = @as(?*const Node, c.cbm_gbuf_find_by_id(ctx.gb, edges[i].*.target_id)) orelse continue; + if (child.name == null or child.label == null) continue; + const is_fn = streq(child.label, "Function"); + const is_var = streq(child.label, "Variable"); + if (!is_fn and !is_var) continue; + + var method: CStr = null; + var is_actions = false; + switch (kind) { + .server => method = sveltekitServerMethod(child.name), + .page_server => { + if (is_fn and streq(child.name, "load")) { + method = "LOAD"; + } else if (streq(child.name, "actions")) { + // SvelteKit form actions: emit a single Route at this + // path with method=ACTIONS; finer-grained action names + // aren't exposed as top-level identifiers. + method = "ACTIONS"; + is_actions = is_var; + } + }, + .layout_server => { + if (is_fn and streq(child.name, "load")) method = "LOAD"; + }, + .none => {}, + } + if (method == null) continue; + + var route_qn: [ROUTE_QN_SIZE]u8 = undefined; + var cpath: [c.CBM_SZ_256]u8 = undefined; + _ = c.snprintf( + &route_qn, + route_qn.len, + "__route__%s__%s", + method, + cbm_route_canon_path(&route_path, &cpath, cpath.len), + ); + var route_props: [c.CBM_SZ_256]u8 = undefined; + _ = c.snprintf(&route_props, route_props.len, "{\"method\":\"%s\",\"framework\":\"sveltekit\"}", method); + const route_id = c.cbm_gbuf_upsert_node(ctx.gb, "Route", &route_path, &route_qn, "", 0, 0, &route_props); + if (route_id == 0) continue; + ctx.routes_created += 1; + + var hprops: [c.CBM_SZ_256]u8 = undefined; + _ = c.snprintf( + &hprops, + hprops.len, + "{\"handler\":\"%s\",\"framework\":\"sveltekit\"%s}", + if (child.qualified_name != null) child.qualified_name else child.name, + @as(CStr, if (is_actions) ",\"via\":\"actions_object\"" else ""), + ); + _ = c.cbm_gbuf_insert_edge(ctx.gb, child.id, route_id, "HANDLES", &hprops); + ctx.handles_created += 1; + } +} + +/// Scan all File nodes for SvelteKit server modules and synthesise +/// Route + HANDLES nodes from the filesystem convention. +fn createSveltekitRoutes(gb: *Gbuf) void { + var ctx: SveltekitCtx = .{ .gb = gb }; + c.cbm_gbuf_foreach_node(gb, sveltekitFileVisitor, &ctx); + if (ctx.files_seen > 0) { + var b1: [c.CBM_SZ_16]u8 = undefined; + var b2: [c.CBM_SZ_16]u8 = undefined; + var b3: [c.CBM_SZ_16]u8 = undefined; + _ = c.snprintf(&b1, b1.len, "%d", ctx.files_seen); + _ = c.snprintf(&b2, b2.len, "%d", ctx.routes_created); + _ = c.snprintf(&b3, b3.len, "%d", ctx.handles_created); + interop.logInfo("pass.sveltekit_routes", .{ + @as(CStr, "files"), @as(CStr, &b1), + @as(CStr, "routes"), @as(CStr, &b2), + @as(CStr, "handles"), @as(CStr, &b3), + }); + } +} + +pub export fn cbm_pipeline_create_route_nodes(gb_c: ?*Gbuf) void { + const gb = gb_c orelse return; + + var ctx: RouteCtx = .{ .gb = gb, .created = 0 }; + c.cbm_gbuf_foreach_edge(gb, routeEdgeVisitor, &ctx); + + if (ctx.created > 0) logCount("pass.route_nodes", "created", ctx.created); + + // Phase 2a: ensure all functions with route_path have Route+HANDLES. + // Handles incremental mode where unchanged files don't re-extract. + ensureDecoratorRoutes(gb); + + // Phase 2b: connect prefix Routes to decorator handler Functions. + // Must run BEFORE matchInfraRoutes so infra matching can find + // HANDLES edges on prefix Routes for the bridge. + connectPrefixToDecorators(gb); + + // Phase 2b: match infra Routes to handler Routes by URL path + matchInfraRoutes(gb); + + // Phase 3: create DATA_FLOWS edges through Routes + createDataFlows(gb); + + // Phase 4: create gRPC Route nodes from protobuf service definitions. + // Scans Class nodes from .proto files, follows DEFINES_METHOD edges + // to find rpc methods, creates __grpc__ServiceName/MethodName Route nodes. + createGrpcRoutes(gb); + + // Phase 5: filesystem-based SvelteKit routes (+server / +page.server / + // +layout.server) — no call-site equivalent for the call resolver to + // pick up, so we walk File nodes directly here. + createSveltekitRoutes(gb); +} diff --git a/src/pipeline/pass_semantic.c b/src/pipeline/pass_semantic.c deleted file mode 100644 index 4b73fc9d..00000000 --- a/src/pipeline/pass_semantic.c +++ /dev/null @@ -1,156 +0,0 @@ -/* - * pass_semantic.c — Go implicit interface satisfaction. - * - * A Go type implements an interface by having its method set; nothing in the - * source names the relation. After all definitions are in the graph buffer, - * this pass matches each Go Struct/Class against each Go Interface by method - * name and emits IMPLEMENTS (type → interface) plus OVERRIDE (method → - * interface method). The explicit relations (INHERITS, DECORATES, declared - * IMPLEMENTS) come from the resolve workers in pass_parallel.c. - */ -#include "foundation/constants.h" -#include "pipeline/pipeline.h" -#include "pipeline/pipeline_internal.h" -#include "graph_buffer/graph_buffer.h" -#include "foundation/limits.h" - -#include -#include -#include - -/* Check if file_path ends with a suffix. */ -static bool fp_ends_with(const char *fp, const char *suffix) { - if (!fp || !suffix) { - return false; - } - size_t fplen = strlen(fp); - size_t sflen = strlen(suffix); - return fplen >= sflen && strcmp(fp + fplen - sflen, suffix) == 0; -} - -/* Info about one interface method (name + node ID). */ -typedef struct { - const char *name; - int64_t id; -} go_imethod_t; - -/* Check if class has all interface methods and create IMPLEMENTS + OVERRIDE edges. */ -static int check_go_class_implements(cbm_pipeline_ctx_t *ctx, const cbm_gbuf_node_t *cls, - const cbm_gbuf_node_t *iface, const go_imethod_t *imethods, - int im_count) { - if (!cls->file_path || !cls->qualified_name) { - return 0; - } - if (!fp_ends_with(cls->file_path, ".go")) { - return 0; - } - /* Resolve the struct's methods two ways and use whichever finds them: - * (a) the struct's DEFINES_METHOD edges, matched by method NAME — the real - * pipeline path, where Go receiver methods carry a flat QN (e.g. - * "pkg.Area" rather than "pkg.Circle.Area"), so QN-string - * reconstruction would miss; and - * (b) the reconstructed QN "." — used by tests and any - * extractor that does emit class-qualified method QNs without - * DEFINES_METHOD edges from the class. */ - const cbm_gbuf_edge_t **cls_dm = NULL; - int cls_dm_count = 0; - cbm_gbuf_find_edges_by_source_type(ctx->gbuf, cls->id, "DEFINES_METHOD", &cls_dm, - &cls_dm_count); - - char prefix[CBM_SZ_512]; - snprintf(prefix, sizeof(prefix), "%s.", cls->qualified_name); - - /* For each interface method, find the matching struct method node. */ - const cbm_gbuf_node_t *matched[CBM_SZ_128]; - for (int m = 0; m < im_count; m++) { - const cbm_gbuf_node_t *found = NULL; - /* (a) DEFINES_METHOD edge by name */ - for (int d = 0; d < cls_dm_count; d++) { - const cbm_gbuf_node_t *cm = cbm_gbuf_find_by_id(ctx->gbuf, cls_dm[d]->target_id); - if (cm && cm->name && strcmp(cm->name, imethods[m].name) == 0) { - found = cm; - break; - } - } - /* (b) reconstructed "." */ - if (!found) { - char method_qn[CBM_SZ_512]; - snprintf(method_qn, sizeof(method_qn), "%s%s", prefix, imethods[m].name); - found = cbm_gbuf_find_by_qn(ctx->gbuf, method_qn); - } - if (!found) { - return 0; /* struct does not satisfy the interface */ - } - matched[m] = found; - } - cbm_gbuf_insert_edge(ctx->gbuf, cls->id, iface->id, "IMPLEMENTS", "{}"); - int edges = SKIP_ONE; - for (int m = 0; m < im_count; m++) { - cbm_gbuf_insert_edge(ctx->gbuf, matched[m]->id, imethods[m].id, "OVERRIDE", "{}"); - edges++; - } - return edges; -} - -int cbm_pipeline_implements_go(cbm_pipeline_ctx_t *ctx) { - int edge_count = 0; - - /* Find all Interface nodes */ - const cbm_gbuf_node_t **ifaces = NULL; - int iface_count = 0; - if (cbm_gbuf_find_by_label(ctx->gbuf, "Interface", &ifaces, &iface_count) != 0) { - return 0; - } - - /* Find candidate concrete types. In Go the type that satisfies an interface - * is a struct (now labelled "Struct") or a named type (labelled "Class"); both - * sets are checked. Each call returns a borrowed internal array (no free). */ - const cbm_gbuf_node_t **classes = NULL; - int class_count = 0; - cbm_gbuf_find_by_label(ctx->gbuf, "Class", &classes, &class_count); - const cbm_gbuf_node_t **structs = NULL; - int struct_count = 0; - cbm_gbuf_find_by_label(ctx->gbuf, "Struct", &structs, &struct_count); - if (class_count == 0 && struct_count == 0) { - return 0; - } - - for (int i = 0; i < iface_count; i++) { - const cbm_gbuf_node_t *iface = ifaces[i]; - if (!iface->file_path || !fp_ends_with(iface->file_path, ".go")) { - continue; - } - - /* Get interface methods via DEFINES_METHOD edges */ - const cbm_gbuf_edge_t **dm_edges = NULL; - int dm_count = 0; - if (cbm_gbuf_find_edges_by_source_type(ctx->gbuf, iface->id, "DEFINES_METHOD", &dm_edges, - &dm_count) != 0 || - dm_count == 0) { - continue; - } - - /* Collect interface method info */ - go_imethod_t imethods[CBM_SZ_128]; - int im_count = 0; - for (int j = 0; j < dm_count && im_count < CBM_SZ_128; j++) { - const cbm_gbuf_node_t *m = cbm_gbuf_find_by_id(ctx->gbuf, dm_edges[j]->target_id); - if (m && m->name) { - imethods[im_count++] = (go_imethod_t){m->name, m->id}; - } - } - if (im_count == 0) { - continue; - } - - /* Check each concrete-type node (Struct + Class) for method-set - * satisfaction. */ - for (int c = 0; c < struct_count; c++) { - edge_count += check_go_class_implements(ctx, structs[c], iface, imethods, im_count); - } - for (int c = 0; c < class_count; c++) { - edge_count += check_go_class_implements(ctx, classes[c], iface, imethods, im_count); - } - } - return edge_count; -} diff --git a/src/pipeline/pass_semantic.zig b/src/pipeline/pass_semantic.zig new file mode 100644 index 00000000..33c1b350 --- /dev/null +++ b/src/pipeline/pass_semantic.zig @@ -0,0 +1,149 @@ +//! pass_semantic.zig — Go implicit interface satisfaction. +//! +//! A Go type implements an interface by having its method set; nothing in the +//! source names the relation. After all definitions are in the graph buffer, +//! this pass matches each Go Struct/Class against each Go Interface by method +//! name and emits IMPLEMENTS (type → interface) plus OVERRIDE (method → +//! interface method). The explicit relations (INHERITS, DECORATES, declared +//! IMPLEMENTS) come from the resolve workers in pass_parallel. + +const std = @import("std"); +const c = @import("c"); +const util = @import("util.zig"); + +const CStr = util.CStr; +const ix = util.ix; +const Node = c.cbm_gbuf_node_t; +const Edge = c.cbm_gbuf_edge_t; + +/// Check if file_path ends with a suffix. +fn fpEndsWith(fp: CStr, suffix: CStr) bool { + if (fp == null or suffix == null) return false; + const fplen = c.strlen(fp); + const sflen = c.strlen(suffix); + return fplen >= sflen and util.streq(fp + fplen - sflen, suffix); +} + +/// Info about one interface method (name + node ID). +const GoImethod = struct { + name: CStr, + id: i64, +}; + +/// Check if class has all interface methods and create IMPLEMENTS + OVERRIDE edges. +fn checkGoClassImplements( + ctx: *c.cbm_pipeline_ctx_t, + cls: *const Node, + iface: *const Node, + imethods: []const GoImethod, +) c_int { + if (cls.file_path == null or cls.qualified_name == null) return 0; + if (!fpEndsWith(cls.file_path, ".go")) return 0; + // Resolve the struct's methods two ways and use whichever finds them: + // (a) the struct's DEFINES_METHOD edges, matched by method NAME — the real + // pipeline path, where Go receiver methods carry a flat QN (e.g. + // "pkg.Area" rather than "pkg.Circle.Area"), so QN-string + // reconstruction would miss; and + // (b) the reconstructed QN "." — used by tests and any + // extractor that does emit class-qualified method QNs without + // DEFINES_METHOD edges from the class. + var cls_dm: [*c][*c]const Edge = null; + var cls_dm_count: c_int = 0; + _ = c.cbm_gbuf_find_edges_by_source_type(ctx.gbuf, cls.id, "DEFINES_METHOD", &cls_dm, &cls_dm_count); + + var prefix: [c.CBM_SZ_512]u8 = undefined; + _ = c.snprintf(&prefix, prefix.len, "%s.", cls.qualified_name); + + // For each interface method, find the matching struct method node. + var matched: [c.CBM_SZ_128]*const Node = undefined; + for (imethods, 0..) |imethod, m| { + var found: ?*const Node = null; + // (a) DEFINES_METHOD edge by name + var d: usize = 0; + while (d < cls_dm_count) : (d += 1) { + const cm: ?*const Node = c.cbm_gbuf_find_by_id(ctx.gbuf, cls_dm[d].*.target_id); + if (cm) |node| { + if (node.name != null and util.streq(node.name, imethod.name)) { + found = node; + break; + } + } + } + // (b) reconstructed "." + if (found == null) { + var method_qn: [c.CBM_SZ_512]u8 = undefined; + _ = c.snprintf(&method_qn, method_qn.len, "%s%s", @as(CStr, &prefix), imethod.name); + found = c.cbm_gbuf_find_by_qn(ctx.gbuf, &method_qn); + } + matched[m] = found orelse return 0; // struct does not satisfy the interface + } + _ = c.cbm_gbuf_insert_edge(ctx.gbuf, cls.id, iface.id, "IMPLEMENTS", "{}"); + var edges: c_int = 1; + for (imethods, 0..) |imethod, m| { + _ = c.cbm_gbuf_insert_edge(ctx.gbuf, matched[m].id, imethod.id, "OVERRIDE", "{}"); + edges += 1; + } + return edges; +} + +pub export fn cbm_pipeline_implements_go(ctx: *c.cbm_pipeline_ctx_t) c_int { + var edge_count: c_int = 0; + + // Find all Interface nodes + var ifaces: [*c][*c]const Node = null; + var iface_count: c_int = 0; + if (c.cbm_gbuf_find_by_label(ctx.gbuf, "Interface", &ifaces, &iface_count) != 0) return 0; + + // Find candidate concrete types. In Go the type that satisfies an interface + // is a struct (now labelled "Struct") or a named type (labelled "Class"); both + // sets are checked. Each call returns a borrowed internal array (no free). + var classes: [*c][*c]const Node = null; + var class_count: c_int = 0; + _ = c.cbm_gbuf_find_by_label(ctx.gbuf, "Class", &classes, &class_count); + var structs: [*c][*c]const Node = null; + var struct_count: c_int = 0; + _ = c.cbm_gbuf_find_by_label(ctx.gbuf, "Struct", &structs, &struct_count); + if (class_count == 0 and struct_count == 0) return 0; + + var i: usize = 0; + while (i < iface_count) : (i += 1) { + const iface: *const Node = ifaces[i]; + if (iface.file_path == null or !fpEndsWith(iface.file_path, ".go")) continue; + + // Get interface methods via DEFINES_METHOD edges + var dm_edges: [*c][*c]const Edge = null; + var dm_count: c_int = 0; + if (c.cbm_gbuf_find_edges_by_source_type(ctx.gbuf, iface.id, "DEFINES_METHOD", &dm_edges, &dm_count) != 0 or + dm_count == 0) + { + continue; + } + + // Collect interface method info + var imethods: [c.CBM_SZ_128]GoImethod = undefined; + var im_count: usize = 0; + var j: usize = 0; + while (j < dm_count and im_count < imethods.len) : (j += 1) { + const m: ?*const Node = c.cbm_gbuf_find_by_id(ctx.gbuf, dm_edges[j].*.target_id); + if (m) |node| { + if (node.name != null) { + imethods[im_count] = .{ .name = node.name, .id = node.id }; + im_count += 1; + } + } + } + if (im_count == 0) continue; + + // Check each concrete-type node (Struct + Class) for method-set + // satisfaction. + var k: usize = 0; + while (k < struct_count) : (k += 1) { + edge_count += checkGoClassImplements(ctx, structs[k], iface, imethods[0..im_count]); + } + k = 0; + while (k < class_count) : (k += 1) { + edge_count += checkGoClassImplements(ctx, classes[k], iface, imethods[0..im_count]); + } + } + return edge_count; +} diff --git a/src/pipeline/pass_semantic_edges.c b/src/pipeline/pass_semantic_edges.c deleted file mode 100644 index c70f1a11..00000000 --- a/src/pipeline/pass_semantic_edges.c +++ /dev/null @@ -1,1427 +0,0 @@ -/* - * pass_semantic_edges.c — Emit SEMANTICALLY_RELATED edges from combined - * algorithmic embeddings (11 signals, zero external dependencies). - * - * Runs as a post-pass after pass_similarity. Reads all Function/Method - * nodes from the graph buffer, computes TF-IDF + Random Indexing + API - * signatures + type/decorator vectors + AST profile, builds LSH index, - * scores candidate pairs, applies graph diffusion, emits edges. - * - * Runs in moderate and full modes (not fast). Controlled by pipeline mode. - */ -#include "foundation/constants.h" -#include "pipeline/pipeline.h" -#include -#include "pipeline/pipeline_internal.h" -#include "graph_buffer/graph_buffer.h" -#include "semantic/semantic.h" -#include "semantic/ast_profile.h" -#include "simhash/minhash.h" -#include "foundation/hash_table.h" -#include "foundation/log.h" -#include "foundation/compat.h" -#define XXH_INLINE_ALL -#include "xxhash/xxhash.h" - -#include "pipeline/worker_pool.h" -#include "foundation/platform.h" -#include "foundation/profile.h" -#include "foundation/str_util.h" - -#include -#include -#include -#include -#include - -/* ── Helpers ─────────────────────────────────────────────────────── */ - -enum { - PROPS_BUF = 512, - MAX_FUNCS_INIT = 4096, - GROW = 2, - MAX_CALLEES = 64, - MAX_DEFERRED_EDGES = 8192, - /* Bit-mask for the low order parity bit used by sparse random indexing. */ - PSE_PARITY_BIT = 1, - /* Bit shift count for signature bit positions (1 << k). */ - PSE_BIT_1 = 1, - PSE_BIT_64 = 64, - PSE_MOD_64 = 64, - PSE_MINHASH_K = 64, - PSE_LSH_BAND_COUNT = 2, - PSE_FP_PREFIX_LEN = 6, /* strlen("\"fp\":\"") */ - PSE_LSH_ROWS_PER_BAND = 2, - PSE_MIN_FUNCS_FOR_PAIR = 2, -}; - -/* Scalar weight constants used in score_worker and related helpers. */ -#define PSE_UNIT_POS 1.0F -#define PSE_INT8_MAX 127.0F -#define PSE_ROUND_BIAS 0.5F -#define PSE_FLOW_WEIGHT 0.01F -#define PSE_ONE_ULL ((uint64_t)1) - -/* ── Deferred edge buffer (thread-local, merged after parallel pass) ── */ - -typedef struct { - int64_t source_id; - int64_t target_id; - float score; - bool same_file; - /* Canonical admission keys (determinism): func index of the discovering - * side, its candidate rank, and the partner's func index. The sequential - * admission pass replays pairs in (i, c) order so which pairs win the - * per-node max_edges budget no longer depends on worker scheduling. */ - int i; - int j; - int c; -} deferred_edge_t; - -typedef struct { - deferred_edge_t *edges; - int count; - int cap; -} deferred_edge_buf_t; - -static void deferred_buf_init(deferred_edge_buf_t *buf) { - buf->edges = NULL; - buf->count = 0; - buf->cap = 0; -} - -static void deferred_buf_push(deferred_edge_buf_t *buf, int64_t src, int64_t tgt, float score, - bool same_file, int i, int j, int c) { - if (buf->count >= buf->cap) { - int nc = buf->cap < CBM_SZ_256 ? CBM_SZ_256 : buf->cap * GROW; - deferred_edge_t *grown = realloc(buf->edges, (size_t)nc * sizeof(deferred_edge_t)); - if (!grown) { - return; - } - buf->edges = grown; - buf->cap = nc; - } - buf->edges[buf->count++] = (deferred_edge_t){.source_id = src, - .target_id = tgt, - .score = score, - .same_file = same_file, - .i = i, - .j = j, - .c = c}; -} - -static void deferred_buf_free(deferred_edge_buf_t *buf) { - free(buf->edges); - buf->edges = NULL; - buf->count = buf->cap = 0; -} - -/* Forward declare helpers used by pattern injection. */ -static const char *json_str_value(const char *json, const char *key, char *buf, int bufsize); - -/* ── Technique 2: Code pattern vocabulary injection ──────────────── */ -/* Inject semantic tokens based on detected code patterns. - * This bridges the vocabulary gap for abstract concepts like "error handling". */ - -/* Append a single token to `tokens` if there's capacity. Consolidates the - * `if (count < max_tokens) { tokens[count++] = strdup(...); }` pattern. */ -static int push_pattern_token(char **tokens, int count, int max_tokens, const char *text) { - if (count >= max_tokens) { - return count; - } - tokens[count] = strdup(text); - return tokens[count] ? count + SKIP_ONE : count; -} - -/* True if `s` contains any of the space-separated substrings in `needles`. */ -static bool has_any(const char *s, const char *const *needles) { - if (!s) { - return false; - } - for (const char *const *n = needles; *n; n++) { - if (strstr(s, *n)) { - return true; - } - } - return false; -} - -/* Inject tokens derived from body-text patterns (try/catch, raise, log). */ -static int inject_body_pattern_tokens(const char *bt, char **tokens, int count, int max_tokens) { - if (!bt) { - return count; - } - static const char *const ERR_HANDLING[] = {"except", "catch", "rescue", NULL}; - if (has_any(bt, ERR_HANDLING)) { - count = push_pattern_token(tokens, count, max_tokens, "error"); - count = push_pattern_token(tokens, count, max_tokens, "handling"); - count = push_pattern_token(tokens, count, max_tokens, "exception"); - } - static const char *const ERR_THROW[] = {"raise", "throw", NULL}; - if (has_any(bt, ERR_THROW)) { - count = push_pattern_token(tokens, count, max_tokens, "error"); - count = push_pattern_token(tokens, count, max_tokens, "exception"); - count = push_pattern_token(tokens, count, max_tokens, "throw"); - } - static const char *const LOGGING[] = {"logger", "logging", "log_", NULL}; - if (has_any(bt, LOGGING)) { - count = push_pattern_token(tokens, count, max_tokens, "logging"); - count = push_pattern_token(tokens, count, max_tokens, "log"); - } - return count; -} - -/* Inject tokens for one CALLS-target name based on keyword groups. */ -static int inject_callee_tokens(const char *name, char **tokens, int count, int max_tokens) { - static const char *const LOG_FNS[] = {"log", "Log", "warn", "debug", "info", NULL}; - static const char *const ERR_FNS[] = {"Error", "error", "Errorf", "panic", NULL}; - static const char *const IO_FNS[] = {"open", "read", "write", "close", "Open", "Read", NULL}; - if (has_any(name, LOG_FNS)) { - count = push_pattern_token(tokens, count, max_tokens, "logging"); - count = push_pattern_token(tokens, count, max_tokens, "log"); - } - if (has_any(name, ERR_FNS)) { - count = push_pattern_token(tokens, count, max_tokens, "error"); - count = push_pattern_token(tokens, count, max_tokens, "handling"); - } - if (has_any(name, IO_FNS)) { - count = push_pattern_token(tokens, count, max_tokens, "io"); - count = push_pattern_token(tokens, count, max_tokens, "file"); - } - return count; -} - -static const char **collect_sorted_call_neighbors(const cbm_gbuf_t *gbuf, int64_t node_id, - bool outbound, int *out_n); - -/* Walk the outbound CALLS edges of n and inject tokens for each target. */ -static int inject_calls_pattern_tokens(const cbm_gbuf_node_t *n, const cbm_gbuf_t *gbuf, - char **tokens, int count, int max_tokens) { - if (!gbuf) { - return count; - } - int neighbor_count = 0; - const char **names = collect_sorted_call_neighbors(gbuf, n->id, true, &neighbor_count); - for (int i = 0; i < neighbor_count && count < max_tokens; i++) { - count = inject_callee_tokens(names[i], tokens, count, max_tokens); - } - free(names); - return count; -} - -/* Inject tokens from decorator annotations (@route, @middleware, @pytest.*). */ -static int inject_decorator_tokens(const char *decs, char **tokens, int count, int max_tokens) { - if (!decs) { - return count; - } - static const char *const ROUTING[] = {"route", "Route", "app.", NULL}; - if (has_any(decs, ROUTING)) { - count = push_pattern_token(tokens, count, max_tokens, "routing"); - count = push_pattern_token(tokens, count, max_tokens, "endpoint"); - count = push_pattern_token(tokens, count, max_tokens, "handler"); - } - static const char *const MIDDLEWARE[] = {"middleware", "Middleware", NULL}; - if (has_any(decs, MIDDLEWARE)) { - count = push_pattern_token(tokens, count, max_tokens, "middleware"); - } - static const char *const TEST[] = {"test", "Test", "pytest", NULL}; - if (has_any(decs, TEST)) { - count = push_pattern_token(tokens, count, max_tokens, "test"); - count = push_pattern_token(tokens, count, max_tokens, "testing"); - } - return count; -} - -/* Inject tokens keyed off the node's own name (test_*, *Handler, validator). */ -static int inject_name_pattern_tokens(const char *name, char **tokens, int count, int max_tokens) { - if (!name) { - return count; - } - static const char *const TEST[] = {"test_", "Test", NULL}; - if (has_any(name, TEST)) { - count = push_pattern_token(tokens, count, max_tokens, "test"); - count = push_pattern_token(tokens, count, max_tokens, "testing"); - } - static const char *const MIDDLEWARE[] = {"middleware", "Middleware", NULL}; - if (has_any(name, MIDDLEWARE)) { - count = push_pattern_token(tokens, count, max_tokens, "middleware"); - } - static const char *const HANDLER[] = {"handler", "Handler", NULL}; - if (has_any(name, HANDLER)) { - count = push_pattern_token(tokens, count, max_tokens, "handler"); - } - static const char *const VALIDATOR[] = {"validator", "Validator", "validate", "Validate", NULL}; - if (has_any(name, VALIDATOR)) { - count = push_pattern_token(tokens, count, max_tokens, "validation"); - } - return count; -} - -static int inject_pattern_tokens(const cbm_gbuf_node_t *n, const cbm_gbuf_t *gbuf, char **tokens, - int count, int max_tokens) { - if (!n || count >= max_tokens) { - return count; - } - - char bt_buf[CBM_SZ_512]; - const char *bt = n->properties_json - ? json_str_value(n->properties_json, "bt", bt_buf, sizeof(bt_buf)) - : NULL; - char dec_buf[CBM_SZ_256]; - const char *decs = n->properties_json ? json_str_value(n->properties_json, "decorators", - dec_buf, sizeof(dec_buf)) - : NULL; - - count = inject_body_pattern_tokens(bt, tokens, count, max_tokens); - count = inject_calls_pattern_tokens(n, gbuf, tokens, count, max_tokens); - count = inject_decorator_tokens(decs, tokens, count, max_tokens); - count = inject_name_pattern_tokens(n->name, tokens, count, max_tokens); - return count; -} - -/* ── Technique 3: Field weights for token sources ────────────────── */ -enum { - FW_NAME = 30, /* ×3.0 */ - FW_CALLEE = 20, /* ×2.0 */ - FW_BODY = 15, /* ×1.5 */ - FW_SIGNATURE = 10, /* ×1.0 */ - FW_PARAM = 10, /* ×1.0 */ - FW_PATTERN = 25, /* ×2.5 — injected semantic tokens are high value */ - FW_PATH = 5, /* ×0.5 */ - FW_SCALE = 10, /* divisor */ -}; - -/* Extract a JSON string value by key (simple strstr-based, no full parse). */ -static const char *json_str_value(const char *json, const char *key, char *buf, int bufsize) { - if (!json || !key) { - return NULL; - } - char search[CBM_SZ_64]; - snprintf(search, sizeof(search), "\"%s\":\"", key); - const char *start = strstr(json, search); - if (!start) { - return NULL; - } - start += strlen(search); - const char *end = strchr(start, '"'); - if (!end) { - return NULL; - } - int len = (int)(end - start); - if (len >= bufsize) { - len = bufsize - SKIP_ONE; - } - memcpy(buf, start, (size_t)len); - buf[len] = '\0'; - return buf; -} - -/* Extract a JSON array of strings by key. Returns count. */ -static int json_str_array(const char *json, const char *key, char **out, int max_out) { - if (!json || !key) { - return 0; - } - char search[CBM_SZ_64]; - snprintf(search, sizeof(search), "\"%s\":[", key); - const char *start = strstr(json, search); - if (!start) { - return 0; - } - start += strlen(search); - int count = 0; - while (*start && *start != ']' && count < max_out) { - if (*start == '"') { - start++; - const char *end = strchr(start, '"'); - if (!end) { - break; - } - int len = (int)(end - start); - out[count] = malloc((size_t)len + SKIP_ONE); - memcpy(out[count], start, (size_t)len); - out[count][len] = '\0'; - count++; - start = end + SKIP_ONE; - } else { - start++; - } - } - return count; -} - -/* ── Tokenize node metadata ──────────────────────────────────────── */ - -/* Tokenize a single string field keyed out of node->properties_json, if - * present. Returns the new count after appending any tokens. */ -static int tokenize_json_string_field(const char *json, const char *key, char **tokens, int count, - int max_tokens) { - if (count >= max_tokens) { - return count; - } - char buf[CBM_SZ_512]; - if (!json_str_value(json, key, buf, sizeof(buf))) { - return count; - } - count += cbm_sem_tokenize(buf, tokens + count, max_tokens - count); - return count; -} - -/* Tokenize a JSON array field (e.g. "param_names", "decorators"). */ -static int tokenize_json_array_field(const char *json, const char *key, char **tokens, int count, - int max_tokens) { - if (count >= max_tokens) { - return count; - } - char *arr[CBM_SZ_16]; - int n = json_str_array(json, key, arr, CBM_SZ_16); - for (int p = 0; p < n; p++) { - if (count < max_tokens) { - count += cbm_sem_tokenize(arr[p], tokens + count, max_tokens - count); - } - free(arr[p]); - } - return count; -} - -/* Walk the CALLS edges rooted at n (either outbound or inbound depending on - * `outbound`) and tokenize the names of the target/source nodes. Caller-side - * caps via max_tokens and MAX_CALLEES. */ -static int cmp_name_ptr(const void *pa, const void *pb) { - const char *a = *(const char *const *)pa; - const char *b = *(const char *const *)pb; - return strcmp(a, b); -} - -/* Collect the CALLS-neighbor names of `node_id`, SORTED by name. The edge - * arrays are in insertion order — which under parallel extraction varies run - * to run — and both consumers truncate (MAX_CALLEES / max_tokens), so an - * unstable order changed WHICH neighbors contribute to the semantic vectors - * and flickered near-threshold SEMANTICALLY_RELATED edges (determinism). - * Returns a malloc'd array of borrowed name pointers; caller frees the array. */ -static const char **collect_sorted_call_neighbors(const cbm_gbuf_t *gbuf, int64_t node_id, - bool outbound, int *out_n) { - *out_n = 0; - const cbm_gbuf_edge_t **edges = NULL; - int ec = 0; - int rc = outbound ? cbm_gbuf_find_edges_by_source_type(gbuf, node_id, "CALLS", &edges, &ec) - : cbm_gbuf_find_edges_by_target_type(gbuf, node_id, "CALLS", &edges, &ec); - if (rc != 0 || ec <= 0) { - return NULL; - } - const char **names = malloc((size_t)ec * sizeof(char *)); - if (!names) { - return NULL; - } - int n = 0; - for (int e = 0; e < ec; e++) { - int64_t id = outbound ? edges[e]->target_id : edges[e]->source_id; - const cbm_gbuf_node_t *neighbor = cbm_gbuf_find_by_id(gbuf, id); - if (neighbor && neighbor->name) { - names[n++] = neighbor->name; - } - } - if (n > 1) { - qsort(names, (size_t)n, sizeof(char *), cmp_name_ptr); - } - *out_n = n; - return names; -} - -static int tokenize_call_neighbors(const cbm_gbuf_node_t *n, const cbm_gbuf_t *gbuf, bool outbound, - char **tokens, int count, int max_tokens) { - if (!gbuf || count >= max_tokens) { - return count; - } - int nn = 0; - const char **names = collect_sorted_call_neighbors(gbuf, n->id, outbound, &nn); - if (!names) { - return count; - } - for (int e = 0; e < nn && e < MAX_CALLEES && count < max_tokens; e++) { - count += cbm_sem_tokenize(names[e], tokens + count, max_tokens - count); - } - free((void *)names); - return count; -} - -static int tokenize_node(const cbm_gbuf_node_t *n, const cbm_gbuf_t *gbuf, char **tokens, - int max_tokens) { - int count = 0; - count += cbm_sem_tokenize(n->name, tokens + count, max_tokens - count); - if (n->qualified_name && count < max_tokens) { - count += cbm_sem_tokenize(n->qualified_name, tokens + count, max_tokens - count); - } - if (n->file_path && count < max_tokens) { - count += cbm_sem_tokenize(n->file_path, tokens + count, max_tokens - count); - } - if (n->properties_json) { - count = - tokenize_json_string_field(n->properties_json, "signature", tokens, count, max_tokens); - count = tokenize_json_string_field(n->properties_json, "return_type", tokens, count, - max_tokens); - count = - tokenize_json_string_field(n->properties_json, "docstring", tokens, count, max_tokens); - count = - tokenize_json_array_field(n->properties_json, "param_names", tokens, count, max_tokens); - count = - tokenize_json_array_field(n->properties_json, "param_types", tokens, count, max_tokens); - count = - tokenize_json_array_field(n->properties_json, "decorators", tokens, count, max_tokens); - count = tokenize_json_string_field(n->properties_json, "bt", tokens, count, max_tokens); - } - count = tokenize_call_neighbors(n, gbuf, /*outbound=*/true, tokens, count, max_tokens); - - /* Caller names: what CALLS this function (contextual vocabulary). - * Functions called by error handlers inherit "error" context. */ - count = tokenize_call_neighbors(n, gbuf, /*outbound=*/false, tokens, count, max_tokens); - return count; -} - -/* ── Build per-function semantic data ────────────────────────────── */ - -static void build_api_vec(const cbm_gbuf_t *gbuf, int64_t node_id, cbm_sem_vec_t *out) { - memset(out, 0, sizeof(*out)); - /* Sorted neighbors: stable MAX_CALLEES subset + stable float-accumulation - * order (see collect_sorted_call_neighbors). */ - int n = 0; - const char **names = collect_sorted_call_neighbors(gbuf, node_id, /*outbound=*/true, &n); - if (!names) { - return; - } - for (int i = 0; i < n && i < MAX_CALLEES; i++) { - cbm_sem_vec_t callee_ri; - cbm_sem_random_index(names[i], &callee_ri); - cbm_sem_vec_add_scaled(out, &callee_ri, PSE_UNIT_POS); - } - free((void *)names); - cbm_sem_normalize(out); -} - -static void build_type_vec(const char *props_json, cbm_sem_vec_t *out) { - memset(out, 0, sizeof(*out)); - if (!props_json) { - return; - } - /* Extract param_types and return_type */ - char rt_buf[CBM_SZ_128]; - if (json_str_value(props_json, "return_type", rt_buf, sizeof(rt_buf))) { - cbm_sem_vec_t ri; - cbm_sem_random_index(rt_buf, &ri); - cbm_sem_vec_add_scaled(out, &ri, PSE_UNIT_POS); - } - char *ptypes[CBM_SZ_16]; - int pt_count = json_str_array(props_json, "param_types", ptypes, CBM_SZ_16); - for (int i = 0; i < pt_count; i++) { - cbm_sem_vec_t ri; - cbm_sem_random_index(ptypes[i], &ri); - cbm_sem_vec_add_scaled(out, &ri, PSE_UNIT_POS); - free(ptypes[i]); - } - cbm_sem_normalize(out); -} - -static void build_deco_vec(const char *props_json, cbm_sem_vec_t *out) { - memset(out, 0, sizeof(*out)); - if (!props_json) { - return; - } - char *decos[CBM_SZ_16]; - int dc = json_str_array(props_json, "decorators", decos, CBM_SZ_16); - for (int i = 0; i < dc; i++) { - cbm_sem_vec_t ri; - cbm_sem_random_index(decos[i], &ri); - cbm_sem_vec_add_scaled(out, &ri, PSE_UNIT_POS); - free(decos[i]); - } - cbm_sem_normalize(out); -} - -static void decode_struct_profile(const char *props_json, float *out) { - memset(out, 0, sizeof(float) * CBM_AST_PROFILE_DIMS); - if (!props_json) { - return; - } - char sp_buf[CBM_AST_PROFILE_BUF]; - if (json_str_value(props_json, "sp", sp_buf, sizeof(sp_buf))) { - cbm_ast_profile_t profile; - if (cbm_ast_profile_from_str(sp_buf, &profile)) { - cbm_ast_profile_to_vector(&profile, out); - } - } -} - -static void decode_minhash(const char *props_json, cbm_sem_func_t *func) { - func->has_minhash = false; - if (!props_json) { - return; - } - const char *fp_key = strstr(props_json, "\"fp\":\""); - if (!fp_key) { - return; - } - const char *hex = fp_key + PSE_FP_PREFIX_LEN; /* strlen("\"fp\":\"") */ - const char *end = strchr(hex, '"'); - if (!end || (int)(end - hex) != CBM_MINHASH_HEX_LEN) { - return; - } - char hex_buf[CBM_MINHASH_HEX_BUF]; - memcpy(hex_buf, hex, CBM_MINHASH_HEX_LEN); - hex_buf[CBM_MINHASH_HEX_LEN] = '\0'; - cbm_minhash_t mh; - if (cbm_minhash_from_hex(hex_buf, &mh)) { - memcpy(func->minhash, mh.values, sizeof(func->minhash)); - func->has_minhash = true; - } -} - -/* ── Parallel Phase 2: Tokenize nodes ────────────────────────────── */ - -typedef struct { - const cbm_gbuf_node_t **node_ptrs; /* node pointer per function index */ - cbm_gbuf_t *gbuf; /* read-only during tokenization */ - char **all_tokens; /* output: all_tokens[f * MAX + t] */ - int *token_counts; /* output: token count per function */ - int func_count; - _Atomic int next_idx; - /* Per-worker token intern pools (key==value==the one owned strdup): - * identical tokens ("xfs", "error", ...) recur across hundreds of - * thousands of functions; per-func strdups made all_tokens hold every - * instance. Interned, it holds at most workers x unique tokens. */ - CBMHashTable **pools; -} tokenize_ctx_t; - -static void tokenize_worker(int worker_id, void *ctx_ptr) { - tokenize_ctx_t *tc = ctx_ptr; - while (true) { - int f = atomic_fetch_add_explicit(&tc->next_idx, SKIP_ONE, memory_order_relaxed); - if (f >= tc->func_count) { - break; - } - - const cbm_gbuf_node_t *n = tc->node_ptrs[f]; - /* Write directly into the shared buffer slice for this function — the - * strdup'd tokens are owned by all_tokens[] from the moment they land - * in this slot, which avoids a spurious analyzer "leak" diagnostic on - * the previous stack-local relay pattern. */ - char **dst = &tc->all_tokens[(ptrdiff_t)f * CBM_SEM_MAX_TOKENS]; - int count = tokenize_node(n, tc->gbuf, dst, CBM_SEM_MAX_TOKENS); - count = inject_pattern_tokens(n, tc->gbuf, dst, count, CBM_SEM_MAX_TOKENS); - if (tc->pools && tc->pools[worker_id]) { - CBMHashTable *pool = tc->pools[worker_id]; - for (int t = 0; t < count; t++) { - char *canon = cbm_ht_get(pool, dst[t]); - if (canon) { - free(dst[t]); - dst[t] = canon; - } else { - cbm_ht_set(pool, dst[t], dst[t]); /* key borrows the value */ - } - } - } - tc->token_counts[f] = count; - } -} - -/* ── Parallel Phase 4: Build per-function vectors ────────────────── */ - -typedef struct { - cbm_sem_func_t *funcs; - char **all_tokens; - int *token_counts; - cbm_sem_corpus_t *corpus; - uint8_t *qvecs; /* output: pre-quantized int8 vectors [func_count * CBM_SEM_DIM] */ - int func_count; - _Atomic int next_idx; -} vec_build_ctx_t; - -static void vec_build_worker(int worker_id, void *ctx_ptr) { - (void)worker_id; - vec_build_ctx_t *vc = ctx_ptr; - while (true) { - int f = atomic_fetch_add_explicit(&vc->next_idx, SKIP_ONE, memory_order_relaxed); - if (f >= vc->func_count) { - break; - } - - int tc = vc->token_counts[f]; - char **tokens = &vc->all_tokens[(ptrdiff_t)f * CBM_SEM_MAX_TOKENS]; - - /* TF-IDF weights */ - int *indices = malloc((size_t)tc * sizeof(int)); - float *weights = malloc((size_t)tc * sizeof(float)); - int tfidf_len = 0; - for (int t = 0; t < tc; t++) { - float idf = cbm_sem_corpus_idf(vc->corpus, tokens[t]); - if (idf > 0.0F) { - indices[tfidf_len] = t; - weights[tfidf_len] = idf; - tfidf_len++; - } - } - vc->funcs[f].tfidf_indices = indices; - vc->funcs[f].tfidf_weights = weights; - vc->funcs[f].tfidf_len = tfidf_len; - - /* RI vector: sum of enriched token vectors weighted by IDF. Built in a - * LOCAL dense buffer; only the quantized code is retained per func - * (the resident dense floats were ~9.4 GB on the kernel). */ - cbm_sem_vec_t ri_dense; - memset(&ri_dense, 0, sizeof(cbm_sem_vec_t)); - for (int t = 0; t < tc; t++) { - const cbm_sem_vec_t *ri = cbm_sem_corpus_ri_vec(vc->corpus, tokens[t]); - if (ri) { - float idf = cbm_sem_corpus_idf(vc->corpus, tokens[t]); - cbm_sem_vec_add_scaled(&ri_dense, ri, idf); - } - } - cbm_sem_normalize(&ri_dense); - cbm_rsq_encode(ri_dense.v, &vc->funcs[f].ri_code); - - /* Int8 quantize into pre-allocated output array (parallel-safe) */ - uint8_t *qv = &vc->qvecs[(ptrdiff_t)f * CBM_SEM_DIM]; - for (int d = 0; d < CBM_SEM_DIM; d++) { - float v = ri_dense.v[d]; - if (v > PSE_UNIT_POS) { - v = PSE_UNIT_POS; - } - if (v < -PSE_UNIT_POS) { - v = -PSE_UNIT_POS; - } - qv[d] = (uint8_t)(int8_t)(v * PSE_INT8_MAX); - } - } -} - -/* ── Parallel Phase 5: LSH signatures ────────────────────────────── */ - -enum { - NUM_HYPERPLANES = 64, - SEM_LSH_BANDS = 16, - SEM_LSH_ROWS = 4, - SEM_BUCKET_COUNT = 65536, - SEM_BUCKET_MASK = 65535, - SEM_BUCKET_CAP_INIT = 16, - SEM_MAX_CANDIDATES = 200, -}; - -/* Row of a hyperplane matrix in the ROTATED (quantized) basis: LSH only - * needs signs of dots against random directions, and a random direction in - * the rotated basis is as random as one in the original — so signatures are - * computed from dequantized codes without keeping dense originals. */ -typedef float hyperplane_row_t[CBM_RSQ_DIM]; - -typedef struct { - cbm_sem_func_t *funcs; - uint64_t *signatures; - hyperplane_row_t *hyperplanes; - int func_count; - _Atomic int next_idx; -} sig_build_ctx_t; - -static void sig_build_worker(int worker_id, void *ctx_ptr) { - (void)worker_id; - sig_build_ctx_t *sc = ctx_ptr; - while (true) { - int f = atomic_fetch_add_explicit(&sc->next_idx, SKIP_ONE, memory_order_relaxed); - if (f >= sc->func_count) { - break; - } - - float dec[CBM_RSQ_DIM]; - cbm_rsq_decode(&sc->funcs[f].ri_code, dec); - uint64_t sig = 0; - for (int h = 0; h < NUM_HYPERPLANES; h++) { - float dot = 0.0F; - for (int d = 0; d < CBM_RSQ_DIM; d++) { - dot += dec[d] * sc->hyperplanes[h][d]; - } - if (dot > 0.0F) { - sig |= (PSE_ONE_ULL << h); - } - } - sc->signatures[f] = sig; - } -} - -/* ── Parallel Phase 6: Score candidates + collect edges ──────────── */ - -typedef struct { - cbm_sem_func_t *funcs; - uint64_t *signatures; - int *edge_counts; /* budget applied sequentially in phase6b (determinism) */ - cbm_sem_config_t cfg; - int func_count; - - /* LSH buckets (read-only during scoring) */ - struct { - int *items; - int count; - int cap; - } **band_buckets; - - /* Per-worker edge buffer */ - deferred_edge_buf_t *worker_bufs; - int max_workers; - _Atomic int next_idx; -} score_ctx_t; - -enum { - SCORE_SEEN_CAP = 8192, - SCORE_SEEN_MASK = 8191, - SCORE_SEEN_EMPTY = -1, -}; - -/* Check whether `j` has already been recorded in the open-addressed `seen` - * set for this function; insert it if not. Returns true when the insertion - * was fresh (caller should add to candidates). */ -static bool score_seen_insert(int *seen, int j) { - uint32_t slot = (uint32_t)j & SCORE_SEEN_MASK; - for (int p = 0; p < SCORE_SEEN_CAP; p++) { - uint32_t idx = (slot + (uint32_t)p) & SCORE_SEEN_MASK; - if (seen[idx] == SCORE_SEEN_EMPTY) { - seen[idx] = j; - return true; - } - if (seen[idx] == j) { - return false; - } - } - return false; -} - -/* Collect the unique candidate function indices for node `i` by iterating - * every LSH band and merging bucket members via `seen[]`. Returns the - * populated candidate count. */ -static int score_collect_candidates(score_ctx_t *sc, int i, int *seen, int *candidates, - int cand_cap) { - int cand_count = 0; - for (int b = 0; b < SEM_LSH_BANDS && cand_count < cand_cap; b++) { - int shift = b * SEM_LSH_ROWS; - uint32_t band_val = (uint32_t)((sc->signatures[i] >> shift) & - ((PSE_ONE_ULL << SEM_LSH_ROWS) - PSE_ONE_ULL)); - uint64_t bh = XXH3_64bits_withSeed(&band_val, sizeof(band_val), (uint64_t)b); - uint32_t bucket_idx = (uint32_t)(bh & SEM_BUCKET_MASK); - int bcount = sc->band_buckets[b][bucket_idx].count; - int *bitems = sc->band_buckets[b][bucket_idx].items; - if (bcount > SEM_MAX_CANDIDATES) { - continue; - } - for (int k = 0; k < bcount && cand_count < cand_cap; k++) { - int j = bitems[k]; - if (j <= i) { - continue; - } - if (score_seen_insert(seen, j)) { - candidates[cand_count++] = j; - } - } - } - return cand_count; -} - -/* Score one candidate pair (i, j) and push a deferred candidate edge if the - * score passes the threshold. ADMISSION (the per-node max_edges budget) is - * deliberately NOT decided here: the old check-then-increment on shared - * atomic counts made the admitted edge SET depend on worker scheduling — - * multi-threaded runs lost edges vs single-threaded and differed run-to-run - * (repro_parallel_edge_determinism). Scoring is pure math and stays parallel; - * the budget is applied afterwards in one sequential pass over the pairs in - * canonical (i, candidate-rank) order. */ -static void score_try_emit(score_ctx_t *sc, int i, int j, int c, deferred_edge_buf_t *my_buf) { - if (strcmp(sc->funcs[i].file_ext, sc->funcs[j].file_ext) != 0) { - return; - } - float score = cbm_sem_combined_score(&sc->funcs[i], &sc->funcs[j], &sc->cfg); - if (score < sc->cfg.threshold) { - return; - } - bool same_file = sc->funcs[i].file_path && sc->funcs[j].file_path && - strcmp(sc->funcs[i].file_path, sc->funcs[j].file_path) == 0; - deferred_buf_push(my_buf, sc->funcs[i].node_id, sc->funcs[j].node_id, score, same_file, i, j, - c); -} - -static void score_worker(int worker_id, void *ctx_ptr) { - score_ctx_t *sc = ctx_ptr; - deferred_edge_buf_t *my_buf = &sc->worker_bufs[worker_id]; - - while (true) { - int i = atomic_fetch_add_explicit(&sc->next_idx, SKIP_ONE, memory_order_relaxed); - if (i >= sc->func_count) { - break; - } - int seen[SCORE_SEEN_CAP]; - for (int s = 0; s < SCORE_SEEN_CAP; s++) { - seen[s] = SCORE_SEEN_EMPTY; - } - int candidates[SEM_MAX_CANDIDATES]; - int cand_count = score_collect_candidates(sc, i, seen, candidates, SEM_MAX_CANDIDATES); - for (int c = 0; c < cand_count; c++) { - score_try_emit(sc, i, candidates[c], c, my_buf); - } - } -} - -/* ── Parallel Phase 1b: decode minhash/profile + build per-func vectors ── */ - -typedef struct { - cbm_sem_func_t *funcs; - const cbm_gbuf_node_t **node_ptrs; - const cbm_gbuf_t *gbuf; - int func_count; - _Atomic int next_idx; -} collect_ctx_t; - -static void collect_worker(int worker_id, void *ctx_ptr) { - (void)worker_id; - collect_ctx_t *cc = ctx_ptr; - while (true) { - int f = atomic_fetch_add_explicit(&cc->next_idx, PSE_MOD_64, memory_order_relaxed); - if (f >= cc->func_count) { - break; - } - int end = f + PSE_MOD_64; - if (end > cc->func_count) { - end = cc->func_count; - } - for (int i = f; i < end; i++) { - const cbm_gbuf_node_t *n = cc->node_ptrs[i]; - decode_minhash(n->properties_json, &cc->funcs[i]); - decode_struct_profile(n->properties_json, cc->funcs[i].struct_profile); - cbm_sem_vec_t tmp_vec; - build_api_vec(cc->gbuf, n->id, &tmp_vec); - cbm_rsq_encode(tmp_vec.v, &cc->funcs[i].api_code); - build_type_vec(n->properties_json, &tmp_vec); - cbm_rsq_encode(tmp_vec.v, &cc->funcs[i].type_code); - build_deco_vec(n->properties_json, &tmp_vec); - cbm_rsq_encode(tmp_vec.v, &cc->funcs[i].deco_code); - } - } -} - -/* ── Phase helpers (keep cbm_pipeline_pass_semantic_edges complexity low) ── */ - -typedef struct { - int *items; - int count; - int cap; -} sem_bucket_t; - -/* Canonical node order: by qualified name (unique per node), id tie-break - * for defensiveness. Gives the semantic pass a stable, content-derived input - * order regardless of how parallel extraction merged the graph buffer. */ -static int cmp_node_ptr_by_qn(const void *pa, const void *pb) { - const cbm_gbuf_node_t *a = *(const cbm_gbuf_node_t *const *)pa; - const cbm_gbuf_node_t *b = *(const cbm_gbuf_node_t *const *)pb; - const char *qa = a->qualified_name ? a->qualified_name : ""; - const char *qb = b->qualified_name ? b->qualified_name : ""; - int r = strcmp(qa, qb); - if (r != 0) { - return r; - } - if (a->id != b->id) { - return a->id < b->id ? -1 : 1; - } - return 0; -} - -/* Phase 1a: seed the funcs[] / node_ptrs[] arrays from all Function and - * Method nodes in the graph buffer. Returns the number of functions collected - * (0 on OOM), and fills *out_funcs / *out_nodes with newly malloc'd arrays. */ -static int phase1_scan_functions(cbm_gbuf_t *gbuf, cbm_sem_func_t **out_funcs, - const cbm_gbuf_node_t ***out_nodes) { - *out_funcs = NULL; - *out_nodes = NULL; - cbm_sem_func_t *funcs = NULL; - const cbm_gbuf_node_t **node_ptrs = NULL; - int func_count = 0; - int func_cap = 0; - const char *labels[] = {"Function", "Method", NULL}; - for (int li = 0; labels[li]; li++) { - const cbm_gbuf_node_t **nodes = NULL; - int node_count = 0; - if (cbm_gbuf_find_by_label(gbuf, labels[li], &nodes, &node_count) != 0) { - continue; - } - for (int i = 0; i < node_count; i++) { - if (func_count >= func_cap) { - int new_cap = func_cap < MAX_FUNCS_INIT ? MAX_FUNCS_INIT : func_cap * GROW; - cbm_sem_func_t *grown = realloc(funcs, (size_t)new_cap * sizeof(cbm_sem_func_t)); - if (!grown) { - break; - } - funcs = grown; - const cbm_gbuf_node_t **np_grown = - realloc(node_ptrs, (size_t)new_cap * sizeof(cbm_gbuf_node_t *)); - if (!np_grown) { - break; - } - node_ptrs = np_grown; - func_cap = new_cap; - } - memset(&funcs[func_count], 0, sizeof(cbm_sem_func_t)); - funcs[func_count].node_id = nodes[i]->id; - funcs[func_count].file_path = nodes[i]->file_path; - funcs[func_count].file_ext = cbm_path_last_dot(nodes[i]->file_path); - node_ptrs[func_count] = nodes[i]; - func_count++; - } - } - /* Canonicalize the func order (determinism): the label-index order above - * is gbuf insertion order = parallel-extraction merge order, which varies - * run to run. Everything downstream is order-sensitive — LSH bucket chain - * order, the SEM_MAX_CANDIDATES truncation, seen[] and the admission - * sequence — so an unstable order changes WHICH semantic edges are - * emitted. Sort the cheap pointer array by qualified name (unique) and - * re-derive the three fields set so far; the heavy per-func payloads are - * filled in later phases, so no 12.7 KB structs are moved. */ - if (func_count > 1) { - qsort(node_ptrs, (size_t)func_count, sizeof(node_ptrs[0]), cmp_node_ptr_by_qn); - } - for (int k = 0; k < func_count; k++) { - funcs[k].node_id = node_ptrs[k]->id; - funcs[k].file_path = node_ptrs[k]->file_path; - funcs[k].file_ext = cbm_path_last_dot(node_ptrs[k]->file_path); - } - *out_funcs = funcs; - *out_nodes = node_ptrs; - return func_count; -} - -/* Phase 5c: partition functions into LSH buckets by their signature bands. - * Sequential because each bucket grows its `items` array via realloc. */ -static void phase5c_build_lsh_buckets(const uint64_t *signatures, int func_count, - sem_bucket_t **band_buckets) { - for (int f = 0; f < func_count; f++) { - for (int b = 0; b < SEM_LSH_BANDS; b++) { - int shift = b * SEM_LSH_ROWS; - uint32_t band_val = (uint32_t)((signatures[f] >> shift) & - ((PSE_ONE_ULL << SEM_LSH_ROWS) - PSE_ONE_ULL)); - uint64_t bh = XXH3_64bits_withSeed(&band_val, sizeof(band_val), (uint64_t)b); - uint32_t bucket_idx = (uint32_t)(bh & SEM_BUCKET_MASK); - sem_bucket_t *bucket = &band_buckets[b][bucket_idx]; - if (bucket->count >= bucket->cap) { - int nc = - bucket->cap < SEM_BUCKET_CAP_INIT ? SEM_BUCKET_CAP_INIT : bucket->cap * GROW; - int *ni = realloc(bucket->items, (size_t)nc * sizeof(int)); - if (!ni) { - continue; - } - bucket->items = ni; - bucket->cap = nc; - } - bucket->items[bucket->count++] = f; - } - } -} - -/* Phase 6b: serialize deferred edges from all worker buffers into the graph - * buffer (sequential because gbuf isn't thread-safe). */ -/* Canonical order for candidate pairs: ascending discovering-func index, - * then ascending candidate rank — the order a sequential scoring loop over - * canonically-sorted funcs would have produced. */ -static int cmp_deferred_edge_canonical(const void *pa, const void *pb) { - const deferred_edge_t *a = pa; - const deferred_edge_t *b = pb; - if (a->i != b->i) { - return a->i < b->i ? -1 : 1; - } - if (a->c != b->c) { - return a->c < b->c ? -1 : 1; - } - return 0; -} - -/* Sequential, deterministic admission + merge: gather every above-threshold - * pair from the worker buffers, replay them in canonical (i, rank) order, - * and apply the per-node max_edges budget HERE — single-threaded — so the - * admitted edge set is a pure function of the (canonically sorted) inputs, - * independent of worker count and scheduling. */ -static int phase6b_merge_edges(cbm_gbuf_t *gbuf, deferred_edge_buf_t *worker_bufs, int worker_count, - int *edge_counts, int max_edges) { - int total_pairs = 0; - for (int w = 0; w < worker_count; w++) { - total_pairs += worker_bufs[w].count; - } - deferred_edge_t *pairs = NULL; - if (total_pairs > 0) { - pairs = malloc((size_t)total_pairs * sizeof(deferred_edge_t)); - } - if (!pairs) { - for (int w = 0; w < worker_count; w++) { - deferred_buf_free(&worker_bufs[w]); - } - return 0; - } - int n = 0; - for (int w = 0; w < worker_count; w++) { - if (worker_bufs[w].count > 0) { - memcpy(&pairs[n], worker_bufs[w].edges, - (size_t)worker_bufs[w].count * sizeof(deferred_edge_t)); - } - n += worker_bufs[w].count; - deferred_buf_free(&worker_bufs[w]); - } - if (n > 1) { - qsort(pairs, (size_t)n, sizeof(deferred_edge_t), cmp_deferred_edge_canonical); - } - - int total_edges = 0; - for (int e = 0; e < n; e++) { - deferred_edge_t *de = &pairs[e]; - if (edge_counts[de->i] >= max_edges || edge_counts[de->j] >= max_edges) { - continue; - } - char props[PROPS_BUF]; - snprintf(props, sizeof(props), "{\"score\":%.3f,\"same_file\":%s}", de->score, - de->same_file ? "true" : "false"); - cbm_gbuf_insert_edge(gbuf, de->source_id, de->target_id, "SEMANTICALLY_RELATED", props); - edge_counts[de->i]++; - edge_counts[de->j]++; - total_edges++; - } - free(pairs); - return total_edges; -} - -/* Phase 3c: export co-occurrence-enriched token vectors into the graph buffer - * so query-time lookups can use them without re-running the finalize step. */ -static void phase3c_export_token_vectors(cbm_gbuf_t *gbuf, cbm_sem_corpus_t *corpus) { - int tv_count = cbm_sem_corpus_token_count(corpus); - for (int t = 0; t < tv_count; t++) { - const cbm_sem_vec_t *vec = NULL; - float idf = 0.0F; - const char *tok = cbm_sem_corpus_token_at(corpus, t, &vec, &idf); - if (!tok || !vec || idf <= PSE_FLOW_WEIGHT) { - continue; - } - uint8_t qvec[CBM_SEM_DIM]; - for (int d = 0; d < CBM_SEM_DIM; d++) { - float clamped = vec->v[d]; - if (clamped > PSE_UNIT_POS) { - clamped = PSE_UNIT_POS; - } - if (clamped < -PSE_UNIT_POS) { - clamped = -PSE_UNIT_POS; - } - qvec[d] = (uint8_t)(int8_t)(clamped * PSE_INT8_MAX); - } - cbm_gbuf_store_token_vector(gbuf, tok, qvec, CBM_SEM_DIM, idf); - } - cbm_log_info("pass.semantic.token_vectors", "count", cbm_log_itoa(tv_count)); -} - -/* Phase 5a: generate NUM_HYPERPLANES × CBM_SEM_DIM deterministic random - * float hyperplanes seeded from XXH3 so signatures are reproducible. */ -static hyperplane_row_t *phase5a_build_hyperplanes(void) { - hyperplane_row_t *hyperplanes = malloc(sizeof(hyperplane_row_t) * NUM_HYPERPLANES); - if (!hyperplanes) { - return NULL; - } - for (int h = 0; h < NUM_HYPERPLANES; h++) { - for (int d = 0; d < CBM_RSQ_DIM; d++) { - uint64_t seed = XXH3_64bits_withSeed(&d, sizeof(d), (uint64_t)h * CBM_RSQ_DIM); - hyperplanes[h][d] = ((float)(seed & UINT32_MAX) / (float)UINT32_MAX) - PSE_ROUND_BIAS; - } - } - return hyperplanes; -} - -/* Phase 1b: decode per-function minhash/profile/API/type/deco vectors in - * parallel. Must be called after cbm_sem_ensure_ready() so the pretrained - * token map is initialized. */ -static void phase1b_decode_and_build(cbm_sem_func_t *funcs, const cbm_gbuf_node_t **node_ptrs, - const cbm_gbuf_t *gbuf, int func_count, int worker_count) { - if (func_count <= 0) { - return; - } - collect_ctx_t cc = { - .funcs = funcs, - .node_ptrs = node_ptrs, - .gbuf = gbuf, - .func_count = func_count, - }; - atomic_init(&cc.next_idx, 0); - cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false}; - cbm_parallel_for(worker_count, collect_worker, &cc, opts); -} - -/* Phase 2: tokenize each function's metadata in parallel, filling - * all_tokens[] and token_counts[]. Caller allocates the arrays. */ -static void phase2_tokenize(const cbm_gbuf_node_t **node_ptrs, cbm_gbuf_t *gbuf, char **all_tokens, - int *token_counts, int func_count, int worker_count, - CBMHashTable **pools) { - tokenize_ctx_t tc = { - .node_ptrs = node_ptrs, - .gbuf = gbuf, - .all_tokens = all_tokens, - .token_counts = token_counts, - .func_count = func_count, - .pools = pools, - }; - atomic_init(&tc.next_idx, 0); - cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false}; - cbm_parallel_for(worker_count, tokenize_worker, &tc, opts); -} - -/* Phase 4a: build per-function TF-IDF + RI vectors in parallel, producing - * int8-quantized qvecs for subsequent storage. Phase 4b runs sequentially - * to store them in gbuf because gbuf is not thread-safe. */ -static void phase4_build_and_store_vectors(cbm_gbuf_t *gbuf, cbm_sem_func_t *funcs, - char **all_tokens, int *token_counts, - cbm_sem_corpus_t *corpus, int func_count, - int worker_count) { - uint8_t *qvecs = malloc((size_t)func_count * CBM_SEM_DIM); - if (!qvecs) { - return; - } - vec_build_ctx_t vc = { - .funcs = funcs, - .all_tokens = all_tokens, - .token_counts = token_counts, - .corpus = corpus, - .qvecs = qvecs, - .func_count = func_count, - }; - atomic_init(&vc.next_idx, 0); - cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false}; - cbm_parallel_for(worker_count, vec_build_worker, &vc, opts); - for (int f = 0; f < func_count; f++) { - cbm_gbuf_store_vector(gbuf, funcs[f].node_id, &qvecs[(ptrdiff_t)f * CBM_SEM_DIM], - CBM_SEM_DIM); - } - free(qvecs); -} - -/* Phase 5: hyperplane generation → signatures → LSH bucket population. - * Returns the malloc'd signatures array and band_buckets[] via out-params; - * caller frees both. */ -static void phase5_lsh_build(cbm_sem_func_t *funcs, int func_count, int worker_count, - uint64_t **out_signatures, sem_bucket_t ***out_buckets) { - hyperplane_row_t *hyperplanes = phase5a_build_hyperplanes(); - uint64_t *signatures = calloc((size_t)func_count, sizeof(uint64_t)); - if (hyperplanes && signatures) { - sig_build_ctx_t sc = { - .funcs = funcs, - .signatures = signatures, - .hyperplanes = hyperplanes, - .func_count = func_count, - }; - atomic_init(&sc.next_idx, 0); - cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false}; - cbm_parallel_for(worker_count, sig_build_worker, &sc, opts); - } - free(hyperplanes); - - sem_bucket_t **band_buckets = calloc(SEM_LSH_BANDS, sizeof(sem_bucket_t *)); - if (band_buckets) { - for (int b = 0; b < SEM_LSH_BANDS; b++) { - band_buckets[b] = calloc(SEM_BUCKET_COUNT, sizeof(sem_bucket_t)); - } - phase5c_build_lsh_buckets(signatures, func_count, band_buckets); - } - *out_signatures = signatures; - *out_buckets = band_buckets; -} - -/* Phase 6a: score candidate pairs in parallel and collect deferred edges. */ -static void phase6a_score_candidates(cbm_sem_func_t *funcs, uint64_t *signatures, int *edge_counts, - sem_bucket_t **band_buckets, cbm_sem_config_t cfg, - deferred_edge_buf_t *worker_bufs, int func_count, - int worker_count) { - score_ctx_t sc = { - .funcs = funcs, - .signatures = signatures, - .edge_counts = edge_counts, - .cfg = cfg, - .func_count = func_count, - .band_buckets = (void *)band_buckets, - .worker_bufs = worker_bufs, - .max_workers = worker_count, - }; - atomic_init(&sc.next_idx, 0); - cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false}; - cbm_parallel_for(worker_count, score_worker, &sc, opts); -} - -/* Phase 7: free LSH bucket storage (items arrays and the per-band arrays). */ -static void free_lsh_buckets(sem_bucket_t **band_buckets) { - if (!band_buckets) { - return; - } - for (int b = 0; b < SEM_LSH_BANDS; b++) { - if (!band_buckets[b]) { - continue; - } - for (int h = 0; h < SEM_BUCKET_COUNT; h++) { - free(band_buckets[b][h].items); - } - free(band_buckets[b]); - } - free(band_buckets); -} - -/* Phases 3a/3b/3c bundled: create corpus, batch-add docs, finalize, export - * enriched token vectors to the graph buffer. Returns the new corpus, which - * the caller must cbm_sem_corpus_free() later. */ -static cbm_sem_corpus_t *run_corpus_phase(cbm_gbuf_t *gbuf, char **all_tokens, int *token_counts, - int func_count) { - CBM_PROF_START(t_phase3a); - cbm_sem_corpus_t *corpus = cbm_sem_corpus_new(); - cbm_sem_corpus_add_docs_batch(corpus, all_tokens, token_counts, func_count, CBM_SEM_MAX_TOKENS); - CBM_PROF_END_N("semantic_edges", "3a_corpus_batch", t_phase3a, func_count); - - CBM_PROF_START(t_phase3b); - cbm_sem_corpus_finalize(corpus); - CBM_PROF_END_N("semantic_edges", "3b_corpus_finalize_seq", t_phase3b, - cbm_sem_corpus_token_count(corpus)); - - CBM_PROF_START(t_phase3c); - phase3c_export_token_vectors(gbuf, corpus); - CBM_PROF_END_N("semantic_edges", "3c_token_vec_export_seq", t_phase3c, - cbm_sem_corpus_token_count(corpus)); - return corpus; -} - -/* Phases 6a/6b bundled: parallel scoring + sequential merge of deferred - * edges. Returns the number of emitted edges. */ -static int run_scoring_phase(cbm_gbuf_t *gbuf, cbm_sem_func_t *funcs, uint64_t *signatures, - sem_bucket_t **band_buckets, cbm_sem_config_t cfg, int func_count, - int worker_count) { - int *edge_counts = calloc((size_t)func_count, sizeof(int)); - deferred_edge_buf_t *worker_bufs = calloc((size_t)worker_count, sizeof(deferred_edge_buf_t)); - if (!edge_counts || !worker_bufs) { - free(edge_counts); - free(worker_bufs); - return 0; - } - for (int w = 0; w < worker_count; w++) { - deferred_buf_init(&worker_bufs[w]); - } - - CBM_PROF_START(t_phase6a); - phase6a_score_candidates(funcs, signatures, edge_counts, band_buckets, cfg, worker_bufs, - func_count, worker_count); - CBM_PROF_END_N("semantic_edges", "6a_score_parallel", t_phase6a, func_count); - - CBM_PROF_START(t_phase6b); - int total = phase6b_merge_edges(gbuf, worker_bufs, worker_count, edge_counts, cfg.max_edges); - CBM_PROF_END_N("semantic_edges", "6b_edge_merge_seq", t_phase6b, total); - - free(worker_bufs); - free(edge_counts); - return total; -} - -/* Free the per-function arrays malloc'd during phases 1b and 4a. */ -static void free_token_pool_entry(const char *key, void *value, void *ud) { - (void)key; /* key == value: one owned string per unique token */ - (void)ud; - free(value); -} - -/* all_tokens slots BORROW their strings from the per-worker intern pools; - * the pools own exactly one copy per unique token per worker. */ -static void free_funcs_and_tokens(cbm_sem_func_t *funcs, int func_count, char **all_tokens, - const int *token_counts, CBMHashTable **pools, int worker_count) { - (void)token_counts; - for (int f = 0; f < func_count; f++) { - free(funcs[f].tfidf_indices); - free(funcs[f].tfidf_weights); - } - free(all_tokens); - free(funcs); - if (pools) { - for (int w = 0; w < worker_count; w++) { - if (pools[w]) { - cbm_ht_foreach(pools[w], free_token_pool_entry, NULL); - cbm_ht_free(pools[w]); - } - } - free(pools); - } -} - -/* ── Pass entry point ────────────────────────────────────────────── */ - -int cbm_pipeline_pass_semantic_edges(cbm_pipeline_ctx_t *ctx) { - /* Controlled by pipeline mode (moderate/full), not env var */ - cbm_log_info("pass.start", "pass", "semantic_edges"); - - cbm_gbuf_t *gbuf = ctx->gbuf; - cbm_sem_config_t cfg = cbm_sem_get_config(); - - CBM_PROF_START(t_phase1a); - cbm_sem_func_t *funcs = NULL; - const cbm_gbuf_node_t **node_ptrs = NULL; - int func_count = phase1_scan_functions(gbuf, &funcs, &node_ptrs); - CBM_PROF_END_N("semantic_edges", "1a_scan_seq", t_phase1a, func_count); - - /* Phase 1b: Decode minhash + profile + build api/type/deco vectors (PARALLEL). */ - cbm_sem_ensure_ready(); - CBM_PROF_START(t_phase1b); - phase1b_decode_and_build(funcs, node_ptrs, gbuf, func_count, cbm_default_worker_count(false)); - CBM_PROF_END_N("semantic_edges", "1b_decode_build_parallel", t_phase1b, func_count); - cbm_log_info("pass.semantic.collected", "functions", cbm_log_itoa(func_count)); - - if (func_count < PSE_MIN_FUNCS_FOR_PAIR) { - free(funcs); - free(node_ptrs); - cbm_log_info("pass.done", "pass", "semantic_edges", "edges", "0"); - return 0; - } - - /* Phase 2: Tokenize all nodes (PARALLEL) */ - int worker_count = cbm_default_worker_count(false); - char **all_tokens = malloc((size_t)func_count * sizeof(char *) * CBM_SEM_MAX_TOKENS); - int *token_counts = calloc((size_t)func_count, sizeof(int)); - - CBM_PROF_START(t_phase2); - CBMHashTable **token_pools = calloc((size_t)worker_count, sizeof(CBMHashTable *)); - if (token_pools) { - for (int w = 0; w < worker_count; w++) { - token_pools[w] = cbm_ht_create(CBM_SZ_1K); - } - } - phase2_tokenize(node_ptrs, gbuf, all_tokens, token_counts, func_count, worker_count, - token_pools); - CBM_PROF_END_N("semantic_edges", "2_tokenize_parallel", t_phase2, func_count); - free(node_ptrs); - - /* Phase 3: Build corpus (batch add), finalize, export enriched token vectors. */ - cbm_sem_corpus_t *corpus = run_corpus_phase(gbuf, all_tokens, token_counts, func_count); - - /* Phase 4: Build per-function TF-IDF + RI vectors (PARALLEL) and store them. */ - CBM_PROF_START(t_phase4); - phase4_build_and_store_vectors(gbuf, funcs, all_tokens, token_counts, corpus, func_count, - worker_count); - CBM_PROF_END_N("semantic_edges", "4_build_and_store_vec", t_phase4, func_count); - - cbm_log_info("pass.semantic.vectors_stored", "count", cbm_log_itoa(func_count)); - - /* Phase 5: LSH hyperplanes → signatures → buckets. */ - CBM_PROF_START(t_phase5); - uint64_t *signatures = NULL; - sem_bucket_t **band_buckets = NULL; - phase5_lsh_build(funcs, func_count, worker_count, &signatures, &band_buckets); - CBM_PROF_END_N("semantic_edges", "5_lsh_build", t_phase5, func_count); - - cbm_log_info("pass.semantic.lsh_built", "functions", cbm_log_itoa(func_count), "bands", - cbm_log_itoa(SEM_LSH_BANDS)); - - /* Phase 6: Parallel scoring + sequential edge merge. */ - int total_edges = - run_scoring_phase(gbuf, funcs, signatures, band_buckets, cfg, func_count, worker_count); - - /* Phase 7: Cleanup */ - CBM_PROF_START(t_phase7); - free_lsh_buckets(band_buckets); - free(signatures); - cbm_log_info("pass.done", "pass", "semantic_edges", "edges", cbm_log_itoa(total_edges)); - free_funcs_and_tokens(funcs, func_count, all_tokens, token_counts, token_pools, worker_count); - free(token_counts); - cbm_sem_corpus_free(corpus); - CBM_PROF_END("semantic_edges", "7_cleanup", t_phase7); - - return 0; -} diff --git a/src/pipeline/pass_semantic_edges.zig b/src/pipeline/pass_semantic_edges.zig new file mode 100644 index 00000000..c5542bf3 --- /dev/null +++ b/src/pipeline/pass_semantic_edges.zig @@ -0,0 +1,1242 @@ +//! pass_semantic_edges.zig — Emit SEMANTICALLY_RELATED edges from combined +//! algorithmic embeddings (11 signals, zero external dependencies). +//! +//! Runs as a post-pass after pass_similarity. Reads all Function/Method +//! nodes from the graph buffer, computes TF-IDF + Random Indexing + API +//! signatures + type/decorator vectors + AST profile, builds LSH index, +//! scores candidate pairs, applies graph diffusion, emits edges. +//! +//! Runs in moderate and full modes (not fast). Controlled by pipeline mode. + +const std = @import("std"); +const c = @import("c"); +const interop = @import("../foundation/interop.zig"); +const util = @import("util.zig"); +const fp = @import("../semantic/fp.zig"); + +const XxHash3 = std.hash.XxHash3; +const AtomicInt = std.atomic.Value(c_int); + +const CStr = util.CStr; +const MutStr = util.MutStr; +const ix = util.ix; +const streq = util.streq; +const Gbuf = c.cbm_gbuf_t; +const Node = c.cbm_gbuf_node_t; +const Edge = c.cbm_gbuf_edge_t; +const Vec = c.cbm_sem_vec_t; +const Func = c.cbm_sem_func_t; +const HashTable = c.CBMHashTable; +const NodePtrs = [*c][*c]const Node; + +// ── Helpers ───────────────────────────────────────────────────────── + +const PROPS_BUF = 512; +const MAX_FUNCS_INIT = 4096; +const GROW = 2; +const MAX_CALLEES = 64; +const PSE_FP_PREFIX_LEN = 6; // strlen("\"fp\":\"") +const PSE_MIN_FUNCS_FOR_PAIR = 2; +const PSE_CHUNK = 64; + +const SEM_DIM: comptime_int = c.CBM_SEM_DIM; +const RSQ_DIM: comptime_int = c.CBM_RSQ_DIM; +const MAX_TOKENS: comptime_int = c.CBM_SEM_MAX_TOKENS; + +/// Scalar weight constants used in scoreWorker and related helpers. +const PSE_UNIT_POS: f32 = 1.0; +const PSE_INT8_MAX: f32 = 127.0; +const PSE_ROUND_BIAS: f32 = 0.5; +const PSE_FLOW_WEIGHT: f32 = 0.01; + +fn parallelOpts(worker_count: c_int) c.cbm_parallel_for_opts_t { + return .{ .max_workers = worker_count, .force_pthreads = false }; +} + +// ── Deferred edge buffer (thread-local, merged after parallel pass) ── + +const DeferredEdge = extern struct { + source_id: i64, + target_id: i64, + score: f32, + same_file: bool, + /// Canonical admission keys (determinism): func index of the discovering + /// side, its candidate rank, and the partner's func index. The sequential + /// admission pass replays pairs in (i, c) order so which pairs win the + /// per-node max_edges budget no longer depends on worker scheduling. + i: c_int, + j: c_int, + c: c_int, +}; + +const DeferredEdgeBuf = extern struct { + edges: [*c]DeferredEdge, + count: c_int, + cap: c_int, +}; + +fn deferredBufInit(buf: *DeferredEdgeBuf) void { + buf.edges = null; + buf.count = 0; + buf.cap = 0; +} + +fn deferredBufPush(buf: *DeferredEdgeBuf, edge: DeferredEdge) void { + if (buf.count >= buf.cap) { + const nc: c_int = if (buf.cap < c.CBM_SZ_256) c.CBM_SZ_256 else buf.cap * GROW; + const grown = interop.reallocN(DeferredEdge, buf.edges, ix(nc)); + if (grown == null) return; + buf.edges = grown; + buf.cap = nc; + } + buf.edges[ix(buf.count)] = edge; + buf.count += 1; +} + +fn deferredBufFree(buf: *DeferredEdgeBuf) void { + c.free(buf.edges); + buf.edges = null; + buf.count = 0; + buf.cap = 0; +} + +// ── Technique 2: Code pattern vocabulary injection ────────────────── +// Inject semantic tokens based on detected code patterns. +// This bridges the vocabulary gap for abstract concepts like "error handling". + +/// Append the tokens in `texts` while there is capacity. A token whose strdup +/// fails is dropped. +fn pushPatternTokens(tokens: [*]MutStr, count_in: c_int, max_tokens: c_int, texts: []const [*:0]const u8) c_int { + var count = count_in; + for (texts) |text| { + if (count >= max_tokens) continue; + tokens[ix(count)] = c.strdup(text); + if (tokens[ix(count)] != null) count += 1; + } + return count; +} + +/// True if `s` contains any of the substrings in `needles`. +fn hasAny(s: CStr, needles: []const [*:0]const u8) bool { + if (s == null) return false; + for (needles) |needle| { + if (c.strstr(s, needle) != null) return true; + } + return false; +} + +/// Inject tokens derived from body-text patterns (try/catch, raise, log). +fn injectBodyPatternTokens(bt: CStr, tokens: [*]MutStr, count_in: c_int, max_tokens: c_int) c_int { + var count = count_in; + if (bt == null) return count; + if (hasAny(bt, &.{ "except", "catch", "rescue" })) { + count = pushPatternTokens(tokens, count, max_tokens, &.{ "error", "handling", "exception" }); + } + if (hasAny(bt, &.{ "raise", "throw" })) { + count = pushPatternTokens(tokens, count, max_tokens, &.{ "error", "exception", "throw" }); + } + if (hasAny(bt, &.{ "logger", "logging", "log_" })) { + count = pushPatternTokens(tokens, count, max_tokens, &.{ "logging", "log" }); + } + return count; +} + +/// Inject tokens for one CALLS-target name based on keyword groups. +fn injectCalleeTokens(name: CStr, tokens: [*]MutStr, count_in: c_int, max_tokens: c_int) c_int { + var count = count_in; + if (hasAny(name, &.{ "log", "Log", "warn", "debug", "info" })) { + count = pushPatternTokens(tokens, count, max_tokens, &.{ "logging", "log" }); + } + if (hasAny(name, &.{ "Error", "error", "Errorf", "panic" })) { + count = pushPatternTokens(tokens, count, max_tokens, &.{ "error", "handling" }); + } + if (hasAny(name, &.{ "open", "read", "write", "close", "Open", "Read" })) { + count = pushPatternTokens(tokens, count, max_tokens, &.{ "io", "file" }); + } + return count; +} + +/// Walk the outbound CALLS edges of n and inject tokens for each target. +fn injectCallsPatternTokens(n: *const Node, gbuf: ?*const Gbuf, tokens: [*]MutStr, count_in: c_int, max_tokens: c_int) c_int { + var count = count_in; + if (gbuf == null) return count; + var neighbor_count: c_int = 0; + const names = collectSortedCallNeighbors(gbuf, n.id, true, &neighbor_count); + var i: usize = 0; + while (i < neighbor_count and count < max_tokens) : (i += 1) { + count = injectCalleeTokens(names[i], tokens, count, max_tokens); + } + c.free(@ptrCast(names)); + return count; +} + +/// Inject tokens from decorator annotations (@route, @middleware, @pytest.*). +fn injectDecoratorTokens(decs: CStr, tokens: [*]MutStr, count_in: c_int, max_tokens: c_int) c_int { + var count = count_in; + if (decs == null) return count; + if (hasAny(decs, &.{ "route", "Route", "app." })) { + count = pushPatternTokens(tokens, count, max_tokens, &.{ "routing", "endpoint", "handler" }); + } + if (hasAny(decs, &.{ "middleware", "Middleware" })) { + count = pushPatternTokens(tokens, count, max_tokens, &.{"middleware"}); + } + if (hasAny(decs, &.{ "test", "Test", "pytest" })) { + count = pushPatternTokens(tokens, count, max_tokens, &.{ "test", "testing" }); + } + return count; +} + +/// Inject tokens keyed off the node's own name (test_*, *Handler, validator). +fn injectNamePatternTokens(name: CStr, tokens: [*]MutStr, count_in: c_int, max_tokens: c_int) c_int { + var count = count_in; + if (name == null) return count; + if (hasAny(name, &.{ "test_", "Test" })) { + count = pushPatternTokens(tokens, count, max_tokens, &.{ "test", "testing" }); + } + if (hasAny(name, &.{ "middleware", "Middleware" })) { + count = pushPatternTokens(tokens, count, max_tokens, &.{"middleware"}); + } + if (hasAny(name, &.{ "handler", "Handler" })) { + count = pushPatternTokens(tokens, count, max_tokens, &.{"handler"}); + } + if (hasAny(name, &.{ "validator", "Validator", "validate", "Validate" })) { + count = pushPatternTokens(tokens, count, max_tokens, &.{"validation"}); + } + return count; +} + +fn injectPatternTokens(n_c: ?*const Node, gbuf: ?*const Gbuf, tokens: [*]MutStr, count_in: c_int, max_tokens: c_int) c_int { + var count = count_in; + const n = n_c orelse return count; + if (count >= max_tokens) return count; + + var bt_buf: [c.CBM_SZ_512]u8 = undefined; + const bt: CStr = if (n.properties_json != null) + jsonStrValue(n.properties_json, "bt", &bt_buf) + else + null; + var dec_buf: [c.CBM_SZ_256]u8 = undefined; + const decs: CStr = if (n.properties_json != null) + jsonStrValue(n.properties_json, "decorators", &dec_buf) + else + null; + + count = injectBodyPatternTokens(bt, tokens, count, max_tokens); + count = injectCallsPatternTokens(n, gbuf, tokens, count, max_tokens); + count = injectDecoratorTokens(decs, tokens, count, max_tokens); + count = injectNamePatternTokens(n.name, tokens, count, max_tokens); + return count; +} + +/// Extract a JSON string value by key (simple strstr-based, no full parse). +fn jsonStrValue(json: CStr, key: CStr, buf: []u8) CStr { + if (json == null or key == null) return null; + var search: [c.CBM_SZ_64]u8 = undefined; + _ = c.snprintf(&search, search.len, "\"%s\":\"", key); + var start: CStr = c.strstr(json, &search); + if (start == null) return null; + start += c.strlen(&search); + const end: CStr = c.strchr(start, '"'); + if (end == null) return null; + var len = util.ptrDiff(end, start); + if (len >= buf.len) len = buf.len - 1; + @memcpy(buf[0..len], start[0..len]); + buf[len] = 0; + return buf.ptr; +} + +/// Extract a JSON array of strings by key. Returns count. +fn jsonStrArray(json: CStr, key: CStr, out: []MutStr) usize { + if (json == null or key == null) return 0; + var search: [c.CBM_SZ_64]u8 = undefined; + _ = c.snprintf(&search, search.len, "\"%s\":[", key); + var start: CStr = c.strstr(json, &search); + if (start == null) return 0; + start += c.strlen(&search); + var count: usize = 0; + while (start[0] != 0 and start[0] != ']' and count < out.len) { + if (start[0] == '"') { + start += 1; + const end: CStr = c.strchr(start, '"'); + if (end == null) break; + const len = util.ptrDiff(end, start); + out[count] = @ptrCast(c.malloc(len + 1)); + @memcpy(out[count][0..len], start[0..len]); + out[count][len] = 0; + count += 1; + start = end + 1; + } else { + start += 1; + } + } + return count; +} + +// ── Tokenize node metadata ────────────────────────────────────────── + +/// Tokenize a single string field keyed out of node.properties_json, if +/// present. Returns the new count after appending any tokens. +fn tokenizeJsonStringField(json: CStr, key: CStr, tokens: [*]MutStr, count_in: c_int, max_tokens: c_int) c_int { + var count = count_in; + if (count >= max_tokens) return count; + var buf: [c.CBM_SZ_512]u8 = undefined; + if (jsonStrValue(json, key, &buf) == null) return count; + count += c.cbm_sem_tokenize(&buf, tokens + ix(count), max_tokens - count); + return count; +} + +/// Tokenize a JSON array field (e.g. "param_names", "decorators"). +fn tokenizeJsonArrayField(json: CStr, key: CStr, tokens: [*]MutStr, count_in: c_int, max_tokens: c_int) c_int { + var count = count_in; + if (count >= max_tokens) return count; + var arr: [c.CBM_SZ_16]MutStr = undefined; + const n = jsonStrArray(json, key, &arr); + for (arr[0..n]) |item| { + if (count < max_tokens) { + count += c.cbm_sem_tokenize(item, tokens + ix(count), max_tokens - count); + } + c.free(item); + } + return count; +} + +fn cmpNamePtr(pa: ?*const anyopaque, pb: ?*const anyopaque) callconv(.c) c_int { + const a: *const CStr = @ptrCast(@alignCast(pa)); + const b: *const CStr = @ptrCast(@alignCast(pb)); + return c.strcmp(a.*, b.*); +} + +/// Collect the CALLS-neighbor names of `node_id`, SORTED by name. The edge +/// arrays are in insertion order — which under parallel extraction varies run +/// to run — and both consumers truncate (MAX_CALLEES / max_tokens), so an +/// unstable order changed WHICH neighbors contribute to the semantic vectors +/// and flickered near-threshold SEMANTICALLY_RELATED edges (determinism). +/// Returns a malloc'd array of borrowed name pointers; caller frees the array. +fn collectSortedCallNeighbors(gbuf: ?*const Gbuf, node_id: i64, outbound: bool, out_n: *c_int) [*c]CStr { + out_n.* = 0; + var edges: [*c][*c]const Edge = null; + var ec: c_int = 0; + const rc = if (outbound) + c.cbm_gbuf_find_edges_by_source_type(gbuf, node_id, "CALLS", &edges, &ec) + else + c.cbm_gbuf_find_edges_by_target_type(gbuf, node_id, "CALLS", &edges, &ec); + if (rc != 0 or ec <= 0) return null; + const names = interop.mallocN(CStr, ix(ec)); + if (names == null) return null; + var n: usize = 0; + for (0..ix(ec)) |e| { + const id = if (outbound) edges[e].*.target_id else edges[e].*.source_id; + const neighbor: ?*const Node = c.cbm_gbuf_find_by_id(gbuf, id); + if (neighbor) |nb| { + if (nb.name != null) { + names[n] = nb.name; + n += 1; + } + } + } + if (n > 1) { + c.qsort(@ptrCast(names), n, @sizeOf(CStr), cmpNamePtr); + } + out_n.* = @intCast(n); + return names; +} + +/// Walk the CALLS edges rooted at n (either outbound or inbound depending on +/// `outbound`) and tokenize the names of the target/source nodes. Caller-side +/// caps via max_tokens and MAX_CALLEES. +fn tokenizeCallNeighbors(n: *const Node, gbuf: ?*const Gbuf, outbound: bool, tokens: [*]MutStr, count_in: c_int, max_tokens: c_int) c_int { + var count = count_in; + if (gbuf == null or count >= max_tokens) return count; + var nn: c_int = 0; + const names = collectSortedCallNeighbors(gbuf, n.id, outbound, &nn); + if (names == null) return count; + var e: usize = 0; + while (e < nn and e < MAX_CALLEES and count < max_tokens) : (e += 1) { + count += c.cbm_sem_tokenize(names[e], tokens + ix(count), max_tokens - count); + } + c.free(@ptrCast(names)); + return count; +} + +fn tokenizeNode(n: *const Node, gbuf: ?*const Gbuf, tokens: [*]MutStr, max_tokens: c_int) c_int { + var count: c_int = 0; + count += c.cbm_sem_tokenize(n.name, tokens + ix(count), max_tokens - count); + if (n.qualified_name != null and count < max_tokens) { + count += c.cbm_sem_tokenize(n.qualified_name, tokens + ix(count), max_tokens - count); + } + if (n.file_path != null and count < max_tokens) { + count += c.cbm_sem_tokenize(n.file_path, tokens + ix(count), max_tokens - count); + } + if (n.properties_json != null) { + const props = n.properties_json; + count = tokenizeJsonStringField(props, "signature", tokens, count, max_tokens); + count = tokenizeJsonStringField(props, "return_type", tokens, count, max_tokens); + count = tokenizeJsonStringField(props, "docstring", tokens, count, max_tokens); + count = tokenizeJsonArrayField(props, "param_names", tokens, count, max_tokens); + count = tokenizeJsonArrayField(props, "param_types", tokens, count, max_tokens); + count = tokenizeJsonArrayField(props, "decorators", tokens, count, max_tokens); + count = tokenizeJsonStringField(props, "bt", tokens, count, max_tokens); + } + count = tokenizeCallNeighbors(n, gbuf, true, tokens, count, max_tokens); + + // Caller names: what CALLS this function (contextual vocabulary). + // Functions called by error handlers inherit "error" context. + count = tokenizeCallNeighbors(n, gbuf, false, tokens, count, max_tokens); + return count; +} + +// ── Build per-function semantic data ──────────────────────────────── + +/// Add the random-index vector of `token` to `out`. +fn addRandomIndex(out: *Vec, token: CStr) void { + var ri: Vec = undefined; + c.cbm_sem_random_index(token, &ri); + c.cbm_sem_vec_add_scaled(out, &ri, PSE_UNIT_POS); +} + +fn buildApiVec(gbuf: ?*const Gbuf, node_id: i64, out: *Vec) void { + @memset(&out.v, 0.0); + // Sorted neighbors: stable MAX_CALLEES subset + stable float-accumulation + // order (see collectSortedCallNeighbors). + var n: c_int = 0; + const names = collectSortedCallNeighbors(gbuf, node_id, true, &n); + if (names == null) return; + var i: usize = 0; + while (i < n and i < MAX_CALLEES) : (i += 1) { + addRandomIndex(out, names[i]); + } + c.free(@ptrCast(names)); + c.cbm_sem_normalize(out); +} + +fn buildTypeVec(props_json: CStr, out: *Vec) void { + @memset(&out.v, 0.0); + if (props_json == null) return; + // Extract param_types and return_type + var rt_buf: [c.CBM_SZ_128]u8 = undefined; + if (jsonStrValue(props_json, "return_type", &rt_buf) != null) { + addRandomIndex(out, &rt_buf); + } + var ptypes: [c.CBM_SZ_16]MutStr = undefined; + const pt_count = jsonStrArray(props_json, "param_types", &ptypes); + for (ptypes[0..pt_count]) |ptype| { + addRandomIndex(out, ptype); + c.free(ptype); + } + c.cbm_sem_normalize(out); +} + +fn buildDecoVec(props_json: CStr, out: *Vec) void { + @memset(&out.v, 0.0); + if (props_json == null) return; + var decos: [c.CBM_SZ_16]MutStr = undefined; + const dc = jsonStrArray(props_json, "decorators", &decos); + for (decos[0..dc]) |deco| { + addRandomIndex(out, deco); + c.free(deco); + } + c.cbm_sem_normalize(out); +} + +fn decodeStructProfile(props_json: CStr, out: *[c.CBM_AST_PROFILE_DIMS]f32) void { + @memset(out, 0.0); + if (props_json == null) return; + var sp_buf: [c.CBM_AST_PROFILE_BUF]u8 = undefined; + if (jsonStrValue(props_json, "sp", &sp_buf) != null) { + var profile: c.cbm_ast_profile_t = undefined; + if (c.cbm_ast_profile_from_str(&sp_buf, &profile)) { + c.cbm_ast_profile_to_vector(&profile, out); + } + } +} + +fn decodeMinhash(props_json: CStr, func: *Func) void { + func.has_minhash = false; + if (props_json == null) return; + const fp_key: CStr = c.strstr(props_json, "\"fp\":\""); + if (fp_key == null) return; + const hex: CStr = fp_key + PSE_FP_PREFIX_LEN; // strlen("\"fp\":\"") + const end: CStr = c.strchr(hex, '"'); + if (end == null or util.ptrDiff(end, hex) != c.CBM_MINHASH_HEX_LEN) return; + var hex_buf: [c.CBM_MINHASH_HEX_BUF]u8 = undefined; + @memcpy(hex_buf[0..c.CBM_MINHASH_HEX_LEN], hex[0..c.CBM_MINHASH_HEX_LEN]); + hex_buf[c.CBM_MINHASH_HEX_LEN] = 0; + var mh: c.cbm_minhash_t = undefined; + if (c.cbm_minhash_from_hex(&hex_buf, &mh)) { + func.minhash = mh.values; + func.has_minhash = true; + } +} + +// ── Parallel Phase 2: Tokenize nodes ──────────────────────────────── + +const TokenizeCtx = struct { + node_ptrs: NodePtrs, // node pointer per function index + gbuf: ?*Gbuf, // read-only during tokenization + all_tokens: [*c]MutStr, // output: all_tokens[f * MAX + t] + token_counts: [*c]c_int, // output: token count per function + func_count: c_int, + next_idx: AtomicInt, + /// Per-worker token intern pools (key==value==the one owned strdup): + /// identical tokens ("xfs", "error", ...) recur across hundreds of + /// thousands of functions; per-func strdups made all_tokens hold every + /// instance. Interned, it holds at most workers x unique tokens. + pools: [*c]?*HashTable, +}; + +fn tokenizeWorker(worker_id: c_int, ctx_ptr: ?*anyopaque) callconv(.c) void { + const tc: *TokenizeCtx = @ptrCast(@alignCast(ctx_ptr)); + while (true) { + const f = tc.next_idx.fetchAdd(1, .monotonic); + if (f >= tc.func_count) break; + + const n: *const Node = tc.node_ptrs[ix(f)]; + // Write directly into the shared buffer slice for this function — the + // strdup'd tokens are owned by all_tokens[] from the moment they land + // in this slot. + const dst: [*]MutStr = @as([*]MutStr, tc.all_tokens) + ix(f) * MAX_TOKENS; + var count = tokenizeNode(n, tc.gbuf, dst, MAX_TOKENS); + count = injectPatternTokens(n, tc.gbuf, dst, count, MAX_TOKENS); + if (tc.pools != null) { + if (tc.pools[ix(worker_id)]) |pool| { + for (dst[0..ix(count)]) |*token| { + const canon: MutStr = @ptrCast(c.cbm_ht_get(pool, token.*)); + if (canon != null) { + c.free(token.*); + token.* = canon; + } else { + _ = c.cbm_ht_set(pool, token.*, token.*); // key borrows the value + } + } + } + } + tc.token_counts[ix(f)] = count; + } +} + +// ── Parallel Phase 4: Build per-function vectors ──────────────────── + +const VecBuildCtx = struct { + funcs: [*c]Func, + all_tokens: [*c]MutStr, + token_counts: [*c]c_int, + corpus: ?*c.cbm_sem_corpus_t, + qvecs: [*]u8, // output: pre-quantized int8 vectors [func_count * CBM_SEM_DIM] + func_count: c_int, + next_idx: AtomicInt, +}; + +/// (uint8_t)(int8_t)(clamp(v, -1, 1) * 127): the int8 quantization stored in +/// the graph buffer. +inline fn quantizeUnit(v_in: f32) u8 { + var v = v_in; + if (v > PSE_UNIT_POS) v = PSE_UNIT_POS; + if (v < -PSE_UNIT_POS) v = -PSE_UNIT_POS; + return @bitCast(@as(i8, @intFromFloat(v * PSE_INT8_MAX))); +} + +fn vecBuildWorker(worker_id: c_int, ctx_ptr: ?*anyopaque) callconv(.c) void { + _ = worker_id; + const vc: *VecBuildCtx = @ptrCast(@alignCast(ctx_ptr)); + while (true) { + const f_c = vc.next_idx.fetchAdd(1, .monotonic); + if (f_c >= vc.func_count) break; + const f = ix(f_c); + const func = util.at(vc.funcs, f); + + const tc = ix(vc.token_counts[f]); + const tokens: [*]MutStr = @as([*]MutStr, vc.all_tokens) + f * MAX_TOKENS; + + // TF-IDF weights + const indices = interop.mallocN(c_int, tc); + const weights = interop.mallocN(f32, tc); + var tfidf_len: usize = 0; + for (0..tc) |t| { + const idf = c.cbm_sem_corpus_idf(vc.corpus, tokens[t]); + if (idf > 0.0) { + indices[tfidf_len] = @intCast(t); + weights[tfidf_len] = idf; + tfidf_len += 1; + } + } + func.tfidf_indices = indices; + func.tfidf_weights = weights; + func.tfidf_len = @intCast(tfidf_len); + + // RI vector: sum of enriched token vectors weighted by IDF. Built in a + // LOCAL dense buffer; only the quantized code is retained per func + // (the resident dense floats were ~9.4 GB on the kernel). + var ri_dense: Vec = undefined; + @memset(&ri_dense.v, 0.0); + for (0..tc) |t| { + const ri = c.cbm_sem_corpus_ri_vec(vc.corpus, tokens[t]); + if (ri != null) { + const idf = c.cbm_sem_corpus_idf(vc.corpus, tokens[t]); + c.cbm_sem_vec_add_scaled(&ri_dense, ri, idf); + } + } + c.cbm_sem_normalize(&ri_dense); + c.cbm_rsq_encode(&ri_dense.v, &func.ri_code); + + // Int8 quantize into pre-allocated output array (parallel-safe) + const qv = vc.qvecs[f * SEM_DIM ..][0..SEM_DIM]; + for (qv, ri_dense.v) |*q, v| q.* = quantizeUnit(v); + } +} + +// ── Parallel Phase 5: LSH signatures ──────────────────────────────── + +const NUM_HYPERPLANES = 64; +const SEM_LSH_BANDS = 16; +const SEM_LSH_ROWS = 4; +const SEM_BUCKET_COUNT = 65536; +const SEM_BUCKET_MASK = 65535; +const SEM_BUCKET_CAP_INIT = 16; +const SEM_MAX_CANDIDATES = 200; + +/// Row of a hyperplane matrix in the ROTATED (quantized) basis: LSH only +/// needs signs of dots against random directions, and a random direction in +/// the rotated basis is as random as one in the original — so signatures are +/// computed from dequantized codes without keeping dense originals. +const HyperplaneRow = [RSQ_DIM]f32; + +const SigBuildCtx = struct { + funcs: [*c]Func, + signatures: [*]u64, + hyperplanes: [*]HyperplaneRow, + func_count: c_int, + next_idx: AtomicInt, +}; + +fn sigBuildWorker(worker_id: c_int, ctx_ptr: ?*anyopaque) callconv(.c) void { + _ = worker_id; + const sc: *SigBuildCtx = @ptrCast(@alignCast(ctx_ptr)); + while (true) { + const f = sc.next_idx.fetchAdd(1, .monotonic); + if (f >= sc.func_count) break; + + var dec: [RSQ_DIM]f32 = undefined; + c.cbm_rsq_decode(&util.at(sc.funcs, ix(f)).ri_code, &dec); + var sig: u64 = 0; + for (0..NUM_HYPERPLANES) |h| { + var dot: f32 = 0.0; + for (dec, sc.hyperplanes[h]) |d, plane| { + dot = fp.mulAdd(f32, d, plane, dot); + } + if (dot > 0.0) sig |= @as(u64, 1) << @intCast(h); + } + sc.signatures[ix(f)] = sig; + } +} + +// ── Parallel Phase 6: Score candidates + collect edges ────────────── + +const SemBucket = extern struct { + items: [*c]c_int, + count: c_int, + cap: c_int, +}; + +const ScoreCtx = struct { + funcs: [*c]Func, + signatures: [*c]u64, + edge_counts: [*c]c_int, // budget applied sequentially in phase6b (determinism) + cfg: c.cbm_sem_config_t, + func_count: c_int, + + /// LSH buckets (read-only during scoring) + band_buckets: [*c][*c]SemBucket, + + /// Per-worker edge buffer + worker_bufs: [*c]DeferredEdgeBuf, + max_workers: c_int, + next_idx: AtomicInt, +}; + +const SCORE_SEEN_CAP = 8192; +const SCORE_SEEN_MASK = 8191; +const SCORE_SEEN_EMPTY = -1; + +/// Check whether `j` has already been recorded in the open-addressed `seen` +/// set for this function; insert it if not. Returns true when the insertion +/// was fresh (caller should add to candidates). +fn scoreSeenInsert(seen: *[SCORE_SEEN_CAP]c_int, j: c_int) bool { + const slot: u32 = @as(u32, @bitCast(j)) & SCORE_SEEN_MASK; + for (0..SCORE_SEEN_CAP) |p| { + const idx = (slot +% @as(u32, @intCast(p))) & SCORE_SEEN_MASK; + if (seen[idx] == SCORE_SEEN_EMPTY) { + seen[idx] = j; + return true; + } + if (seen[idx] == j) return false; + } + return false; +} + +/// The LSH bucket index of a signature in band `b`. +fn bandBucketIndex(signature: u64, b: usize) usize { + const shift: u6 = @intCast(b * SEM_LSH_ROWS); + const band_val: u32 = @truncate((signature >> shift) & ((@as(u64, 1) << SEM_LSH_ROWS) - 1)); + const bh = XxHash3.hash(b, std.mem.asBytes(&band_val)); + return @intCast(bh & SEM_BUCKET_MASK); +} + +/// Collect the unique candidate function indices for node `i` by iterating +/// every LSH band and merging bucket members via `seen[]`. Returns the +/// populated candidate count. +fn scoreCollectCandidates(sc: *ScoreCtx, i: c_int, seen: *[SCORE_SEEN_CAP]c_int, candidates: []c_int) usize { + var cand_count: usize = 0; + var b: usize = 0; + while (b < SEM_LSH_BANDS and cand_count < candidates.len) : (b += 1) { + const bucket = &sc.band_buckets[b][bandBucketIndex(sc.signatures[ix(i)], b)]; + const bcount = bucket.count; + const bitems = bucket.items; + if (bcount > SEM_MAX_CANDIDATES) continue; + var k: usize = 0; + while (k < bcount and cand_count < candidates.len) : (k += 1) { + const j = bitems[k]; + if (j <= i) continue; + if (scoreSeenInsert(seen, j)) { + candidates[cand_count] = j; + cand_count += 1; + } + } + } + return cand_count; +} + +/// Score one candidate pair (i, j) and push a deferred candidate edge if the +/// score passes the threshold. ADMISSION (the per-node max_edges budget) is +/// deliberately NOT decided here: the old check-then-increment on shared +/// atomic counts made the admitted edge SET depend on worker scheduling — +/// multi-threaded runs lost edges vs single-threaded and differed run-to-run +/// (repro_parallel_edge_determinism). Scoring is pure math and stays parallel; +/// the budget is applied afterwards in one sequential pass over the pairs in +/// canonical (i, candidate-rank) order. +fn scoreTryEmit(sc: *ScoreCtx, i: c_int, j: c_int, rank: c_int, my_buf: *DeferredEdgeBuf) void { + const a = util.at(sc.funcs, ix(i)); + const b = util.at(sc.funcs, ix(j)); + if (c.strcmp(a.file_ext, b.file_ext) != 0) return; + const score = c.cbm_sem_combined_score(a, b, &sc.cfg); + if (score < sc.cfg.threshold) return; + const same_file = a.file_path != null and b.file_path != null and + streq(a.file_path, b.file_path); + deferredBufPush(my_buf, .{ + .source_id = a.node_id, + .target_id = b.node_id, + .score = score, + .same_file = same_file, + .i = i, + .j = j, + .c = rank, + }); +} + +fn scoreWorker(worker_id: c_int, ctx_ptr: ?*anyopaque) callconv(.c) void { + const sc: *ScoreCtx = @ptrCast(@alignCast(ctx_ptr)); + const my_buf = util.at(sc.worker_bufs, ix(worker_id)); + + while (true) { + const i = sc.next_idx.fetchAdd(1, .monotonic); + if (i >= sc.func_count) break; + var seen: [SCORE_SEEN_CAP]c_int = @splat(SCORE_SEEN_EMPTY); + var candidates: [SEM_MAX_CANDIDATES]c_int = undefined; + const cand_count = scoreCollectCandidates(sc, i, &seen, &candidates); + for (candidates[0..cand_count], 0..) |j, rank| { + scoreTryEmit(sc, i, j, @intCast(rank), my_buf); + } + } +} + +// ── Parallel Phase 1b: decode minhash/profile + build per-func vectors ── + +const CollectCtx = struct { + funcs: [*c]Func, + node_ptrs: NodePtrs, + gbuf: ?*const Gbuf, + func_count: c_int, + next_idx: AtomicInt, +}; + +fn collectWorker(worker_id: c_int, ctx_ptr: ?*anyopaque) callconv(.c) void { + _ = worker_id; + const cc: *CollectCtx = @ptrCast(@alignCast(ctx_ptr)); + while (true) { + const f = cc.next_idx.fetchAdd(PSE_CHUNK, .monotonic); + if (f >= cc.func_count) break; + var end = f + PSE_CHUNK; + if (end > cc.func_count) end = cc.func_count; + for (ix(f)..ix(end)) |i| { + const n: *const Node = cc.node_ptrs[i]; + const func = util.at(cc.funcs, i); + decodeMinhash(n.properties_json, func); + decodeStructProfile(n.properties_json, &func.struct_profile); + var tmp_vec: Vec = undefined; + buildApiVec(cc.gbuf, n.id, &tmp_vec); + c.cbm_rsq_encode(&tmp_vec.v, &func.api_code); + buildTypeVec(n.properties_json, &tmp_vec); + c.cbm_rsq_encode(&tmp_vec.v, &func.type_code); + buildDecoVec(n.properties_json, &tmp_vec); + c.cbm_rsq_encode(&tmp_vec.v, &func.deco_code); + } + } +} + +// ── Phase helpers (keep cbm_pipeline_pass_semantic_edges complexity low) ── + +/// Canonical node order: by qualified name (unique per node), id tie-break +/// for defensiveness. Gives the semantic pass a stable, content-derived input +/// order regardless of how parallel extraction merged the graph buffer. +fn cmpNodePtrByQn(pa: ?*const anyopaque, pb: ?*const anyopaque) callconv(.c) c_int { + const a: *const Node = @as(*const *const Node, @ptrCast(@alignCast(pa))).*; + const b: *const Node = @as(*const *const Node, @ptrCast(@alignCast(pb))).*; + const qa: CStr = if (a.qualified_name != null) a.qualified_name else ""; + const qb: CStr = if (b.qualified_name != null) b.qualified_name else ""; + const r = c.strcmp(qa, qb); + if (r != 0) return r; + if (a.id != b.id) return if (a.id < b.id) -1 else 1; + return 0; +} + +/// Phase 1a: seed the funcs[] / node_ptrs[] arrays from all Function and +/// Method nodes in the graph buffer. Returns the number of functions collected +/// (0 on OOM), and fills out_funcs / out_nodes with newly malloc'd arrays. +fn phase1ScanFunctions(gbuf: ?*Gbuf, out_funcs: *[*c]Func, out_nodes: *NodePtrs) c_int { + out_funcs.* = null; + out_nodes.* = null; + var funcs: [*c]Func = null; + var node_ptrs: NodePtrs = null; + var func_count: c_int = 0; + var func_cap: c_int = 0; + labels: for ([_][*:0]const u8{ "Function", "Method" }) |label| { + var nodes: NodePtrs = null; + var node_count: c_int = 0; + if (c.cbm_gbuf_find_by_label(gbuf, label, &nodes, &node_count) != 0) continue; + for (0..ix(node_count)) |i| { + if (func_count >= func_cap) { + const new_cap: c_int = if (func_cap < MAX_FUNCS_INIT) MAX_FUNCS_INIT else func_cap * GROW; + const grown = interop.reallocN(Func, funcs, ix(new_cap)); + // A failed growth ends this label's scan, as the C `break` did. + if (grown == null) continue :labels; + funcs = grown; + const np_grown = interop.reallocN([*c]const Node, node_ptrs, ix(new_cap)); + if (np_grown == null) continue :labels; + node_ptrs = np_grown; + func_cap = new_cap; + } + const func = util.at(funcs, ix(func_count)); + func.* = std.mem.zeroes(Func); + func.node_id = nodes[i].*.id; + func.file_path = nodes[i].*.file_path; + func.file_ext = c.cbm_path_last_dot(nodes[i].*.file_path); + node_ptrs[ix(func_count)] = nodes[i]; + func_count += 1; + } + } + // Canonicalize the func order (determinism): the label-index order above + // is gbuf insertion order = parallel-extraction merge order, which varies + // run to run. Everything downstream is order-sensitive — LSH bucket chain + // order, the SEM_MAX_CANDIDATES truncation, seen[] and the admission + // sequence — so an unstable order changes WHICH semantic edges are + // emitted. Sort the cheap pointer array by qualified name (unique) and + // re-derive the three fields set so far; the heavy per-func payloads are + // filled in later phases, so no 12.7 KB structs are moved. + if (func_count > 1) { + c.qsort(@ptrCast(node_ptrs), ix(func_count), @sizeOf([*c]const Node), cmpNodePtrByQn); + } + for (0..ix(func_count)) |k| { + funcs[k].node_id = node_ptrs[k].*.id; + funcs[k].file_path = node_ptrs[k].*.file_path; + funcs[k].file_ext = c.cbm_path_last_dot(node_ptrs[k].*.file_path); + } + out_funcs.* = funcs; + out_nodes.* = node_ptrs; + return func_count; +} + +/// Phase 5c: partition functions into LSH buckets by their signature bands. +/// Sequential because each bucket grows its `items` array via realloc. +fn phase5cBuildLshBuckets(signatures: [*c]const u64, func_count: c_int, band_buckets: [*c][*c]SemBucket) void { + for (0..ix(func_count)) |f| { + for (0..SEM_LSH_BANDS) |b| { + const bucket = &band_buckets[b][bandBucketIndex(signatures[f], b)]; + if (bucket.count >= bucket.cap) { + const nc: c_int = if (bucket.cap < SEM_BUCKET_CAP_INIT) SEM_BUCKET_CAP_INIT else bucket.cap * GROW; + const ni = interop.reallocN(c_int, bucket.items, ix(nc)); + if (ni == null) continue; + bucket.items = ni; + bucket.cap = nc; + } + bucket.items[ix(bucket.count)] = @intCast(f); + bucket.count += 1; + } + } +} + +/// Canonical order for candidate pairs: ascending discovering-func index, +/// then ascending candidate rank — the order a sequential scoring loop over +/// canonically-sorted funcs would have produced. +fn cmpDeferredEdgeCanonical(pa: ?*const anyopaque, pb: ?*const anyopaque) callconv(.c) c_int { + const a: *const DeferredEdge = @ptrCast(@alignCast(pa)); + const b: *const DeferredEdge = @ptrCast(@alignCast(pb)); + if (a.i != b.i) return if (a.i < b.i) -1 else 1; + if (a.c != b.c) return if (a.c < b.c) -1 else 1; + return 0; +} + +/// Phase 6b. Sequential, deterministic admission + merge: gather every +/// above-threshold pair from the worker buffers, replay them in canonical +/// (i, rank) order, and apply the per-node max_edges budget HERE — +/// single-threaded — so the admitted edge set is a pure function of the +/// (canonically sorted) inputs, independent of worker count and scheduling. +fn phase6bMergeEdges(gbuf: ?*Gbuf, worker_bufs: [*c]DeferredEdgeBuf, worker_count: c_int, edge_counts: [*c]c_int, max_edges: c_int) c_int { + const workers = ix(worker_count); + var total_pairs: c_int = 0; + for (0..workers) |w| total_pairs += worker_bufs[w].count; + var pairs: [*c]DeferredEdge = null; + if (total_pairs > 0) { + pairs = interop.mallocN(DeferredEdge, ix(total_pairs)); + } + if (pairs == null) { + for (0..workers) |w| deferredBufFree(util.at(worker_bufs, w)); + return 0; + } + var n: usize = 0; + for (0..workers) |w| { + const buf = util.at(worker_bufs, w); + if (buf.count > 0) { + @memcpy(pairs[n..][0..ix(buf.count)], buf.edges[0..ix(buf.count)]); + } + n += ix(buf.count); + deferredBufFree(buf); + } + if (n > 1) { + c.qsort(pairs, n, @sizeOf(DeferredEdge), cmpDeferredEdgeCanonical); + } + + var total_edges: c_int = 0; + for (0..n) |e| { + const de = util.at(pairs, e); + if (edge_counts[ix(de.i)] >= max_edges or edge_counts[ix(de.j)] >= max_edges) continue; + var props: [PROPS_BUF]u8 = undefined; + _ = c.snprintf( + &props, + props.len, + "{\"score\":%.3f,\"same_file\":%s}", + @as(f64, de.score), + @as(CStr, if (de.same_file) "true" else "false"), + ); + _ = c.cbm_gbuf_insert_edge(gbuf, de.source_id, de.target_id, "SEMANTICALLY_RELATED", &props); + edge_counts[ix(de.i)] += 1; + edge_counts[ix(de.j)] += 1; + total_edges += 1; + } + c.free(pairs); + return total_edges; +} + +/// Phase 3c: export co-occurrence-enriched token vectors into the graph buffer +/// so query-time lookups can use them without re-running the finalize step. +fn phase3cExportTokenVectors(gbuf: ?*Gbuf, corpus: ?*c.cbm_sem_corpus_t) void { + const tv_count = c.cbm_sem_corpus_token_count(corpus); + var t: c_int = 0; + while (t < tv_count) : (t += 1) { + var vec: [*c]const Vec = null; + var idf: f32 = 0.0; + const tok: CStr = c.cbm_sem_corpus_token_at(corpus, t, &vec, &idf); + if (tok == null or vec == null or idf <= PSE_FLOW_WEIGHT) continue; + var qvec: [SEM_DIM]u8 = undefined; + for (&qvec, vec.*.v) |*q, v| q.* = quantizeUnit(v); + _ = c.cbm_gbuf_store_token_vector(gbuf, tok, &qvec, SEM_DIM, idf); + } + interop.logInfo("pass.semantic.token_vectors", .{ + @as(CStr, "count"), @as(CStr, c.cbm_log_itoa(tv_count)), + }); +} + +/// Phase 5a: generate NUM_HYPERPLANES × CBM_RSQ_DIM deterministic random +/// float hyperplanes seeded from XXH3 so signatures are reproducible. +fn phase5aBuildHyperplanes() ?[*]HyperplaneRow { + const hyperplanes: [*]HyperplaneRow = @as(?[*]HyperplaneRow, interop.mallocN(HyperplaneRow, NUM_HYPERPLANES)) orelse return null; + const u32_max: f32 = @floatFromInt(@as(u32, std.math.maxInt(u32))); + for (0..NUM_HYPERPLANES) |h| { + for (0..RSQ_DIM) |d| { + const seed = XxHash3.hash(@as(u64, h) * RSQ_DIM, &std.mem.toBytes(@as(c_int, @intCast(d)))); + const low: u32 = @truncate(seed); + hyperplanes[h][d] = (@as(f32, @floatFromInt(low)) / u32_max) - PSE_ROUND_BIAS; + } + } + return hyperplanes; +} + +/// Phase 1b: decode per-function minhash/profile/API/type/deco vectors in +/// parallel. Must be called after cbm_sem_ensure_ready() so the pretrained +/// token map is initialized. +fn phase1bDecodeAndBuild(funcs: [*c]Func, node_ptrs: NodePtrs, gbuf: ?*const Gbuf, func_count: c_int, worker_count: c_int) void { + if (func_count <= 0) return; + var cc: CollectCtx = .{ + .funcs = funcs, + .node_ptrs = node_ptrs, + .gbuf = gbuf, + .func_count = func_count, + .next_idx = .init(0), + }; + c.cbm_parallel_for(worker_count, collectWorker, &cc, parallelOpts(worker_count)); +} + +/// Phase 2: tokenize each function's metadata in parallel, filling +/// all_tokens[] and token_counts[]. Caller allocates the arrays. +fn phase2Tokenize(node_ptrs: NodePtrs, gbuf: ?*Gbuf, all_tokens: [*c]MutStr, token_counts: [*c]c_int, func_count: c_int, worker_count: c_int, pools: [*c]?*HashTable) void { + var tc: TokenizeCtx = .{ + .node_ptrs = node_ptrs, + .gbuf = gbuf, + .all_tokens = all_tokens, + .token_counts = token_counts, + .func_count = func_count, + .next_idx = .init(0), + .pools = pools, + }; + c.cbm_parallel_for(worker_count, tokenizeWorker, &tc, parallelOpts(worker_count)); +} + +/// Phase 4a: build per-function TF-IDF + RI vectors in parallel, producing +/// int8-quantized qvecs for subsequent storage. Phase 4b runs sequentially +/// to store them in gbuf because gbuf is not thread-safe. +fn phase4BuildAndStoreVectors(gbuf: ?*Gbuf, funcs: [*c]Func, all_tokens: [*c]MutStr, token_counts: [*c]c_int, corpus: ?*c.cbm_sem_corpus_t, func_count: c_int, worker_count: c_int) void { + const qvecs: [*]u8 = @as(?[*]u8, interop.mallocN(u8, ix(func_count) * SEM_DIM)) orelse return; + var vc: VecBuildCtx = .{ + .funcs = funcs, + .all_tokens = all_tokens, + .token_counts = token_counts, + .corpus = corpus, + .qvecs = qvecs, + .func_count = func_count, + .next_idx = .init(0), + }; + c.cbm_parallel_for(worker_count, vecBuildWorker, &vc, parallelOpts(worker_count)); + for (0..ix(func_count)) |f| { + _ = c.cbm_gbuf_store_vector(gbuf, funcs[f].node_id, qvecs + f * SEM_DIM, SEM_DIM); + } + c.free(qvecs); +} + +/// Phase 5: hyperplane generation → signatures → LSH bucket population. +/// Returns the malloc'd signatures array and band_buckets[] via out-params; +/// caller frees both. +fn phase5LshBuild(funcs: [*c]Func, func_count: c_int, worker_count: c_int, out_signatures: *[*c]u64, out_buckets: *[*c][*c]SemBucket) void { + const hyperplanes = phase5aBuildHyperplanes(); + const signatures = interop.callocN(u64, ix(func_count)); + if (hyperplanes != null and signatures != null) { + var sc: SigBuildCtx = .{ + .funcs = funcs, + .signatures = signatures, + .hyperplanes = hyperplanes.?, + .func_count = func_count, + .next_idx = .init(0), + }; + c.cbm_parallel_for(worker_count, sigBuildWorker, &sc, parallelOpts(worker_count)); + } + c.free(@ptrCast(hyperplanes)); + + const band_buckets = interop.callocN([*c]SemBucket, SEM_LSH_BANDS); + if (band_buckets != null) { + for (0..SEM_LSH_BANDS) |b| { + band_buckets[b] = interop.callocN(SemBucket, SEM_BUCKET_COUNT); + } + phase5cBuildLshBuckets(signatures, func_count, band_buckets); + } + out_signatures.* = signatures; + out_buckets.* = band_buckets; +} + +/// Phase 6a: score candidate pairs in parallel and collect deferred edges. +fn phase6aScoreCandidates(funcs: [*c]Func, signatures: [*c]u64, edge_counts: [*c]c_int, band_buckets: [*c][*c]SemBucket, cfg: c.cbm_sem_config_t, worker_bufs: [*c]DeferredEdgeBuf, func_count: c_int, worker_count: c_int) void { + var sc: ScoreCtx = .{ + .funcs = funcs, + .signatures = signatures, + .edge_counts = edge_counts, + .cfg = cfg, + .func_count = func_count, + .band_buckets = band_buckets, + .worker_bufs = worker_bufs, + .max_workers = worker_count, + .next_idx = .init(0), + }; + c.cbm_parallel_for(worker_count, scoreWorker, &sc, parallelOpts(worker_count)); +} + +/// Phase 7: free LSH bucket storage (items arrays and the per-band arrays). +fn freeLshBuckets(band_buckets: [*c][*c]SemBucket) void { + if (band_buckets == null) return; + for (0..SEM_LSH_BANDS) |b| { + if (band_buckets[b] == null) continue; + for (0..SEM_BUCKET_COUNT) |h| { + c.free(band_buckets[b][h].items); + } + c.free(band_buckets[b]); + } + c.free(@ptrCast(band_buckets)); +} + +/// Phases 3a/3b/3c bundled: create corpus, batch-add docs, finalize, export +/// enriched token vectors to the graph buffer. Returns the new corpus, which +/// the caller must cbm_sem_corpus_free() later. +fn runCorpusPhase(gbuf: ?*Gbuf, all_tokens: [*c]MutStr, token_counts: [*c]c_int, func_count: c_int) ?*c.cbm_sem_corpus_t { + const t_phase3a = util.profStart(); + const corpus = c.cbm_sem_corpus_new(); + c.cbm_sem_corpus_add_docs_batch(corpus, all_tokens, token_counts, func_count, MAX_TOKENS); + util.profEnd("semantic_edges", "3a_corpus_batch", &t_phase3a, func_count); + + const t_phase3b = util.profStart(); + c.cbm_sem_corpus_finalize(corpus); + util.profEnd("semantic_edges", "3b_corpus_finalize_seq", &t_phase3b, c.cbm_sem_corpus_token_count(corpus)); + + const t_phase3c = util.profStart(); + phase3cExportTokenVectors(gbuf, corpus); + util.profEnd("semantic_edges", "3c_token_vec_export_seq", &t_phase3c, c.cbm_sem_corpus_token_count(corpus)); + return corpus; +} + +/// Phases 6a/6b bundled: parallel scoring + sequential merge of deferred +/// edges. Returns the number of emitted edges. +fn runScoringPhase(gbuf: ?*Gbuf, funcs: [*c]Func, signatures: [*c]u64, band_buckets: [*c][*c]SemBucket, cfg: c.cbm_sem_config_t, func_count: c_int, worker_count: c_int) c_int { + const edge_counts = interop.callocN(c_int, ix(func_count)); + const worker_bufs = interop.callocN(DeferredEdgeBuf, ix(worker_count)); + if (edge_counts == null or worker_bufs == null) { + c.free(edge_counts); + c.free(worker_bufs); + return 0; + } + for (0..ix(worker_count)) |w| deferredBufInit(util.at(worker_bufs, w)); + + const t_phase6a = util.profStart(); + phase6aScoreCandidates(funcs, signatures, edge_counts, band_buckets, cfg, worker_bufs, func_count, worker_count); + util.profEnd("semantic_edges", "6a_score_parallel", &t_phase6a, func_count); + + const t_phase6b = util.profStart(); + const total = phase6bMergeEdges(gbuf, worker_bufs, worker_count, edge_counts, cfg.max_edges); + util.profEnd("semantic_edges", "6b_edge_merge_seq", &t_phase6b, total); + + c.free(worker_bufs); + c.free(edge_counts); + return total; +} + +fn freeTokenPoolEntry(key: CStr, value: ?*anyopaque, ud: ?*anyopaque) callconv(.c) void { + _ = key; // key == value: one owned string per unique token + _ = ud; + c.free(value); +} + +/// Free the per-function arrays malloc'd during phases 1b and 4a. +/// all_tokens slots BORROW their strings from the per-worker intern pools; +/// the pools own exactly one copy per unique token per worker. +fn freeFuncsAndTokens(funcs: [*c]Func, func_count: c_int, all_tokens: [*c]MutStr, pools: [*c]?*HashTable, worker_count: c_int) void { + for (0..ix(func_count)) |f| { + c.free(funcs[f].tfidf_indices); + c.free(funcs[f].tfidf_weights); + } + c.free(@ptrCast(all_tokens)); + c.free(funcs); + if (pools != null) { + for (0..ix(worker_count)) |w| { + if (pools[w]) |pool| { + c.cbm_ht_foreach(pool, freeTokenPoolEntry, null); + c.cbm_ht_free(pool); + } + } + c.free(@ptrCast(pools)); + } +} + +fn logCount(msg: [*:0]const u8, key: [*:0]const u8, value: c_int) void { + interop.logInfo(msg, .{ @as(CStr, key), @as(CStr, c.cbm_log_itoa(value)) }); +} + +// ── Pass entry point ──────────────────────────────────────────────── + +pub export fn cbm_pipeline_pass_semantic_edges(ctx: *c.cbm_pipeline_ctx_t) c_int { + // Controlled by pipeline mode (moderate/full), not env var + interop.logInfo("pass.start", .{ @as(CStr, "pass"), @as(CStr, "semantic_edges") }); + + const gbuf = ctx.gbuf; + const cfg = c.cbm_sem_get_config(); + + const t_phase1a = util.profStart(); + var funcs: [*c]Func = null; + var node_ptrs: NodePtrs = null; + const func_count = phase1ScanFunctions(gbuf, &funcs, &node_ptrs); + util.profEnd("semantic_edges", "1a_scan_seq", &t_phase1a, func_count); + + // Phase 1b: Decode minhash + profile + build api/type/deco vectors (PARALLEL). + c.cbm_sem_ensure_ready(); + const t_phase1b = util.profStart(); + phase1bDecodeAndBuild(funcs, node_ptrs, gbuf, func_count, c.cbm_default_worker_count(false)); + util.profEnd("semantic_edges", "1b_decode_build_parallel", &t_phase1b, func_count); + logCount("pass.semantic.collected", "functions", func_count); + + if (func_count < PSE_MIN_FUNCS_FOR_PAIR) { + c.free(funcs); + c.free(@ptrCast(node_ptrs)); + interop.logInfo("pass.done", .{ + @as(CStr, "pass"), @as(CStr, "semantic_edges"), @as(CStr, "edges"), @as(CStr, "0"), + }); + return 0; + } + + // Phase 2: Tokenize all nodes (PARALLEL) + const worker_count = c.cbm_default_worker_count(false); + const all_tokens = interop.mallocN(MutStr, ix(func_count) * MAX_TOKENS); + const token_counts = interop.callocN(c_int, ix(func_count)); + + const t_phase2 = util.profStart(); + const token_pools = interop.callocN(?*HashTable, ix(worker_count)); + if (token_pools != null) { + for (0..ix(worker_count)) |w| { + token_pools[w] = c.cbm_ht_create(c.CBM_SZ_1K); + } + } + phase2Tokenize(node_ptrs, gbuf, all_tokens, token_counts, func_count, worker_count, token_pools); + util.profEnd("semantic_edges", "2_tokenize_parallel", &t_phase2, func_count); + c.free(@ptrCast(node_ptrs)); + + // Phase 3: Build corpus (batch add), finalize, export enriched token vectors. + const corpus = runCorpusPhase(gbuf, all_tokens, token_counts, func_count); + + // Phase 4: Build per-function TF-IDF + RI vectors (PARALLEL) and store them. + const t_phase4 = util.profStart(); + phase4BuildAndStoreVectors(gbuf, funcs, all_tokens, token_counts, corpus, func_count, worker_count); + util.profEnd("semantic_edges", "4_build_and_store_vec", &t_phase4, func_count); + + logCount("pass.semantic.vectors_stored", "count", func_count); + + // Phase 5: LSH hyperplanes → signatures → buckets. + const t_phase5 = util.profStart(); + var signatures: [*c]u64 = null; + var band_buckets: [*c][*c]SemBucket = null; + phase5LshBuild(funcs, func_count, worker_count, &signatures, &band_buckets); + util.profEnd("semantic_edges", "5_lsh_build", &t_phase5, func_count); + + interop.logInfo("pass.semantic.lsh_built", .{ + @as(CStr, "functions"), @as(CStr, c.cbm_log_itoa(func_count)), + @as(CStr, "bands"), @as(CStr, c.cbm_log_itoa(SEM_LSH_BANDS)), + }); + + // Phase 6: Parallel scoring + sequential edge merge. + const total_edges = runScoringPhase(gbuf, funcs, signatures, band_buckets, cfg, func_count, worker_count); + + // Phase 7: Cleanup + const t_phase7 = util.profStart(); + freeLshBuckets(band_buckets); + c.free(signatures); + interop.logInfo("pass.done", .{ + @as(CStr, "pass"), @as(CStr, "semantic_edges"), + @as(CStr, "edges"), @as(CStr, c.cbm_log_itoa(total_edges)), + }); + freeFuncsAndTokens(funcs, func_count, all_tokens, token_pools, worker_count); + c.free(token_counts); + c.cbm_sem_corpus_free(corpus); + util.profEnd("semantic_edges", "7_cleanup", &t_phase7, 0); + + return 0; +} diff --git a/src/pipeline/pass_similarity.c b/src/pipeline/pass_similarity.c deleted file mode 100644 index cdeb77ca..00000000 --- a/src/pipeline/pass_similarity.c +++ /dev/null @@ -1,336 +0,0 @@ -/* - * pass_similarity.c — Generate SIMILAR_TO edges from MinHash fingerprints. - * - * Reads "fp" hex strings from Function/Method node properties, - * builds an LSH index, and emits SIMILAR_TO edges for pairs with - * Jaccard similarity ≥ threshold. - * - * Runs as a post-pass after enrichment (both full and incremental). - */ -#include "foundation/constants.h" -#include "pipeline/pipeline.h" -#include -#include "pipeline/pipeline_internal.h" -#include "graph_buffer/graph_buffer.h" -#include "simhash/minhash.h" -#include "foundation/log.h" -#include "foundation/compat.h" - -#include "foundation/profile.h" -#include "foundation/platform.h" -#include "foundation/str_util.h" - -enum { - SIM_EDGE_INIT_CAP = 256, - SIM_EDGE_GROW = 2, -}; -#include "pipeline/worker_pool.h" - -#include -#include -#include -#include - -enum { FP_KEY_PREFIX_LEN = 6, MIN_FP_ENTRIES = 2 }; /* strlen("\"fp\":\"") */ - -/* ── Helpers ─────────────────────────────────────────────────────── */ - -/* Parse "fp" hex string from a node's properties_json. - * Returns true if found and decoded successfully. */ -static bool parse_fp_from_props(const char *props_json, cbm_minhash_t *out) { - if (!props_json) { - return false; - } - const char *fp_key = strstr(props_json, "\"fp\":\""); - if (!fp_key) { - return false; - } - const char *hex_start = fp_key + FP_KEY_PREFIX_LEN; - /* Find closing quote */ - const char *hex_end = strchr(hex_start, '"'); - if (!hex_end) { - return false; - } - int hex_len = (int)(hex_end - hex_start); - if (hex_len != CBM_MINHASH_HEX_LEN) { - return false; - } - char hex_buf[CBM_MINHASH_HEX_BUF]; - memcpy(hex_buf, hex_start, (size_t)hex_len); - hex_buf[hex_len] = '\0'; - return cbm_minhash_from_hex(hex_buf, out); -} - -/* ── Internal types ──────────────────────────────────────────────── */ - -enum { FP_ENTRY_INIT_CAP = 256, FP_ENTRY_GROW = 2, PROPS_BUF_LEN = 256 }; - -typedef struct { - int64_t node_id; - cbm_minhash_t fp; - const char *file_path; - const char *ext; - const char *qn; /* canonical ordering + pair-ownership (determinism) */ -} fp_entry_t; - -/* Canonical entry order: by qualified name (unique). The label-index order - * from collect_fp_entries is gbuf insertion order = parallel-extraction merge - * order, which varies run to run; LSH bucket chains and the per-node edge-cap - * truncation both inherit it, flickering WHICH SIMILAR_TO edges are emitted. */ -static int cmp_fp_entry_by_qn(const void *pa, const void *pb) { - const fp_entry_t *a = pa; - const fp_entry_t *b = pb; - const char *qa = a->qn ? a->qn : ""; - const char *qb = b->qn ? b->qn : ""; - int r = strcmp(qa, qb); - if (r != 0) { - return r; - } - if (a->node_id != b->node_id) { - return a->node_id < b->node_id ? -1 : 1; - } - return 0; -} - -/* Collect all Function/Method nodes with fingerprints from graph buffer. */ -static int collect_fp_entries(cbm_gbuf_t *gbuf, fp_entry_t **out_entries) { - fp_entry_t *entries = NULL; - int count = 0; - int cap = 0; - - const char *labels[] = {"Function", "Method", NULL}; - for (int li = 0; labels[li]; li++) { - const cbm_gbuf_node_t **nodes = NULL; - int node_count = 0; - if (cbm_gbuf_find_by_label(gbuf, labels[li], &nodes, &node_count) != 0) { - continue; - } - for (int i = 0; i < node_count; i++) { - const cbm_gbuf_node_t *n = nodes[i]; - cbm_minhash_t fp; - if (!parse_fp_from_props(n->properties_json, &fp)) { - continue; - } - if (count >= cap) { - int new_cap = cap < FP_ENTRY_INIT_CAP ? FP_ENTRY_INIT_CAP : cap * FP_ENTRY_GROW; - fp_entry_t *grown = realloc(entries, (size_t)new_cap * sizeof(fp_entry_t)); - if (!grown) { - break; - } - entries = grown; - cap = new_cap; - } - entries[count++] = (fp_entry_t){ - .node_id = n->id, - .fp = fp, - .file_path = n->file_path, - .ext = cbm_path_last_dot(n->file_path), - .qn = n->qualified_name, - }; - } - } - /* Canonicalize (determinism) — see cmp_fp_entry_by_qn. */ - if (count > 1) { - qsort(entries, (size_t)count, sizeof(fp_entry_t), cmp_fp_entry_by_qn); - } - *out_entries = entries; - return count; -} - -/* ── Parallel query + emit ────────────────────────────────────────── */ - -/* Deferred edge record; collected per-worker, merged into gbuf sequentially. */ -typedef struct { - int64_t source_id; - int64_t target_id; - double jaccard; - bool same_file; -} sim_deferred_edge_t; - -typedef struct { - sim_deferred_edge_t *edges; - int count; - int cap; -} sim_edge_buf_t; - -static void sim_edge_buf_push(sim_edge_buf_t *buf, int64_t src, int64_t tgt, double jaccard, - bool same_file) { - if (buf->count >= buf->cap) { - int nc = buf->cap < SIM_EDGE_INIT_CAP ? SIM_EDGE_INIT_CAP : buf->cap * SIM_EDGE_GROW; - sim_deferred_edge_t *grown = realloc(buf->edges, (size_t)nc * sizeof(sim_deferred_edge_t)); - if (!grown) { - return; - } - buf->edges = grown; - buf->cap = nc; - } - buf->edges[buf->count++] = (sim_deferred_edge_t){ - .source_id = src, .target_id = tgt, .jaccard = jaccard, .same_file = same_file}; -} - -typedef struct { - const fp_entry_t *entries; - int entry_count; - const cbm_lsh_index_t *lsh; - sim_edge_buf_t *worker_bufs; - _Atomic int next_idx; - _Atomic int *edge_counts; /* shared atomic array, one per entry */ -} sim_query_ctx_t; - -enum { SIM_CAND_CAP = 4096 }; - -static void sim_query_worker(int worker_id, void *ctx_ptr) { - sim_query_ctx_t *sc = ctx_ptr; - sim_edge_buf_t *my_buf = &sc->worker_bufs[worker_id]; - - /* Thread-local candidate buffer (stack-allocated) */ - const cbm_lsh_entry_t *cands[SIM_CAND_CAP]; - - while (true) { - int i = atomic_fetch_add_explicit(&sc->next_idx, SKIP_ONE, memory_order_relaxed); - if (i >= sc->entry_count) { - break; - } - - int ec = atomic_load_explicit(&sc->edge_counts[i], memory_order_relaxed); - if (ec >= CBM_MINHASH_MAX_EDGES_PER_NODE) { - continue; - } - - const fp_entry_t *src = &sc->entries[i]; - int cand_count = cbm_lsh_query_into(sc->lsh, &src->fp, cands, SIM_CAND_CAP); - - int emitted = 0; - for (int c = 0; c < cand_count; c++) { - const cbm_lsh_entry_t *cand = cands[c]; - if (cand->node_id == src->node_id) { - continue; - } - if (strcmp(src->ext, cand->file_ext) != 0) { - continue; - } - /* Pair ownership by canonical QN order, not node id: ids are - * assigned in parallel-merge order and vary run to run, which - * flipped which side owned a pair and (with the per-source edge - * cap) flickered the emitted set (determinism). */ - if (!src->qn || !cand->qualified_name || strcmp(src->qn, cand->qualified_name) >= 0) { - continue; - } - - int cur = atomic_load_explicit(&sc->edge_counts[i], memory_order_relaxed); - if (cur + emitted >= CBM_MINHASH_MAX_EDGES_PER_NODE) { - break; - } - - double jaccard = cbm_minhash_jaccard(&src->fp, cand->fingerprint); - if (jaccard < CBM_MINHASH_JACCARD_THRESHOLD) { - continue; - } - - bool same_file = - src->file_path && cand->file_path && strcmp(src->file_path, cand->file_path) == 0; - sim_edge_buf_push(my_buf, src->node_id, cand->node_id, jaccard, same_file); - emitted++; - } - if (emitted > 0) { - atomic_fetch_add_explicit(&sc->edge_counts[i], emitted, memory_order_relaxed); - } - } -} - -/* Merge worker edge buffers into gbuf. Returns total edge count. Frees worker buffers. */ -static int merge_sim_edges(cbm_gbuf_t *gbuf, sim_edge_buf_t *worker_bufs, int worker_count) { - int total = 0; - for (int w = 0; w < worker_count; w++) { - for (int e = 0; e < worker_bufs[w].count; e++) { - sim_deferred_edge_t *de = &worker_bufs[w].edges[e]; - char props[PROPS_BUF_LEN]; - snprintf(props, sizeof(props), "{\"jaccard\":%.3f,\"same_file\":%s}", de->jaccard, - de->same_file ? "true" : "false"); - cbm_gbuf_insert_edge(gbuf, de->source_id, de->target_id, "SIMILAR_TO", props); - total++; - } - free(worker_bufs[w].edges); - } - free(worker_bufs); - return total; -} - -/* ── Pass entry point ────────────────────────────────────────────── */ - -int cbm_pipeline_pass_similarity(cbm_pipeline_ctx_t *ctx) { - cbm_log_info("pass.start", "pass", "similarity"); - - cbm_gbuf_t *gbuf = ctx->gbuf; - - /* Phase 1: Collect fingerprints from Function/Method nodes */ - CBM_PROF_START(t_collect); - fp_entry_t *entries = NULL; - int entry_count = collect_fp_entries(gbuf, &entries); - CBM_PROF_END_N("similarity", "1_collect_fp", t_collect, entry_count); - - cbm_log_info("pass.similarity.collected", "nodes_with_fp", cbm_log_itoa(entry_count)); - - if (entry_count < MIN_FP_ENTRIES) { - free(entries); - cbm_log_info("pass.done", "pass", "similarity", "edges", "0"); - return 0; - } - - /* Phase 2: Build LSH index (sequential — cbm_lsh_insert mutates shared state) */ - CBM_PROF_START(t_lsh_build); - cbm_lsh_index_t *lsh = cbm_lsh_new(); - cbm_lsh_entry_t *lsh_entries = malloc((size_t)entry_count * sizeof(cbm_lsh_entry_t)); - if (!lsh_entries) { - free(entries); - cbm_lsh_free(lsh); - return CBM_NOT_FOUND; - } - - for (int i = 0; i < entry_count; i++) { - lsh_entries[i] = (cbm_lsh_entry_t){ - .node_id = entries[i].node_id, - .fingerprint = &entries[i].fp, - .file_path = entries[i].file_path, - .file_ext = entries[i].ext, - .qualified_name = entries[i].qn, - }; - cbm_lsh_insert(lsh, &lsh_entries[i]); - } - CBM_PROF_END_N("similarity", "2_lsh_build_seq", t_lsh_build, entry_count); - - /* Phase 3: Query LSH + emit edges (PARALLEL via cbm_lsh_query_into). - * Each worker claims entries, queries, scores candidates, stashes edges - * in its own deferred buffer. Shared edge_counts is atomic. - * Final merge into gbuf is sequential (gbuf not thread-safe). */ - CBM_PROF_START(t_query_emit); - _Atomic int *edge_counts = calloc((size_t)entry_count, sizeof(_Atomic int)); - int worker_count = cbm_default_worker_count(false); - sim_edge_buf_t *worker_bufs = calloc((size_t)worker_count, sizeof(sim_edge_buf_t)); - - { - sim_query_ctx_t sc = { - .entries = entries, - .entry_count = entry_count, - .lsh = lsh, - .worker_bufs = worker_bufs, - .edge_counts = edge_counts, - }; - atomic_init(&sc.next_idx, 0); - cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false}; - cbm_parallel_for(worker_count, sim_query_worker, &sc, opts); - } - CBM_PROF_END_N("similarity", "3_query_parallel", t_query_emit, entry_count); - - CBM_PROF_START(t_merge); - int total_edges = merge_sim_edges(gbuf, worker_bufs, worker_count); - CBM_PROF_END_N("similarity", "4_edge_merge_seq", t_merge, total_edges); - - cbm_log_info("pass.done", "pass", "similarity", "edges", cbm_log_itoa(total_edges)); - - free(edge_counts); - free(lsh_entries); - free(entries); - cbm_lsh_free(lsh); - return 0; -} diff --git a/src/pipeline/pass_similarity.zig b/src/pipeline/pass_similarity.zig new file mode 100644 index 00000000..e111556a --- /dev/null +++ b/src/pipeline/pass_similarity.zig @@ -0,0 +1,317 @@ +//! pass_similarity.zig — Generate SIMILAR_TO edges from MinHash fingerprints. +//! +//! Reads "fp" hex strings from Function/Method node properties, +//! builds an LSH index, and emits SIMILAR_TO edges for pairs with +//! Jaccard similarity ≥ threshold. +//! +//! Runs as a post-pass after enrichment (both full and incremental). + +const std = @import("std"); +const c = @import("c"); +const interop = @import("../foundation/interop.zig"); +const util = @import("util.zig"); + +const CStr = util.CStr; +const ix = util.ix; +const Node = c.cbm_gbuf_node_t; +const AtomicInt = std.atomic.Value(c_int); + +const SIM_EDGE_INIT_CAP = 256; +const SIM_EDGE_GROW = 2; + +const FP_KEY_PREFIX_LEN = 6; // strlen("\"fp\":\"") +const MIN_FP_ENTRIES = 2; + +// ── Helpers ───────────────────────────────────────────────────────── + +/// Parse "fp" hex string from a node's properties_json. +/// Returns true if found and decoded successfully. +fn parseFpFromProps(props_json: CStr, out: *c.cbm_minhash_t) bool { + if (props_json == null) return false; + const fp_key: CStr = c.strstr(props_json, "\"fp\":\""); + if (fp_key == null) return false; + const hex_start: CStr = fp_key + FP_KEY_PREFIX_LEN; + // Find closing quote + const hex_end: CStr = c.strchr(hex_start, '"'); + if (hex_end == null) return false; + const hex_len = util.ptrDiff(hex_end, hex_start); + if (hex_len != c.CBM_MINHASH_HEX_LEN) return false; + var hex_buf: [c.CBM_MINHASH_HEX_BUF]u8 = undefined; + @memcpy(hex_buf[0..hex_len], hex_start[0..hex_len]); + hex_buf[hex_len] = 0; + return c.cbm_minhash_from_hex(&hex_buf, out); +} + +// ── Internal types ────────────────────────────────────────────────── + +const FP_ENTRY_INIT_CAP = 256; +const FP_ENTRY_GROW = 2; +const PROPS_BUF_LEN = 256; + +const FpEntry = extern struct { + node_id: i64, + fp: c.cbm_minhash_t, + file_path: CStr, + ext: CStr, + qn: CStr, // canonical ordering + pair-ownership (determinism) +}; + +/// Canonical entry order: by qualified name (unique). The label-index order +/// from collectFpEntries is gbuf insertion order = parallel-extraction merge +/// order, which varies run to run; LSH bucket chains and the per-node edge-cap +/// truncation both inherit it, flickering WHICH SIMILAR_TO edges are emitted. +fn cmpFpEntryByQn(pa: ?*const anyopaque, pb: ?*const anyopaque) callconv(.c) c_int { + const a: *const FpEntry = @ptrCast(@alignCast(pa)); + const b: *const FpEntry = @ptrCast(@alignCast(pb)); + const qa: CStr = if (a.qn != null) a.qn else ""; + const qb: CStr = if (b.qn != null) b.qn else ""; + const r = c.strcmp(qa, qb); + if (r != 0) return r; + if (a.node_id != b.node_id) return if (a.node_id < b.node_id) -1 else 1; + return 0; +} + +/// Collect all Function/Method nodes with fingerprints from graph buffer. +fn collectFpEntries(gbuf: ?*c.cbm_gbuf_t, out_entries: *[*c]FpEntry) c_int { + var entries: [*c]FpEntry = null; + var count: c_int = 0; + var cap: c_int = 0; + + outer: for ([_][*:0]const u8{ "Function", "Method" }) |label| { + var nodes: [*c][*c]const Node = null; + var node_count: c_int = 0; + if (c.cbm_gbuf_find_by_label(gbuf, label, &nodes, &node_count) != 0) continue; + var i: usize = 0; + while (i < node_count) : (i += 1) { + const n: *const Node = nodes[i]; + var fp: c.cbm_minhash_t = undefined; + if (!parseFpFromProps(n.properties_json, &fp)) continue; + if (count >= cap) { + const new_cap: c_int = if (cap < FP_ENTRY_INIT_CAP) FP_ENTRY_INIT_CAP else cap * FP_ENTRY_GROW; + const grown = interop.reallocN(FpEntry, entries, ix(new_cap)); + // The C loop broke out of this label only; the next label's + // first fingerprint then failed the same realloc. + if (grown == null) continue :outer; + entries = grown; + cap = new_cap; + } + entries[ix(count)] = .{ + .node_id = n.id, + .fp = fp, + .file_path = n.file_path, + .ext = c.cbm_path_last_dot(n.file_path), + .qn = n.qualified_name, + }; + count += 1; + } + } + // Canonicalize (determinism) — see cmpFpEntryByQn. + if (count > 1) { + c.qsort(entries, ix(count), @sizeOf(FpEntry), cmpFpEntryByQn); + } + out_entries.* = entries; + return count; +} + +// ── Parallel query + emit ─────────────────────────────────────────── + +/// Deferred edge record; collected per-worker, merged into gbuf sequentially. +const SimDeferredEdge = extern struct { + source_id: i64, + target_id: i64, + jaccard: f64, + same_file: bool, +}; + +const SimEdgeBuf = extern struct { + edges: [*c]SimDeferredEdge, + count: c_int, + cap: c_int, +}; + +fn simEdgeBufPush(buf: *SimEdgeBuf, src: i64, tgt: i64, jaccard: f64, same_file: bool) void { + if (buf.count >= buf.cap) { + const nc: c_int = if (buf.cap < SIM_EDGE_INIT_CAP) SIM_EDGE_INIT_CAP else buf.cap * SIM_EDGE_GROW; + const grown = interop.reallocN(SimDeferredEdge, buf.edges, ix(nc)); + if (grown == null) return; + buf.edges = grown; + buf.cap = nc; + } + buf.edges[ix(buf.count)] = .{ + .source_id = src, + .target_id = tgt, + .jaccard = jaccard, + .same_file = same_file, + }; + buf.count += 1; +} + +const SimQueryCtx = struct { + entries: [*]const FpEntry, + entry_count: c_int, + lsh: ?*const c.cbm_lsh_index_t, + worker_bufs: [*c]SimEdgeBuf, + next_idx: AtomicInt, + edge_counts: [*c]AtomicInt, // shared atomic array, one per entry +}; + +const SIM_CAND_CAP = 4096; + +fn simQueryWorker(worker_id: c_int, ctx_ptr: ?*anyopaque) callconv(.c) void { + const sc: *SimQueryCtx = @ptrCast(@alignCast(ctx_ptr)); + const my_buf = util.at(sc.worker_bufs, ix(worker_id)); + + // Thread-local candidate buffer (stack-allocated) + var cands: [SIM_CAND_CAP][*c]const c.cbm_lsh_entry_t = undefined; + + while (true) { + const i = sc.next_idx.fetchAdd(1, .monotonic); + if (i >= sc.entry_count) break; + const edge_count = util.at(sc.edge_counts, ix(i)); + + const ec = edge_count.load(.monotonic); + if (ec >= c.CBM_MINHASH_MAX_EDGES_PER_NODE) continue; + + const src = util.at(sc.entries, ix(i)); + const cand_count = c.cbm_lsh_query_into(sc.lsh, &src.fp, &cands, SIM_CAND_CAP); + + var emitted: c_int = 0; + for (cands[0..ix(cand_count)]) |cand_c| { + const cand: *const c.cbm_lsh_entry_t = cand_c; + if (cand.node_id == src.node_id) continue; + if (c.strcmp(src.ext, cand.file_ext) != 0) continue; + // Pair ownership by canonical QN order, not node id: ids are + // assigned in parallel-merge order and vary run to run, which + // flipped which side owned a pair and (with the per-source edge + // cap) flickered the emitted set (determinism). + if (src.qn == null or cand.qualified_name == null or + c.strcmp(src.qn, cand.qualified_name) >= 0) + { + continue; + } + + const cur = edge_count.load(.monotonic); + if (cur + emitted >= c.CBM_MINHASH_MAX_EDGES_PER_NODE) break; + + const jaccard = c.cbm_minhash_jaccard(&src.fp, cand.fingerprint); + if (jaccard < c.CBM_MINHASH_JACCARD_THRESHOLD) continue; + + const same_file = src.file_path != null and cand.file_path != null and + util.streq(src.file_path, cand.file_path); + simEdgeBufPush(my_buf, src.node_id, cand.node_id, jaccard, same_file); + emitted += 1; + } + if (emitted > 0) { + _ = edge_count.fetchAdd(emitted, .monotonic); + } + } +} + +/// Merge worker edge buffers into gbuf. Returns total edge count. Frees worker buffers. +fn mergeSimEdges(gbuf: ?*c.cbm_gbuf_t, worker_bufs: [*c]SimEdgeBuf, worker_count: c_int) c_int { + var total: c_int = 0; + for (0..ix(worker_count)) |w| { + for (0..ix(worker_bufs[w].count)) |e| { + const de = util.at(worker_bufs[w].edges, e); + var props: [PROPS_BUF_LEN]u8 = undefined; + _ = c.snprintf( + &props, + props.len, + "{\"jaccard\":%.3f,\"same_file\":%s}", + de.jaccard, + @as(CStr, if (de.same_file) "true" else "false"), + ); + _ = c.cbm_gbuf_insert_edge(gbuf, de.source_id, de.target_id, "SIMILAR_TO", &props); + total += 1; + } + c.free(worker_bufs[w].edges); + } + c.free(worker_bufs); + return total; +} + +// ── Pass entry point ──────────────────────────────────────────────── + +pub export fn cbm_pipeline_pass_similarity(ctx: *c.cbm_pipeline_ctx_t) c_int { + interop.logInfo("pass.start", .{ @as(CStr, "pass"), @as(CStr, "similarity") }); + + const gbuf = ctx.gbuf; + + // Phase 1: Collect fingerprints from Function/Method nodes + const t_collect = util.profStart(); + var entries: [*c]FpEntry = null; + const entry_count = collectFpEntries(gbuf, &entries); + util.profEnd("similarity", "1_collect_fp", &t_collect, entry_count); + + interop.logInfo("pass.similarity.collected", .{ + @as(CStr, "nodes_with_fp"), @as(CStr, c.cbm_log_itoa(entry_count)), + }); + + if (entry_count < MIN_FP_ENTRIES) { + c.free(entries); + interop.logInfo("pass.done", .{ + @as(CStr, "pass"), @as(CStr, "similarity"), @as(CStr, "edges"), @as(CStr, "0"), + }); + return 0; + } + + // Phase 2: Build LSH index (sequential — cbm_lsh_insert mutates shared state) + const t_lsh_build = util.profStart(); + const lsh = c.cbm_lsh_new(); + const lsh_entries = interop.mallocN(c.cbm_lsh_entry_t, ix(entry_count)); + if (lsh_entries == null) { + c.free(entries); + c.cbm_lsh_free(lsh); + return c.CBM_NOT_FOUND; + } + + for (0..ix(entry_count)) |i| { + lsh_entries[i] = .{ + .node_id = entries[i].node_id, + .fingerprint = &util.at(entries, i).fp, + .file_path = entries[i].file_path, + .file_ext = entries[i].ext, + .qualified_name = entries[i].qn, + }; + c.cbm_lsh_insert(lsh, util.at(lsh_entries, i)); + } + util.profEnd("similarity", "2_lsh_build_seq", &t_lsh_build, entry_count); + + // Phase 3: Query LSH + emit edges (PARALLEL via cbm_lsh_query_into). + // Each worker claims entries, queries, scores candidates, stashes edges + // in its own deferred buffer. Shared edge_counts is atomic. + // Final merge into gbuf is sequential (gbuf not thread-safe). + const t_query_emit = util.profStart(); + const edge_counts = interop.callocN(AtomicInt, ix(entry_count)); + const worker_count = c.cbm_default_worker_count(false); + const worker_bufs = interop.callocN(SimEdgeBuf, ix(worker_count)); + + { + var sc: SimQueryCtx = .{ + .entries = entries, + .entry_count = entry_count, + .lsh = lsh, + .worker_bufs = worker_bufs, + .next_idx = .init(0), + .edge_counts = edge_counts, + }; + const opts: c.cbm_parallel_for_opts_t = .{ .max_workers = worker_count, .force_pthreads = false }; + c.cbm_parallel_for(worker_count, simQueryWorker, &sc, opts); + } + util.profEnd("similarity", "3_query_parallel", &t_query_emit, entry_count); + + const t_merge = util.profStart(); + const total_edges = mergeSimEdges(gbuf, worker_bufs, worker_count); + util.profEnd("similarity", "4_edge_merge_seq", &t_merge, total_edges); + + interop.logInfo("pass.done", .{ + @as(CStr, "pass"), @as(CStr, "similarity"), + @as(CStr, "edges"), @as(CStr, c.cbm_log_itoa(total_edges)), + }); + + c.free(edge_counts); + c.free(lsh_entries); + c.free(entries); + c.cbm_lsh_free(lsh); + return 0; +} diff --git a/src/pipeline/pass_tests.c b/src/pipeline/pass_tests.c deleted file mode 100644 index 28a1d1cd..00000000 --- a/src/pipeline/pass_tests.c +++ /dev/null @@ -1,286 +0,0 @@ -/* - * pass_tests.c — Create TESTS edges from test functions to production functions. - * - * Scans CALLS edges in the graph buffer: if the source function has - * is_test=true and the target does not, creates a TESTS edge. - * - * Also creates TESTS_FILE edges from test File nodes to the production - * File nodes they correspond to (naming convention: _test.go → .go, etc.) - * - * Depends on: the call resolver having populated CALLS edges - */ -#include "foundation/constants.h" - -enum { - PT_TEST_LEN = 4, /* strlen("Test") */ - PT_DESCRIBE_LEN = 9, /* strlen("Describe") + 1... actually strlen("describe_") */ - PT_CONTEXT_LEN = 7, /* strlen("context") */ - PT_SUFFIX_LEN = 8, /* strlen("_test.go") */ - PT_EXT_PY = 3, /* strlen(".py") / strlen(".go") */ - PT_PYTEST_PREFIX = 5, /* strlen("test_") */ - PT_SEARCH_COUNT = 2, /* number of search passes */ - PT_FOUND_SKIP = 5, /* strlen("_test") + 1 or strlen("_spec") + 1 */ -}; - -#define SLEN(s) (sizeof(s) - 1) -#include "pipeline/pipeline.h" -#include "pipeline/pipeline_internal.h" -#include "graph_buffer/graph_buffer.h" -#include "foundation/compat.h" -#include "foundation/log.h" - -#include -#include -#include - -/* Check if a node has is_test:true in its properties JSON. */ -static bool node_is_test(const cbm_gbuf_node_t *n) { - if (!n || !n->properties_json) { - return false; - } - return strstr(n->properties_json, "\"is_test\":true") != NULL; -} - -/* Helper to check suffix. */ -static bool str_ends_with(const char *s, size_t slen, const char *suffix) { - size_t sflen = strlen(suffix); - return slen >= sflen && strcmp(s + slen - sflen, suffix) == 0; -} - -/* Check if a file path looks like a test file (language-agnostic). */ -bool cbm_is_test_path(const char *path) { - if (!path) { - return false; - } - const char *base = strrchr(path, '/'); - base = base ? base + SKIP_ONE : path; - size_t len = strlen(path); - - /* Prefix-based: Python test_*.py */ - if (strncmp(base, "test_", SLEN("test_")) == 0) { - return true; - } - - /* Suffix-based: _test. pattern (Go, Python, Rust, C++, Lua) */ - if (str_ends_with(path, len, "_test.go") || str_ends_with(path, len, "_test.py") || - str_ends_with(path, len, "_test.rs") || str_ends_with(path, len, "_test.cpp") || - str_ends_with(path, len, "_test.lua")) { - return true; - } - - /* .test. / .spec. pattern (JS/TS/TSX) */ - if (strstr(path, ".test.ts") || strstr(path, ".spec.ts") || strstr(path, ".test.js") || - strstr(path, ".spec.js") || strstr(path, ".test.tsx") || strstr(path, ".spec.tsx")) { - return true; - } - - /* Name ends with "Test" or "Spec" before extension (Java, Kotlin, C#, PHP, Scala) */ - if (str_ends_with(path, len, "Test.java") || str_ends_with(path, len, "Test.kt") || - str_ends_with(path, len, "Test.cs") || str_ends_with(path, len, "Test.php") || - str_ends_with(path, len, "Spec.scala")) { - return true; - } - - /* Directory-based: __tests__/, tests/, test/, spec/ */ - if (strstr(path, "__tests__/") || strstr(path, "/tests/") || strstr(path, "/test/") || - strstr(path, "/spec/")) { - return true; - } - /* Also match if path STARTS with these directories */ - if (strncmp(path, "tests/", SLEN("tests/")) == 0 || - strncmp(path, "test/", SLEN("test/")) == 0 || strncmp(path, "spec/", SLEN("spec/")) == 0 || - strncmp(path, "__tests__/", SLEN("__tests__/")) == 0) { - return true; - } - - /* Ruby: _spec.rb suffix */ - if (str_ends_with(path, len, "_spec.rb")) { - return true; - } - - return false; -} - -/* Check if a function name looks like a test function (language-agnostic). */ -bool cbm_is_test_func_name(const char *name) { - if (!name) { - return false; - } - /* Go: Test/Benchmark/Example + uppercase letter or end-of-string. - * "TestFoo" = test, "Testable" = not test (lowercase after prefix). */ - if (strncmp(name, "Test", SLEN("Test")) == 0 && - (name[PT_TEST_LEN] == '\0' || (name[PT_TEST_LEN] >= 'A' && name[PT_TEST_LEN] <= 'Z'))) { - return true; - } - if (strncmp(name, "Benchmark", SLEN("Benchmark")) == 0 && - (name[PT_DESCRIBE_LEN] == '\0' || - (name[PT_DESCRIBE_LEN] >= 'A' && name[PT_DESCRIBE_LEN] <= 'Z'))) { - return true; - } - if (strncmp(name, "Example", SLEN("Example")) == 0 && - (name[PT_CONTEXT_LEN] == '\0' || - (name[PT_CONTEXT_LEN] >= 'A' && name[PT_CONTEXT_LEN] <= 'Z'))) { - return true; - } - /* Python/Rust/C++/Lua/Java: test_ or test prefix (lowercase) */ - if (strncmp(name, "test_", SLEN("test_")) == 0) { - return true; - } - if (strncmp(name, "test", SLEN("test")) == 0 && name[PT_TEST_LEN] >= 'A' && - name[PT_TEST_LEN] <= 'Z') { - return true; - } - /* JS/TS: test/it/describe + lifecycle hooks */ - if (strcmp(name, "test") == 0 || strcmp(name, "it") == 0 || strcmp(name, "describe") == 0 || - strcmp(name, "beforeAll") == 0 || strcmp(name, "afterAll") == 0 || - strcmp(name, "beforeEach") == 0 || strcmp(name, "afterEach") == 0) { - return true; - } - /* Julia: @testset, @test */ - if (strcmp(name, "@testset") == 0 || strcmp(name, "@test") == 0) { - return true; - } - return false; -} - -/* Assemble dir/name+ext into a heap-allocated path. */ -static char *build_path(const char *test_path, size_t dir_len, const char *name, size_t name_len, - const char *ext, size_t ext_len) { - char *result = malloc(dir_len + SKIP_ONE + name_len + ext_len + SKIP_ONE); - if (dir_len > 0) { - memcpy(result, test_path, dir_len); - result[dir_len] = '/'; - memcpy(result + dir_len + SKIP_ONE, name, name_len); - memcpy(result + dir_len + SKIP_ONE + name_len, ext, ext_len + SKIP_ONE); - } else { - memcpy(result, name, name_len); - memcpy(result + name_len, ext, ext_len + SKIP_ONE); - } - return result; -} - -/* Try to derive the production file path from a test file path. - * Returns heap-allocated string or NULL. Caller must free(). */ -static char *test_to_prod_path(const char *test_path) { - if (!test_path) { - return NULL; - } - - const char *base = strrchr(test_path, '/'); - size_t dir_len = base ? (size_t)(base - test_path) : 0; - base = base ? base + SKIP_ONE : test_path; - - /* Go: foo_test.go → foo.go */ - const char *suffix = strstr(base, "_test.go"); - if (suffix && suffix[PT_SUFFIX_LEN] == '\0') { - return build_path(test_path, dir_len, base, (size_t)(suffix - base), ".go", PT_EXT_PY); - } - - /* Python: test_foo.py → foo.py */ - if (strncmp(base, "test_", SLEN("test_")) == 0) { - const char *ext = strstr(base, ".py"); - if (ext && ext[PT_EXT_PY] == '\0') { - return build_path(test_path, dir_len, base + PT_PYTEST_PREFIX, - (size_t)(ext - base - PT_PYTEST_PREFIX), ".py", PT_EXT_PY); - } - } - - /* JS/TS: foo.test.ts → foo.ts, foo.spec.ts → foo.ts */ - for (int s = 0; s < PT_SEARCH_COUNT; s++) { - const char *pat = s == 0 ? ".test." : ".spec."; - const char *found = strstr(base, pat); - if (found) { - const char *ext = found + PT_FOUND_SKIP; - return build_path(test_path, dir_len, base, (size_t)(found - base), ext, strlen(ext)); - } - } - - return NULL; -} - -/* Create TESTS edges from test functions to production functions via CALLS edges. */ -static int create_tests_edges(cbm_pipeline_ctx_t *ctx) { - int count = 0; - const cbm_gbuf_edge_t **call_edges = NULL; - int call_count = 0; - int rc = cbm_gbuf_find_edges_by_type(ctx->gbuf, "CALLS", &call_edges, &call_count); - if (rc != 0 || call_count == 0) { - return 0; - } - - for (int i = 0; i < call_count; i++) { - const cbm_gbuf_edge_t *e = call_edges[i]; - const cbm_gbuf_node_t *src = cbm_gbuf_find_by_id(ctx->gbuf, e->source_id); - const cbm_gbuf_node_t *tgt = cbm_gbuf_find_by_id(ctx->gbuf, e->target_id); - if (!src || !tgt) { - continue; - } - - bool src_is_test = - node_is_test(src) || (src->file_path && cbm_is_test_path(src->file_path)); - if (!src_is_test) { - continue; - } - - bool tgt_is_test = - node_is_test(tgt) || (tgt->file_path && cbm_is_test_path(tgt->file_path)); - if (tgt_is_test) { - continue; - } - - if (!cbm_is_test_func_name(src->name)) { - continue; - } - - cbm_gbuf_insert_edge(ctx->gbuf, src->id, tgt->id, "TESTS", "{}"); - count++; - } - return count; -} - -/* Create TESTS_FILE edges from test File nodes to production File nodes. */ -static int create_tests_file_edges(cbm_pipeline_ctx_t *ctx) { - int count = 0; - const cbm_gbuf_node_t **file_nodes = NULL; - int file_node_count = 0; - int rc = cbm_gbuf_find_by_label(ctx->gbuf, "File", &file_nodes, &file_node_count); - if (rc != 0) { - return 0; - } - - for (int i = 0; i < file_node_count; i++) { - const cbm_gbuf_node_t *fnode = file_nodes[i]; - if (!fnode->file_path || !cbm_is_test_path(fnode->file_path)) { - continue; - } - - char *prod_path = test_to_prod_path(fnode->file_path); - if (!prod_path) { - continue; - } - - char *prod_qn = cbm_pipeline_fqn_compute(ctx->project_name, prod_path, "__file__"); - const cbm_gbuf_node_t *prod_node = cbm_gbuf_find_by_qn(ctx->gbuf, prod_qn); - free(prod_qn); - free(prod_path); - - if (prod_node && fnode->id != prod_node->id) { - cbm_gbuf_insert_edge(ctx->gbuf, fnode->id, prod_node->id, "TESTS_FILE", "{}"); - count++; - } - } - return count; -} - -int cbm_pipeline_pass_tests(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count) { - cbm_log_info("pass.start", "pass", "tests"); - (void)files; - (void)file_count; - - int tests_count = create_tests_edges(ctx); - int tests_file_count = create_tests_file_edges(ctx); - - cbm_log_info("pass.done", "pass", "tests", "tests", cbm_log_itoa(tests_count), "tests_file", - cbm_log_itoa(tests_file_count)); - return 0; -} diff --git a/src/pipeline/pass_tests.zig b/src/pipeline/pass_tests.zig new file mode 100644 index 00000000..571916bd --- /dev/null +++ b/src/pipeline/pass_tests.zig @@ -0,0 +1,259 @@ +//! pass_tests.zig — Create TESTS edges from test functions to production functions. +//! +//! Scans CALLS edges in the graph buffer: if the source function has +//! is_test=true and the target does not, creates a TESTS edge. +//! +//! Also creates TESTS_FILE edges from test File nodes to the production +//! File nodes they correspond to (naming convention: _test.go → .go, etc.) +//! +//! Depends on: the call resolver having populated CALLS edges + +const std = @import("std"); +const c = @import("c"); +const interop = @import("../foundation/interop.zig"); +const util = @import("util.zig"); + +const CStr = util.CStr; +const MutStr = util.MutStr; +const streq = util.streq; +const Node = c.cbm_gbuf_node_t; +const Edge = c.cbm_gbuf_edge_t; + +const PT_TEST_LEN = 4; // strlen("Test") +const PT_BENCHMARK_LEN = 9; // strlen("Benchmark") +const PT_EXAMPLE_LEN = 7; // strlen("Example") +const PT_SUFFIX_LEN = 8; // strlen("_test.go") +const PT_EXT_PY = 3; // strlen(".py") / strlen(".go") +const PT_PYTEST_PREFIX = 5; // strlen("test_") +const PT_FOUND_SKIP = 5; // strlen(".test") or strlen(".spec") + +fn hasPrefix(s: CStr, comptime prefix: [:0]const u8) bool { + return c.strncmp(s, prefix, prefix.len) == 0; +} + +/// Check if a node has is_test:true in its properties JSON. +fn nodeIsTest(n: *const Node) bool { + if (n.properties_json == null) return false; + return c.strstr(n.properties_json, "\"is_test\":true") != null; +} + +/// Helper to check suffix. +fn strEndsWith(s: CStr, slen: usize, suffix: [*:0]const u8) bool { + const sflen = c.strlen(suffix); + return slen >= sflen and streq(s + slen - sflen, suffix); +} + +/// Check if a file path looks like a test file (language-agnostic). +pub export fn cbm_is_test_path(path: CStr) bool { + if (path == null) return false; + var base: CStr = c.strrchr(path, '/'); + base = if (base != null) base + 1 else path; + const len = c.strlen(path); + + // Prefix-based: Python test_*.py + if (hasPrefix(base, "test_")) return true; + + // Suffix-based: _test. pattern (Go, Python, Rust, C++, Lua) + if (strEndsWith(path, len, "_test.go") or strEndsWith(path, len, "_test.py") or + strEndsWith(path, len, "_test.rs") or strEndsWith(path, len, "_test.cpp") or + strEndsWith(path, len, "_test.lua")) + { + return true; + } + + // .test. / .spec. pattern (JS/TS/TSX) + if (c.strstr(path, ".test.ts") != null or c.strstr(path, ".spec.ts") != null or + c.strstr(path, ".test.js") != null or c.strstr(path, ".spec.js") != null or + c.strstr(path, ".test.tsx") != null or c.strstr(path, ".spec.tsx") != null) + { + return true; + } + + // Name ends with "Test" or "Spec" before extension (Java, Kotlin, C#, PHP, Scala) + if (strEndsWith(path, len, "Test.java") or strEndsWith(path, len, "Test.kt") or + strEndsWith(path, len, "Test.cs") or strEndsWith(path, len, "Test.php") or + strEndsWith(path, len, "Spec.scala")) + { + return true; + } + + // Directory-based: __tests__/, tests/, test/, spec/ + if (c.strstr(path, "__tests__/") != null or c.strstr(path, "/tests/") != null or + c.strstr(path, "/test/") != null or c.strstr(path, "/spec/") != null) + { + return true; + } + // Also match if path STARTS with these directories + if (hasPrefix(path, "tests/") or hasPrefix(path, "test/") or hasPrefix(path, "spec/") or + hasPrefix(path, "__tests__/")) + { + return true; + } + + // Ruby: _spec.rb suffix + if (strEndsWith(path, len, "_spec.rb")) return true; + + return false; +} + +fn isUpper(ch: u8) bool { + return ch >= 'A' and ch <= 'Z'; +} + +/// Check if a function name looks like a test function (language-agnostic). +pub export fn cbm_is_test_func_name(name: CStr) bool { + if (name == null) return false; + // Go: Test/Benchmark/Example + uppercase letter or end-of-string. + // "TestFoo" = test, "Testable" = not test (lowercase after prefix). + if (hasPrefix(name, "Test") and (name[PT_TEST_LEN] == 0 or isUpper(name[PT_TEST_LEN]))) { + return true; + } + if (hasPrefix(name, "Benchmark") and + (name[PT_BENCHMARK_LEN] == 0 or isUpper(name[PT_BENCHMARK_LEN]))) + { + return true; + } + if (hasPrefix(name, "Example") and + (name[PT_EXAMPLE_LEN] == 0 or isUpper(name[PT_EXAMPLE_LEN]))) + { + return true; + } + // Python/Rust/C++/Lua/Java: test_ or test prefix (lowercase) + if (hasPrefix(name, "test_")) return true; + if (hasPrefix(name, "test") and isUpper(name[PT_TEST_LEN])) return true; + // JS/TS: test/it/describe + lifecycle hooks + if (streq(name, "test") or streq(name, "it") or streq(name, "describe") or + streq(name, "beforeAll") or streq(name, "afterAll") or + streq(name, "beforeEach") or streq(name, "afterEach")) + { + return true; + } + // Julia: @testset, @test + if (streq(name, "@testset") or streq(name, "@test")) return true; + return false; +} + +/// Assemble dir/name+ext into a heap-allocated path. +fn buildPath(test_path: CStr, dir_len: usize, name: CStr, name_len: usize, ext: CStr, ext_len: usize) MutStr { + const result: MutStr = @ptrCast(c.malloc(dir_len + 1 + name_len + ext_len + 1)); + if (dir_len > 0) { + @memcpy(result[0..dir_len], test_path[0..dir_len]); + result[dir_len] = '/'; + @memcpy(result[dir_len + 1 ..][0..name_len], name[0..name_len]); + @memcpy(result[dir_len + 1 + name_len ..][0 .. ext_len + 1], ext[0 .. ext_len + 1]); + } else { + @memcpy(result[0..name_len], name[0..name_len]); + @memcpy(result[name_len..][0 .. ext_len + 1], ext[0 .. ext_len + 1]); + } + return result; +} + +/// Try to derive the production file path from a test file path. +/// Returns heap-allocated string or NULL. Caller must free(). +fn testToProdPath(test_path: CStr) MutStr { + if (test_path == null) return null; + + var base: CStr = c.strrchr(test_path, '/'); + const dir_len: usize = if (base != null) util.ptrDiff(base, test_path) else 0; + base = if (base != null) base + 1 else test_path; + + // Go: foo_test.go → foo.go + const suffix: CStr = c.strstr(base, "_test.go"); + if (suffix != null and suffix[PT_SUFFIX_LEN] == 0) { + return buildPath(test_path, dir_len, base, util.ptrDiff(suffix, base), ".go", PT_EXT_PY); + } + + // Python: test_foo.py → foo.py + if (hasPrefix(base, "test_")) { + const ext: CStr = c.strstr(base, ".py"); + if (ext != null and ext[PT_EXT_PY] == 0) { + return buildPath(test_path, dir_len, base + PT_PYTEST_PREFIX, util.ptrDiff(ext, base) -% PT_PYTEST_PREFIX, ".py", PT_EXT_PY); + } + } + + // JS/TS: foo.test.ts → foo.ts, foo.spec.ts → foo.ts + for ([_][*:0]const u8{ ".test.", ".spec." }) |pat| { + const found: CStr = c.strstr(base, pat); + if (found != null) { + const ext: CStr = found + PT_FOUND_SKIP; + return buildPath(test_path, dir_len, base, util.ptrDiff(found, base), ext, c.strlen(ext)); + } + } + + return null; +} + +fn nodeLooksLikeTest(n: *const Node) bool { + return nodeIsTest(n) or (n.file_path != null and cbm_is_test_path(n.file_path)); +} + +/// Create TESTS edges from test functions to production functions via CALLS edges. +fn createTestsEdges(ctx: *c.cbm_pipeline_ctx_t) c_int { + var count: c_int = 0; + var call_edges: [*c][*c]const Edge = null; + var call_count: c_int = 0; + const rc = c.cbm_gbuf_find_edges_by_type(ctx.gbuf, "CALLS", &call_edges, &call_count); + if (rc != 0 or call_count == 0) return 0; + + var i: usize = 0; + while (i < call_count) : (i += 1) { + const e: *const Edge = call_edges[i]; + const src: *const Node = @as(?*const Node, c.cbm_gbuf_find_by_id(ctx.gbuf, e.source_id)) orelse continue; + const tgt: *const Node = @as(?*const Node, c.cbm_gbuf_find_by_id(ctx.gbuf, e.target_id)) orelse continue; + + if (!nodeLooksLikeTest(src)) continue; + if (nodeLooksLikeTest(tgt)) continue; + if (!cbm_is_test_func_name(src.name)) continue; + + _ = c.cbm_gbuf_insert_edge(ctx.gbuf, src.id, tgt.id, "TESTS", "{}"); + count += 1; + } + return count; +} + +/// Create TESTS_FILE edges from test File nodes to production File nodes. +fn createTestsFileEdges(ctx: *c.cbm_pipeline_ctx_t) c_int { + var count: c_int = 0; + var file_nodes: [*c][*c]const Node = null; + var file_node_count: c_int = 0; + const rc = c.cbm_gbuf_find_by_label(ctx.gbuf, "File", &file_nodes, &file_node_count); + if (rc != 0) return 0; + + var i: usize = 0; + while (i < file_node_count) : (i += 1) { + const fnode: *const Node = file_nodes[i]; + if (fnode.file_path == null or !cbm_is_test_path(fnode.file_path)) continue; + + const prod_path = testToProdPath(fnode.file_path); + if (prod_path == null) continue; + + const prod_qn = c.cbm_pipeline_fqn_compute(ctx.project_name, prod_path, "__file__"); + const prod_node: ?*const Node = c.cbm_gbuf_find_by_qn(ctx.gbuf, prod_qn); + c.free(prod_qn); + c.free(prod_path); + + if (prod_node) |pn| { + if (fnode.id != pn.id) { + _ = c.cbm_gbuf_insert_edge(ctx.gbuf, fnode.id, pn.id, "TESTS_FILE", "{}"); + count += 1; + } + } + } + return count; +} + +pub export fn cbm_pipeline_pass_tests(ctx: *c.cbm_pipeline_ctx_t, files: ?*const c.cbm_file_info_t, file_count: c_int) c_int { + interop.logInfo("pass.start", .{ @as(CStr, "pass"), @as(CStr, "tests") }); + _ = files; + _ = file_count; + + const tests_count = createTestsEdges(ctx); + const tests_file_count = createTestsFileEdges(ctx); + + interop.logInfo("pass.done", .{ + @as(CStr, "pass"), @as(CStr, "tests"), + @as(CStr, "tests"), @as(CStr, c.cbm_log_itoa(tests_count)), + @as(CStr, "tests_file"), @as(CStr, c.cbm_log_itoa(tests_file_count)), + }); + return 0; +} diff --git a/src/pipeline/path_alias.c b/src/pipeline/path_alias.c deleted file mode 100644 index fff5bf05..00000000 --- a/src/pipeline/path_alias.c +++ /dev/null @@ -1,517 +0,0 @@ -/* - * path_alias.c — Resolve build-tool path aliases. - * - * Builds a directory-scoped collection of alias maps from per-language - * config files (currently tsconfig.json / jsconfig.json) so the import - * resolver can turn "@/lib/auth"-style imports into repo-relative paths. - * - * Design notes: - * - Public types and functions are language-agnostic. Adding a Vite / - * Webpack / Python loader means writing a new load_*_file() helper - * and registering it in find_alias_files. The resolver, the - * collection, and the pipeline integration do not change. - * - Sorting uses qsort (n log n). The bubble-sorts that the original - * Layer 1b draft used were O(n^2); with up to 256 alias entries - * and 256 scoped maps per repo, qsort is the right ceiling. - * - The repo walk caps recursion depth and total file count and emits - * a warning when either cap fires, so silent truncation on - * pathological monorepos shows up in the index log. - */ - -#include "pipeline/path_alias.h" - -#include "pipeline/pipeline_internal.h" - -#include "foundation/compat.h" -#include "foundation/compat_fs.h" -#include "foundation/constants.h" -#include "foundation/log.h" -#include "foundation/platform.h" - -#include -#include -#include -#include -#include -#include - -/* Resource ceilings. Chosen to comfortably cover real-world monorepos - * (Next.js Skyline, large nx workspaces) while bounding worst-case - * memory and walk time. Cap hits are logged. */ -enum { - CBM_PATH_ALIAS_MAX_ENTRIES = 256, /* per single config file */ - CBM_PATH_ALIAS_MAX_FILES = 256, /* config files per repo walk */ - CBM_PATH_ALIAS_MAX_FILE_BYTES = 64 * 1024, - CBM_PATH_ALIAS_MAX_DEPTH = 32, /* directory recursion depth */ -}; - -/* ── Helpers ───────────────────────────────────────────────────── */ - -/* Strip .ts/.tsx/.js/.jsx in place. Returns its argument. */ -static char *strip_resolved_ext(char *path) { - if (!path) { - return path; - } - size_t len = strlen(path); - if (len > 3 && path[len - 3] == '.' && (path[len - 2] == 't' || path[len - 2] == 'j') && - path[len - 1] == 's') { - path[len - 3] = '\0'; - return path; - } - if (len > 4 && path[len - 4] == '.' && (path[len - 3] == 't' || path[len - 3] == 'j') && - path[len - 2] == 's' && path[len - 1] == 'x') { - path[len - 4] = '\0'; - } - return path; -} - -/* Join dir_prefix with target, collapsing "." and ".." segments so aliases - * that climb out of their tsconfig's directory (the common monorepo - * pattern: a tsconfig at apps/web/tsconfig.json pointing an alias at a - * wildcard target like "../../packages/shared/src/" + wildcard) resolve - * to a real repo-relative path. Naive concatenation left literal ".." - * components in the target, which never match a module's FQN since - * cbm_pipeline_fqn_module tokenizes on '/' without collapsing them - * (#730). A trailing '/' on target (the usual case right before a - * wildcard) is preserved so the caller's later wildcard-substring - * concat still lines up. Returns heap-allocated - * repo-relative target. */ -static char *resolve_target_relative(const char *dir_prefix, const char *target) { - if (!target) { - return NULL; - } - size_t dp_len = (dir_prefix && dir_prefix[0] != '\0') ? strlen(dir_prefix) : 0; - size_t t_len = strlen(target); - char *buf = malloc(dp_len + t_len + 2); - if (!buf) { - return NULL; - } - buf[0] = '\0'; - if (dp_len > 0) { - memcpy(buf, dir_prefix, dp_len); - buf[dp_len] = '\0'; - } - - bool trailing_slash = t_len > 0 && target[t_len - 1] == '/'; - - const char *p = target; - while (*p) { - while (*p == '/') { - p++; - } - if (!*p) { - break; - } - const char *seg_start = p; - while (*p && *p != '/') { - p++; - } - size_t seg_len = (size_t)(p - seg_start); - if (seg_len == 1 && seg_start[0] == '.') { - continue; - } - if (seg_len == 2 && seg_start[0] == '.' && seg_start[1] == '.') { - char *last = strrchr(buf, '/'); - if (last) { - *last = '\0'; - } else { - buf[0] = '\0'; - } - continue; - } - size_t cur = strlen(buf); - if (cur > 0) { - buf[cur++] = '/'; - } - memcpy(buf + cur, seg_start, seg_len); - buf[cur + seg_len] = '\0'; - } - - if (trailing_slash) { - size_t cur = strlen(buf); - buf[cur] = '/'; - buf[cur + 1] = '\0'; - } - return buf; -} - -/* qsort comparator: alias entries by alias_prefix length, descending. */ -static int cmp_alias_entry_by_specificity(const void *a, const void *b) { - const cbm_path_alias_t *ea = a; - const cbm_path_alias_t *eb = b; - size_t la = strlen(ea->alias_prefix); - size_t lb = strlen(eb->alias_prefix); - if (lb > la) { - return 1; - } - if (lb < la) { - return -1; - } - return 0; -} - -/* qsort comparator: scopes by dir_prefix length, descending. */ -static int cmp_scope_by_specificity(const void *a, const void *b) { - const cbm_path_alias_scope_t *sa = a; - const cbm_path_alias_scope_t *sb = b; - size_t la = strlen(sa->dir_prefix); - size_t lb = strlen(sb->dir_prefix); - if (lb > la) { - return 1; - } - if (lb < la) { - return -1; - } - return 0; -} - -/* ── tsconfig.json / jsconfig.json loader ──────────────────────── */ - -/* Parse compilerOptions.paths and compilerOptions.baseUrl into an alias map. - * dir_prefix is the directory of the config file relative to the repo root - * (e.g. "apps/manager", or "" for repo root). Returns NULL if the file is - * missing, malformed, or has neither a usable paths block nor a baseUrl. */ -static cbm_path_alias_map_t *load_tsconfig_file(const char *abs_path, const char *dir_prefix) { - FILE *f = cbm_fopen(abs_path, "r"); - if (!f) { - return NULL; - } - fseek(f, 0, SEEK_END); - long len = ftell(f); - fseek(f, 0, SEEK_SET); - if (len <= 0 || len > CBM_PATH_ALIAS_MAX_FILE_BYTES) { - fclose(f); - return NULL; - } - char *buf = malloc((size_t)len + 1); - if (!buf) { - fclose(f); - return NULL; - } - size_t nread = fread(buf, 1, (size_t)len, f); - fclose(f); - buf[nread] = '\0'; - - yyjson_read_flag flg = YYJSON_READ_ALLOW_COMMENTS | YYJSON_READ_ALLOW_TRAILING_COMMAS; - yyjson_doc *doc = yyjson_read(buf, nread, flg); - free(buf); - if (!doc) { - return NULL; - } - yyjson_val *root = yyjson_doc_get_root(doc); - yyjson_val *compiler_opts = yyjson_obj_get(root, "compilerOptions"); - if (!compiler_opts) { - yyjson_doc_free(doc); - return NULL; - } - yyjson_val *base_url_val = yyjson_obj_get(compiler_opts, "baseUrl"); - const char *base_url_str = base_url_val ? yyjson_get_str(base_url_val) : NULL; - yyjson_val *paths_obj = yyjson_obj_get(compiler_opts, "paths"); - if (!paths_obj && !base_url_str) { - yyjson_doc_free(doc); - return NULL; - } - - cbm_path_alias_map_t *map = calloc(1, sizeof(*map)); - if (!map) { - yyjson_doc_free(doc); - return NULL; - } - - if (base_url_str && base_url_str[0] != '\0' && strcmp(base_url_str, ".") != 0) { - map->base_url = resolve_target_relative(dir_prefix, base_url_str); - } else if (base_url_str && strcmp(base_url_str, ".") == 0 && dir_prefix && - dir_prefix[0] != '\0') { - map->base_url = strdup(dir_prefix); - } - - if (paths_obj && yyjson_is_obj(paths_obj)) { - size_t obj_size = yyjson_obj_size(paths_obj); - bool capped = obj_size > CBM_PATH_ALIAS_MAX_ENTRIES; - int capacity = (int)(capped ? (size_t)CBM_PATH_ALIAS_MAX_ENTRIES : obj_size); - if (capacity > 0) { - map->entries = calloc((size_t)capacity, sizeof(cbm_path_alias_t)); - if (!map->entries) { - free(map->base_url); - free(map); - yyjson_doc_free(doc); - return NULL; - } - yyjson_val *key; - yyjson_obj_iter iter = yyjson_obj_iter_with(paths_obj); - while ((key = yyjson_obj_iter_next(&iter)) != NULL && map->count < capacity) { - yyjson_val *val = yyjson_obj_iter_get_val(key); - const char *alias_pattern = yyjson_get_str(key); - if (!alias_pattern || !yyjson_is_arr(val) || yyjson_arr_size(val) == 0) { - continue; - } - const char *target_pattern = yyjson_get_str(yyjson_arr_get_first(val)); - if (!target_pattern) { - continue; - } - cbm_path_alias_t *entry = &map->entries[map->count]; - const char *star = strchr(alias_pattern, '*'); - if (star) { - entry->has_wildcard = true; - entry->alias_prefix = - cbm_strndup(alias_pattern, (size_t)(star - alias_pattern)); - entry->alias_suffix = strdup(star + 1); - } else { - entry->has_wildcard = false; - entry->alias_prefix = strdup(alias_pattern); - entry->alias_suffix = strdup(""); - } - const char *tstar = strchr(target_pattern, '*'); - if (tstar) { - char *pre = cbm_strndup(target_pattern, (size_t)(tstar - target_pattern)); - entry->target_prefix = resolve_target_relative(dir_prefix, pre); - free(pre); - entry->target_suffix = strdup(tstar + 1); - } else { - entry->target_prefix = resolve_target_relative(dir_prefix, target_pattern); - entry->target_suffix = strdup(""); - } - map->count++; - } - if (capped) { - cbm_log_warn("path_alias.entries.cap_hit", "config", abs_path, "kept", - /* itoa via thread-local buffer would be tidier; keep simple */ - "256_of_more"); - } - if (map->count > 1) { - qsort(map->entries, (size_t)map->count, sizeof(cbm_path_alias_t), - cmp_alias_entry_by_specificity); - } - } - } - - yyjson_doc_free(doc); - return map; -} - -/* ── Public API ────────────────────────────────────────────────── */ - -void cbm_path_alias_collection_free(cbm_path_alias_collection_t *coll) { - if (!coll) { - return; - } - for (int i = 0; i < coll->count; i++) { - free(coll->scopes[i].dir_prefix); - if (coll->scopes[i].map) { - cbm_path_alias_map_t *map = coll->scopes[i].map; - for (int j = 0; j < map->count; j++) { - free(map->entries[j].alias_prefix); - free(map->entries[j].alias_suffix); - free(map->entries[j].target_prefix); - free(map->entries[j].target_suffix); - } - free(map->entries); - free(map->base_url); - free(map); - } - } - free(coll->scopes); - free(coll); -} - -char *cbm_path_alias_resolve(const cbm_path_alias_map_t *map, const char *module_path) { - if (!map || !module_path) { - return NULL; - } - size_t mod_len = strlen(module_path); - - for (int i = 0; i < map->count; i++) { - const cbm_path_alias_t *e = &map->entries[i]; - - if (e->has_wildcard) { - size_t prefix_len = strlen(e->alias_prefix); - size_t suffix_len = strlen(e->alias_suffix); - if (mod_len < prefix_len + suffix_len) { - continue; - } - if (strncmp(module_path, e->alias_prefix, prefix_len) != 0) { - continue; - } - if (suffix_len > 0 && - strcmp(module_path + mod_len - suffix_len, e->alias_suffix) != 0) { - continue; - } - size_t wild_len = mod_len - prefix_len - suffix_len; - const char *wild_start = module_path + prefix_len; - size_t tp_len = strlen(e->target_prefix); - size_t ts_len = strlen(e->target_suffix); - char *result = malloc(tp_len + wild_len + ts_len + 1); - if (!result) { - return NULL; - } - memcpy(result, e->target_prefix, tp_len); - memcpy(result + tp_len, wild_start, wild_len); - memcpy(result + tp_len + wild_len, e->target_suffix, ts_len); - result[tp_len + wild_len + ts_len] = '\0'; - return strip_resolved_ext(result); - } - - if (strcmp(module_path, e->alias_prefix) == 0) { - return strip_resolved_ext(strdup(e->target_prefix)); - } - } - - /* baseUrl fallback. Apply only to non-relative imports that look - * sub-path-ish (contain '/' but don't start with '.' or '@'); skips - * obvious package names like "react" or "lodash". */ - if (map->base_url && module_path[0] != '.' && module_path[0] != '@' && - strchr(module_path, '/') != NULL) { - size_t bu_len = strlen(map->base_url); - size_t need = bu_len + 1 + mod_len + 1; - char *result = malloc(need); - if (!result) { - return NULL; - } - snprintf(result, need, "%s/%s", map->base_url, module_path); - return strip_resolved_ext(result); - } - return NULL; -} - -/* ── Repo walk ─────────────────────────────────────────────────── */ - -typedef struct { - char abs[CBM_SZ_512]; - char rel[CBM_SZ_256]; -} alias_config_hit_t; - -static const char *const TS_CONFIG_NAMES[] = {"tsconfig.json", "jsconfig.json"}; -enum { TS_CONFIG_NAMES_COUNT = 2 }; - -static void find_alias_files(const char *abs_dir, const char *rel_dir, alias_config_hit_t *out, - int *count, int max_count, int depth, char **excluded_dirs, - int excluded_count) { - if (*count >= max_count || depth > CBM_PATH_ALIAS_MAX_DEPTH) { - return; - } - cbm_dir_t *d = cbm_opendir(abs_dir); - if (!d) { - return; - } - - /* One config file per directory: prefer tsconfig.json over jsconfig.json. */ - for (int i = 0; i < TS_CONFIG_NAMES_COUNT && *count < max_count; i++) { - char check[CBM_SZ_512]; - snprintf(check, sizeof(check), "%s/%s", abs_dir, TS_CONFIG_NAMES[i]); - FILE *f = cbm_fopen(check, "r"); - if (f) { - fclose(f); - snprintf(out[*count].abs, sizeof(out[*count].abs), "%s", check); - snprintf(out[*count].rel, sizeof(out[*count].rel), "%s", rel_dir); - (*count)++; - break; - } - } - - cbm_dirent_t *ent; - while ((ent = cbm_readdir(d)) != NULL && *count < max_count) { - if (!ent->is_dir) { - continue; - } - const char *name = ent->name; - if (name[0] == '.' || strcmp(name, "node_modules") == 0 || strcmp(name, "dist") == 0 || - strcmp(name, "build") == 0 || strcmp(name, ".next") == 0 || - strcmp(name, "coverage") == 0 || strcmp(name, "target") == 0 /* Rust */) { - continue; - } - char child_abs[CBM_SZ_512]; - char child_rel[CBM_SZ_256]; - snprintf(child_abs, sizeof(child_abs), "%s/%s", abs_dir, name); - if (rel_dir[0] == '\0') { - snprintf(child_rel, sizeof(child_rel), "%s", name); - } else { - snprintf(child_rel, sizeof(child_rel), "%s/%s", rel_dir, name); - } - if (cbm_pipeline_relpath_is_excluded(child_rel, excluded_dirs, excluded_count)) { - continue; - } - find_alias_files(child_abs, child_rel, out, count, max_count, depth + 1, excluded_dirs, - excluded_count); - } - cbm_closedir(d); -} - -cbm_path_alias_collection_t *cbm_load_path_aliases_excluded(const char *repo_path, - char **excluded_dirs, - int excluded_count) { - if (!repo_path) { - return NULL; - } - alias_config_hit_t *hits = calloc(CBM_PATH_ALIAS_MAX_FILES, sizeof(*hits)); - if (!hits) { - return NULL; - } - int count = 0; - find_alias_files(repo_path, "", hits, &count, CBM_PATH_ALIAS_MAX_FILES, 0, excluded_dirs, - excluded_count); - if (count >= CBM_PATH_ALIAS_MAX_FILES) { - cbm_log_warn("path_alias.files.cap_hit", "repo", repo_path, "kept", "256_of_more"); - } - if (count == 0) { - free(hits); - return NULL; - } - - cbm_path_alias_collection_t *coll = calloc(1, sizeof(*coll)); - if (!coll) { - free(hits); - return NULL; - } - coll->scopes = calloc((size_t)count, sizeof(cbm_path_alias_scope_t)); - if (!coll->scopes) { - free(coll); - free(hits); - return NULL; - } - - for (int i = 0; i < count; i++) { - cbm_path_alias_map_t *map = load_tsconfig_file(hits[i].abs, hits[i].rel); - if (!map) { - continue; - } - coll->scopes[coll->count].dir_prefix = strdup(hits[i].rel); - coll->scopes[coll->count].map = map; - coll->count++; - } - free(hits); - - if (coll->count == 0) { - free(coll->scopes); - free(coll); - return NULL; - } - - if (coll->count > 1) { - qsort(coll->scopes, (size_t)coll->count, sizeof(cbm_path_alias_scope_t), - cmp_scope_by_specificity); - } - return coll; -} - -cbm_path_alias_collection_t *cbm_load_path_aliases(const char *repo_path) { - return cbm_load_path_aliases_excluded(repo_path, NULL, 0); -} - -const cbm_path_alias_map_t *cbm_path_alias_find_for_file(const cbm_path_alias_collection_t *coll, - const char *rel_path) { - if (!coll || !rel_path) { - return NULL; - } - for (int i = 0; i < coll->count; i++) { - const char *prefix = coll->scopes[i].dir_prefix; - size_t plen = strlen(prefix); - if (plen == 0) { - return coll->scopes[i].map; - } - if (strncmp(rel_path, prefix, plen) == 0 && - (rel_path[plen] == '/' || rel_path[plen] == '\0')) { - return coll->scopes[i].map; - } - } - return NULL; -} diff --git a/src/pipeline/path_alias.zig b/src/pipeline/path_alias.zig new file mode 100644 index 00000000..37ca9d83 --- /dev/null +++ b/src/pipeline/path_alias.zig @@ -0,0 +1,467 @@ +//! path_alias.zig — Resolve build-tool path aliases. +//! +//! Implements src/pipeline/path_alias.h. Builds a directory-scoped collection +//! of alias maps from per-language config files (currently tsconfig.json / +//! jsconfig.json) so the import resolver can turn "@/lib/auth"-style imports +//! into repo-relative paths. +//! +//! Design notes: +//! - Public types and functions are language-agnostic. Adding a Vite / +//! Webpack / Python loader means writing a new load*File() helper +//! and registering it in findAliasFiles. The resolver, the +//! collection, and the pipeline integration do not change. +//! - Sorting uses libc qsort, as the C code did, so entries of equal +//! specificity keep the order they had before the port. +//! - The repo walk caps recursion depth and total file count and emits +//! a warning when either cap fires, so silent truncation on +//! pathological monorepos shows up in the index log. + +const std = @import("std"); +const c = @import("c"); +const interop = @import("../foundation/interop.zig"); +const util = @import("util.zig"); + +const CStr = [*c]const u8; +const MutStr = [*c]u8; +const Alias = c.cbm_path_alias_t; +const AliasMap = c.cbm_path_alias_map_t; +const Scope = c.cbm_path_alias_scope_t; +const Collection = c.cbm_path_alias_collection_t; + +/// Resource ceilings. Chosen to comfortably cover real-world monorepos +/// (Next.js Skyline, large nx workspaces) while bounding worst-case +/// memory and walk time. Cap hits are logged. +const MAX_ENTRIES = 256; // per single config file +const MAX_FILES = 256; // config files per repo walk +const MAX_FILE_BYTES = 64 * 1024; +const MAX_DEPTH = 32; // directory recursion depth + +// ── Helpers ───────────────────────────────────────────────────────── + +/// Strip .ts/.tsx/.js/.jsx in place. Returns its argument. +fn stripResolvedExt(path: MutStr) MutStr { + if (path == null) return path; + const len = c.strlen(path); + if (len > 3 and path[len - 3] == '.' and (path[len - 2] == 't' or path[len - 2] == 'j') and + path[len - 1] == 's') + { + path[len - 3] = 0; + return path; + } + if (len > 4 and path[len - 4] == '.' and (path[len - 3] == 't' or path[len - 3] == 'j') and + path[len - 2] == 's' and path[len - 1] == 'x') + { + path[len - 4] = 0; + } + return path; +} + +/// Join dir_prefix with target, collapsing "." and ".." segments so aliases +/// that climb out of their tsconfig's directory (the common monorepo +/// pattern: a tsconfig at apps/web/tsconfig.json pointing an alias at a +/// wildcard target like "../../packages/shared/src/" + wildcard) resolve +/// to a real repo-relative path. Naive concatenation left literal ".." +/// components in the target, which never match a module's FQN since +/// cbm_pipeline_fqn_module tokenizes on '/' without collapsing them +/// (#730). A trailing '/' on target (the usual case right before a +/// wildcard) is preserved so the caller's later wildcard-substring +/// concat still lines up. Returns heap-allocated +/// repo-relative target. +fn resolveTargetRelative(dir_prefix: CStr, target: CStr) MutStr { + if (target == null) return null; + const dp_len: usize = if (dir_prefix != null and dir_prefix[0] != 0) c.strlen(dir_prefix) else 0; + const t_len = c.strlen(target); + const buf: MutStr = @ptrCast(c.malloc(dp_len + t_len + 2)); + if (buf == null) return null; + buf[0] = 0; + if (dp_len > 0) { + @memcpy(buf[0..dp_len], dir_prefix[0..dp_len]); + buf[dp_len] = 0; + } + + const trailing_slash = t_len > 0 and target[t_len - 1] == '/'; + + var p: [*]const u8 = target; + while (p[0] != 0) { + while (p[0] == '/') p += 1; + if (p[0] == 0) break; + const seg_start = p; + while (p[0] != 0 and p[0] != '/') p += 1; + const seg_len = @intFromPtr(p) - @intFromPtr(seg_start); + if (seg_len == 1 and seg_start[0] == '.') continue; + if (seg_len == 2 and seg_start[0] == '.' and seg_start[1] == '.') { + const last: MutStr = c.strrchr(buf, '/'); + if (last != null) { + last[0] = 0; + } else { + buf[0] = 0; + } + continue; + } + var cur = c.strlen(buf); + if (cur > 0) { + buf[cur] = '/'; + cur += 1; + } + @memcpy(buf[cur..][0..seg_len], seg_start[0..seg_len]); + buf[cur + seg_len] = 0; + } + + if (trailing_slash) { + const cur = c.strlen(buf); + buf[cur] = '/'; + buf[cur + 1] = 0; + } + return buf; +} + +/// qsort comparator: alias entries by alias_prefix length, descending. +fn cmpAliasEntryBySpecificity(a: ?*const anyopaque, b: ?*const anyopaque) callconv(.c) c_int { + const ea: *const Alias = @ptrCast(@alignCast(a)); + const eb: *const Alias = @ptrCast(@alignCast(b)); + const la = c.strlen(ea.alias_prefix); + const lb = c.strlen(eb.alias_prefix); + if (lb > la) return 1; + if (lb < la) return -1; + return 0; +} + +/// qsort comparator: scopes by dir_prefix length, descending. +fn cmpScopeBySpecificity(a: ?*const anyopaque, b: ?*const anyopaque) callconv(.c) c_int { + const sa: *const Scope = @ptrCast(@alignCast(a)); + const sb: *const Scope = @ptrCast(@alignCast(b)); + const la = c.strlen(sa.dir_prefix); + const lb = c.strlen(sb.dir_prefix); + if (lb > la) return 1; + if (lb < la) return -1; + return 0; +} + +// ── tsconfig.json / jsconfig.json loader ──────────────────────────── + +/// Parse compilerOptions.paths and compilerOptions.baseUrl into an alias map. +/// dir_prefix is the directory of the config file relative to the repo root +/// (e.g. "apps/manager", or "" for repo root). Returns NULL if the file is +/// missing, malformed, or has neither a usable paths block nor a baseUrl. +fn loadTsconfigFile(abs_path: CStr, dir_prefix: CStr) ?*AliasMap { + @setEvalBranchQuota(10000); + const f = c.cbm_fopen(abs_path, "r") orelse return null; + _ = c.fseek(f, 0, c.SEEK_END); + const len = c.ftell(f); + _ = c.fseek(f, 0, c.SEEK_SET); + if (len <= 0 or len > MAX_FILE_BYTES) { + _ = c.fclose(f); + return null; + } + const buf: MutStr = @ptrCast(c.malloc(@as(usize, @intCast(len)) + 1)); + if (buf == null) { + _ = c.fclose(f); + return null; + } + const nread = c.fread(buf, 1, @intCast(len), f); + _ = c.fclose(f); + buf[nread] = 0; + + const flg: c.yyjson_read_flag = c.YYJSON_READ_ALLOW_COMMENTS | c.YYJSON_READ_ALLOW_TRAILING_COMMAS; + const doc = c.yyjson_read(buf, nread, flg); + c.free(buf); + if (doc == null) return null; + const root = c.yyjson_doc_get_root(doc); + const compiler_opts = c.yyjson_obj_get(root, "compilerOptions"); + if (compiler_opts == null) { + c.yyjson_doc_free(doc); + return null; + } + const base_url_val = c.yyjson_obj_get(compiler_opts, "baseUrl"); + const base_url_str: CStr = if (base_url_val != null) c.yyjson_get_str(base_url_val) else null; + const paths_obj = c.yyjson_obj_get(compiler_opts, "paths"); + if (paths_obj == null and base_url_str == null) { + c.yyjson_doc_free(doc); + return null; + } + + const map: *AliasMap = @as(?*AliasMap, interop.callocN(AliasMap, 1)) orelse { + c.yyjson_doc_free(doc); + return null; + }; + + if (base_url_str != null and base_url_str[0] != 0 and c.strcmp(base_url_str, ".") != 0) { + map.base_url = resolveTargetRelative(dir_prefix, base_url_str); + } else if (base_url_str != null and c.strcmp(base_url_str, ".") == 0 and dir_prefix != null and + dir_prefix[0] != 0) + { + map.base_url = c.strdup(dir_prefix); + } + + if (paths_obj != null and c.yyjson_is_obj(paths_obj)) { + const obj_size = c.yyjson_obj_size(paths_obj); + const capped = obj_size > MAX_ENTRIES; + const capacity: c_int = @intCast(if (capped) MAX_ENTRIES else obj_size); + if (capacity > 0) { + map.entries = interop.callocN(Alias, @intCast(capacity)); + if (map.entries == null) { + c.free(map.base_url); + c.free(map); + c.yyjson_doc_free(doc); + return null; + } + var iter = c.yyjson_obj_iter_with(paths_obj); + while (true) { + const key = c.yyjson_obj_iter_next(&iter); + if (key == null or map.count >= capacity) break; + const val = c.yyjson_obj_iter_get_val(key); + const alias_pattern: CStr = c.yyjson_get_str(key); + if (alias_pattern == null or !c.yyjson_is_arr(val) or c.yyjson_arr_size(val) == 0) { + continue; + } + const target_pattern: CStr = c.yyjson_get_str(c.yyjson_arr_get_first(val)); + if (target_pattern == null) continue; + const entry = util.at(map.entries, @intCast(map.count)); + const star: CStr = c.strchr(alias_pattern, '*'); + if (star != null) { + entry.has_wildcard = true; + entry.alias_prefix = c.cbm_strndup(alias_pattern, @intFromPtr(star) - @intFromPtr(alias_pattern)); + entry.alias_suffix = c.strdup(star + 1); + } else { + entry.has_wildcard = false; + entry.alias_prefix = c.strdup(alias_pattern); + entry.alias_suffix = c.strdup(""); + } + const tstar: CStr = c.strchr(target_pattern, '*'); + if (tstar != null) { + const pre: MutStr = c.cbm_strndup(target_pattern, @intFromPtr(tstar) - @intFromPtr(target_pattern)); + entry.target_prefix = resolveTargetRelative(dir_prefix, pre); + c.free(pre); + entry.target_suffix = c.strdup(tstar + 1); + } else { + entry.target_prefix = resolveTargetRelative(dir_prefix, target_pattern); + entry.target_suffix = c.strdup(""); + } + map.count += 1; + } + if (capped) { + interop.logWarn("path_alias.entries.cap_hit", .{ + @as(CStr, "config"), abs_path, @as(CStr, "kept"), @as(CStr, "256_of_more"), + }); + } + if (map.count > 1) { + c.qsort(map.entries, @intCast(map.count), @sizeOf(Alias), cmpAliasEntryBySpecificity); + } + } + } + + c.yyjson_doc_free(doc); + return map; +} + +// ── Public API ────────────────────────────────────────────────────── + +pub export fn cbm_path_alias_collection_free(coll_c: ?*Collection) void { + const coll = coll_c orelse return; + var i: usize = 0; + while (i < coll.count) : (i += 1) { + c.free(coll.scopes[i].dir_prefix); + if (coll.scopes[i].map) |map| { + var j: usize = 0; + while (j < map.*.count) : (j += 1) { + c.free(map.*.entries[j].alias_prefix); + c.free(map.*.entries[j].alias_suffix); + c.free(map.*.entries[j].target_prefix); + c.free(map.*.entries[j].target_suffix); + } + c.free(map.*.entries); + c.free(map.*.base_url); + c.free(map); + } + } + c.free(coll.scopes); + c.free(coll); +} + +pub export fn cbm_path_alias_resolve(map_c: ?*const AliasMap, module_path: CStr) MutStr { + const map = map_c orelse return null; + if (module_path == null) return null; + const mod_len = c.strlen(module_path); + + var i: usize = 0; + while (i < map.count) : (i += 1) { + const e = util.at(map.entries, i); + + if (e.has_wildcard) { + const prefix_len = c.strlen(e.alias_prefix); + const suffix_len = c.strlen(e.alias_suffix); + if (mod_len < prefix_len + suffix_len) continue; + if (c.strncmp(module_path, e.alias_prefix, prefix_len) != 0) continue; + if (suffix_len > 0 and + c.strcmp(module_path + mod_len - suffix_len, e.alias_suffix) != 0) + { + continue; + } + const wild_len = mod_len - prefix_len - suffix_len; + const wild_start = module_path + prefix_len; + const tp_len = c.strlen(e.target_prefix); + const ts_len = c.strlen(e.target_suffix); + const result: MutStr = @ptrCast(c.malloc(tp_len + wild_len + ts_len + 1)); + if (result == null) return null; + @memcpy(result[0..tp_len], e.target_prefix[0..tp_len]); + @memcpy(result[tp_len..][0..wild_len], wild_start[0..wild_len]); + @memcpy(result[tp_len + wild_len ..][0..ts_len], e.target_suffix[0..ts_len]); + result[tp_len + wild_len + ts_len] = 0; + return stripResolvedExt(result); + } + + if (c.strcmp(module_path, e.alias_prefix) == 0) { + return stripResolvedExt(c.strdup(e.target_prefix)); + } + } + + // baseUrl fallback. Apply only to non-relative imports that look + // sub-path-ish (contain '/' but don't start with '.' or '@'); skips + // obvious package names like "react" or "lodash". + if (map.base_url != null and module_path[0] != '.' and module_path[0] != '@' and + c.strchr(module_path, '/') != null) + { + const bu_len = c.strlen(map.base_url); + const need = bu_len + 1 + mod_len + 1; + const result: MutStr = @ptrCast(c.malloc(need)); + if (result == null) return null; + _ = c.snprintf(result, need, "%s/%s", map.base_url, module_path); + return stripResolvedExt(result); + } + return null; +} + +// ── Repo walk ─────────────────────────────────────────────────────── + +const AliasConfigHit = extern struct { + abs: [c.CBM_SZ_512]u8, + rel: [c.CBM_SZ_256]u8, +}; + +const TS_CONFIG_NAMES = [_][*:0]const u8{ "tsconfig.json", "jsconfig.json" }; + +// NOLINT(misc-no-recursion) in the C original: bounded by MAX_DEPTH. +fn findAliasFiles( + abs_dir: CStr, + rel_dir: CStr, + out: [*]AliasConfigHit, + count: *c_int, + max_count: c_int, + depth: c_int, + excluded_dirs: [*c][*c]u8, + excluded_count: c_int, +) void { + if (count.* >= max_count or depth > MAX_DEPTH) return; + const d = c.cbm_opendir(abs_dir) orelse return; + + // One config file per directory: prefer tsconfig.json over jsconfig.json. + for (TS_CONFIG_NAMES) |config_name| { + if (count.* >= max_count) break; + var check: [c.CBM_SZ_512]u8 = undefined; + _ = c.snprintf(&check, check.len, "%s/%s", abs_dir, config_name); + if (c.cbm_fopen(&check, "r")) |f| { + _ = c.fclose(f); + const hit = &out[@intCast(count.*)]; + _ = c.snprintf(&hit.abs, hit.abs.len, "%s", @as(CStr, &check)); + _ = c.snprintf(&hit.rel, hit.rel.len, "%s", rel_dir); + count.* += 1; + break; + } + } + + while (true) { + const ent = c.cbm_readdir(d) orelse break; + if (count.* >= max_count) break; + if (!ent.*.is_dir) continue; + const name: CStr = &ent.*.name; + if (name[0] == '.' or c.strcmp(name, "node_modules") == 0 or c.strcmp(name, "dist") == 0 or + c.strcmp(name, "build") == 0 or c.strcmp(name, ".next") == 0 or + c.strcmp(name, "coverage") == 0 or c.strcmp(name, "target") == 0 // Rust + ) { + continue; + } + var child_abs: [c.CBM_SZ_512]u8 = undefined; + var child_rel: [c.CBM_SZ_256]u8 = undefined; + _ = c.snprintf(&child_abs, child_abs.len, "%s/%s", abs_dir, name); + if (rel_dir[0] == 0) { + _ = c.snprintf(&child_rel, child_rel.len, "%s", name); + } else { + _ = c.snprintf(&child_rel, child_rel.len, "%s/%s", rel_dir, name); + } + if (c.cbm_pipeline_relpath_is_excluded(&child_rel, excluded_dirs, excluded_count) != 0) continue; + findAliasFiles(&child_abs, &child_rel, out, count, max_count, depth + 1, excluded_dirs, excluded_count); + } + _ = c.cbm_closedir(d); +} + +pub export fn cbm_load_path_aliases_excluded( + repo_path: CStr, + excluded_dirs: [*c][*c]u8, + excluded_count: c_int, +) ?*Collection { + if (repo_path == null) return null; + const hits: [*]AliasConfigHit = @as(?[*]AliasConfigHit, interop.callocN(AliasConfigHit, MAX_FILES)) orelse return null; + var count: c_int = 0; + findAliasFiles(repo_path, "", hits, &count, MAX_FILES, 0, excluded_dirs, excluded_count); + if (count >= MAX_FILES) { + interop.logWarn("path_alias.files.cap_hit", .{ + @as(CStr, "repo"), repo_path, @as(CStr, "kept"), @as(CStr, "256_of_more"), + }); + } + if (count == 0) { + c.free(hits); + return null; + } + + const coll: *Collection = @as(?*Collection, interop.callocN(Collection, 1)) orelse { + c.free(hits); + return null; + }; + coll.scopes = interop.callocN(Scope, @intCast(count)); + if (coll.scopes == null) { + c.free(coll); + c.free(hits); + return null; + } + + for (hits[0..@intCast(count)]) |*hit| { + const map = loadTsconfigFile(&hit.abs, &hit.rel) orelse continue; + const scope = util.at(coll.scopes, @intCast(coll.count)); + scope.dir_prefix = c.strdup(&hit.rel); + scope.map = map; + coll.count += 1; + } + c.free(hits); + + if (coll.count == 0) { + c.free(coll.scopes); + c.free(coll); + return null; + } + + if (coll.count > 1) { + c.qsort(coll.scopes, @intCast(coll.count), @sizeOf(Scope), cmpScopeBySpecificity); + } + return coll; +} + +pub export fn cbm_load_path_aliases(repo_path: CStr) ?*Collection { + return cbm_load_path_aliases_excluded(repo_path, null, 0); +} + +pub export fn cbm_path_alias_find_for_file(coll_c: ?*const Collection, rel_path: CStr) ?*const AliasMap { + const coll = coll_c orelse return null; + if (rel_path == null) return null; + var i: usize = 0; + while (i < coll.count) : (i += 1) { + const prefix: CStr = coll.scopes[i].dir_prefix; + const plen = c.strlen(prefix); + if (plen == 0) return coll.scopes[i].map; + if (c.strncmp(rel_path, prefix, plen) == 0 and + (rel_path[plen] == '/' or rel_path[plen] == 0)) + { + return coll.scopes[i].map; + } + } + return null; +} diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c deleted file mode 100644 index 0f35cf18..00000000 --- a/src/pipeline/pipeline.c +++ /dev/null @@ -1,1772 +0,0 @@ -/* - * pipeline.c — Indexing pipeline orchestrator. - * - * Coordinates multi-pass indexing: - * 1. Discover files - * 2. Build structure (Project/Folder/Package/File nodes) - * 3. Bulk load sources (read + LZ4 HC compress) - * 4. Extract definitions (fused: extract + write nodes + build registry) - * 5. Resolve imports, calls, usages, semantic edges - * 6. Post-passes: tests, communities, HTTP links, git history - * 7. Dump graph buffer to SQLite - */ -#include "foundation/constants.h" - -enum { CBM_DIR_PERMS = 0755 }; -#include "pipeline/pipeline.h" -#include "pipeline/pipeline_internal.h" -#include "pipeline/pass_lsp_cross.h" -#include "pipeline/worker_pool.h" -#include "graph_buffer/graph_buffer.h" -#include "git/git_context.h" -#include "store/store.h" -#include "zova/cbm_zova.h" -#include "zova/cbm_zova_route.h" -#include "zova/cbm_zova_repository.h" -#include "discover/discover.h" -#include "discover/userconfig.h" -#include "arena.h" -#include "cbm.h" -#include "helpers.h" /* cbm_fqn_module_source_lang, cbm_fqn_folder */ -#include "foundation/platform.h" -#include "foundation/compat_fs.h" -#include "foundation/log.h" -#include "foundation/str_util.h" -#include "foundation/hash_table.h" -#include "foundation/compat.h" -#include "foundation/compat_thread.h" -#include "foundation/profile.h" -#include "foundation/mem.h" -#include "yyjson/yyjson.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -/* ── Global pkgmap (one active pipeline at a time) ─────────────── */ - -static CBMHashTable *g_pkgmap = NULL; - -CBMHashTable *cbm_pipeline_get_pkgmap(void) { - return g_pkgmap; -} - -void cbm_pipeline_set_pkgmap(CBMHashTable *map) { - g_pkgmap = map; -} - -/* ── Internal state ──────────────────────────────────────────────── */ - -struct cbm_pipeline { - char *repo_path; - char *db_path; - char *project_name; - cbm_git_context_t git_ctx; - char *branch_qn; - cbm_index_mode_t mode; - atomic_int cancelled; - - /* Indexing state (set during run) */ - cbm_gbuf_t *gbuf; - cbm_registry_t *registry; - - /* Directory subtrees skipped during discovery (rel paths). Captured from - * cbm_discover_ex so the MCP layer can report excluded subtrees (#411). - * Owned by the pipeline; freed in cbm_pipeline_free. */ - char **excluded_dirs; - int excluded_count; - - /* Set when discovery could not fully materialize the tree (bounded walk - * stack full / allocation failure). The MCP layer reports a degraded index - * instead of a complete-looking partial one (#17, upstream 03dc9c91). */ - bool discover_degraded; - - /* Per-file indexing failures (skipped files) surfaced via MCP/CLI/logfile - * (Stage 2 / Track B). A skip is the expected handled outcome of a bad or - * oversized file — the run still succeeds ("indexed"). Owned by the - * pipeline; freed in cbm_pipeline_free. */ - cbm_file_error_t *file_errors; - int file_errors_count; - int file_errors_cap; - - /* User-defined extension overrides (loaded once per run) */ - cbm_userconfig_t *userconfig; - - /* Committed graph size at dump time (-1 = dump did not run). #334 gate axis. */ - int committed_nodes; - int committed_edges; - double zova_publish_ms; - cbm_pipeline_route_t last_route; - /* The last run rebuilt a generation stamped with an older index format. */ - bool format_migration; - cbm_pipeline_zova_publish_stats_t zova_publish_stats; - - cbm_project_summary_export_t saved_summary; - bool saved_summary_valid; -}; - -/* Log current + peak RSS and the budget at a pipeline phase boundary (memory - * profiling). The process peak is cumulative, so the boundary where peak_mb - * first rises is the phase that set it (#49). */ -void cbm_pipeline_log_phase_mem(const char *phase) { - enum { PL_BYTES_PER_MB = 1024 * 1024 }; - cbm_log_info("mem.phase", "phase", phase, "rss_mb", - cbm_log_itoa((int)(cbm_mem_rss() / PL_BYTES_PER_MB)), "peak_mb", - cbm_log_itoa((int)(cbm_mem_peak_rss() / PL_BYTES_PER_MB)), "budget_mb", - cbm_log_itoa((int)(cbm_mem_budget() / PL_BYTES_PER_MB))); -} - -/* Post-extraction budget gate. The extract phase enforces the budget with - * back-pressure and result spill, but registry/resolve/derived passes/dump - * work on the whole graph at once. Reclaim freed pages, then fail the attempt - * whole if the resident graph is still over budget, before anything is - * published, so the previously served generation keeps answering. - * - * Only an explicit CBM_MEM_BUDGET_MB hard-fails. The detected RAM-fraction - * default stays a soft target: it is sized for extraction's spill heuristics, - * and a run that overshoots it after extraction already finishes today (#49 - * measurements). Hard-failing it would turn successful indexes into repeatable - * over_memory_budget errors with no way to configure out but the env var. - * Returns 0 when under budget, when the budget is implicit, or when none is - * configured. Every call site is single-threaded: the workers have joined, so - * there is no transient to wait for. */ -int cbm_pipeline_budget_gate(const char *phase) { - if (cbm_mem_budget() == 0 || !cbm_mem_budget_explicit()) { - return 0; - } - if (!cbm_mem_over_budget()) { - return 0; - } - cbm_mem_collect(); - if (!cbm_mem_over_budget()) { - return 0; - } - cbm_log_error("mem.budget.exceeded", "rss_mb", - cbm_log_itoa((int)(cbm_mem_rss() / (1024 * 1024))), "budget_mb", - cbm_log_itoa((int)(cbm_mem_budget() / (1024 * 1024))), "phase", - phase ? phase : "post_extract", "action", "fail_whole"); - return CBM_PIPELINE_ABORT_OVER_BUDGET; -} - -/* ── Lifecycle ──────────────────────────────────────────────────── */ - -cbm_pipeline_t *cbm_pipeline_new(const char *repo_path, const char *db_path, - cbm_index_mode_t mode) { - if (!repo_path) { - return NULL; - } - - cbm_pipeline_t *p = calloc(CBM_ALLOC_ONE, sizeof(cbm_pipeline_t)); - if (!p) { - return NULL; - } - - p->repo_path = strdup(repo_path); - p->db_path = db_path ? strdup(db_path) : NULL; - p->project_name = cbm_project_name_from_path(repo_path); - (void)cbm_git_context_resolve(repo_path, &p->git_ctx); - p->branch_qn = cbm_git_context_branch_qn(p->project_name, &p->git_ctx); - p->mode = mode; - p->committed_nodes = -1; - p->committed_edges = -1; - atomic_init(&p->cancelled, 0); - - return p; -} - -bool cbm_pipeline_set_project_name(cbm_pipeline_t *p, const char *name) { - if (!p || !name || !name[0]) { - return false; - } - - char *normalized = cbm_project_name_from_path(name); - if (!normalized) { - return false; - } - if (!cbm_validate_project_name(normalized)) { - free(normalized); - return false; - } - - free(p->project_name); - p->project_name = normalized; - free(p->branch_qn); - p->branch_qn = cbm_git_context_branch_qn(p->project_name, &p->git_ctx); - return true; -} - -void cbm_pipeline_free(cbm_pipeline_t *p) { - if (!p) { - return; - } - free(p->repo_path); - free(p->db_path); - free(p->project_name); - cbm_discover_free_excluded(p->excluded_dirs, p->excluded_count); - p->excluded_dirs = NULL; - p->excluded_count = 0; - for (int i = 0; i < p->file_errors_count; i++) { - free(p->file_errors[i].path); - free(p->file_errors[i].reason); - free(p->file_errors[i].phase); - } - free(p->file_errors); - p->file_errors = NULL; - p->file_errors_count = 0; - p->file_errors_cap = 0; - free(p->branch_qn); - cbm_store_project_summary_export_free(&p->saved_summary); - p->saved_summary_valid = false; - cbm_git_context_free(&p->git_ctx); - /* gbuf, store, registry freed during/after run */ - /* Defensively free userconfig in case run() was never called or panicked */ - if (p->userconfig) { - cbm_set_user_lang_config(NULL); - cbm_userconfig_free(p->userconfig); - p->userconfig = NULL; - } - free(p); -} - -void cbm_pipeline_cancel(cbm_pipeline_t *p) { - if (p) { - atomic_store(&p->cancelled, 1); - } -} - -const char *cbm_pipeline_project_name(const cbm_pipeline_t *p) { - return p ? p->project_name : NULL; -} - -const char *cbm_pipeline_repo_path(const cbm_pipeline_t *p) { - return p ? p->repo_path : NULL; -} - -atomic_int *cbm_pipeline_cancelled_ptr(cbm_pipeline_t *p) { - return p ? &p->cancelled : NULL; -} - -int cbm_pipeline_get_mode(const cbm_pipeline_t *p) { - return p ? (int)p->mode : 0; -} - -void cbm_pipeline_get_excluded(const cbm_pipeline_t *p, char ***out, int *count) { - if (out) { - *out = p ? p->excluded_dirs : NULL; - } - if (count) { - *count = p ? p->excluded_count : 0; - } -} - -bool cbm_pipeline_discover_degraded(const cbm_pipeline_t *p) { - return p ? p->discover_degraded : false; -} - -/* NULL-safe heap strdup (avoids a strdup dependency + guards NULL inputs). */ -static char *fe_strdup(const char *s) { - if (!s) { - return NULL; - } - size_t n = strlen(s) + 1; - char *d = (char *)malloc(n); - if (d) { - memcpy(d, s, n); - } - return d; -} - -bool cbm_pipeline_phase_is_partial(const char *phase) { - return phase && (strcmp(phase, "parse_partial") == 0 || strcmp(phase, "parse_unusable") == 0); -} - -void cbm_pipeline_add_file_error(cbm_pipeline_t *p, const char *path, const char *reason, - const char *phase) { - if (!p) { - return; - } - if (p->file_errors_count >= p->file_errors_cap) { - int ncap = p->file_errors_cap ? p->file_errors_cap * 2 : 16; - cbm_file_error_t *grown = - (cbm_file_error_t *)realloc(p->file_errors, (size_t)ncap * sizeof(*grown)); - if (!grown) { - /* Never abort indexing just to record a skip — drop this record. */ - return; - } - p->file_errors = grown; - p->file_errors_cap = ncap; - } - cbm_file_error_t *e = &p->file_errors[p->file_errors_count]; - e->path = fe_strdup(path); - e->reason = fe_strdup(reason); - e->phase = fe_strdup(phase); - p->file_errors_count++; -} - -void cbm_pipeline_get_file_errors(const cbm_pipeline_t *p, cbm_file_error_t **out, int *count) { - if (out) { - *out = p ? p->file_errors : NULL; - } - if (count) { - *count = p ? p->file_errors_count : 0; - } -} - -/* Rebuild a File node's properties with its coverage stamp. The extension - * key mirrors pass_structure / incr_add_changed_file_structure. */ -static bool stamp_file_coverage(cbm_pipeline_t *p, cbm_gbuf_t *gbuf, const cbm_file_error_t *e, - bool partial) { - char *file_qn = cbm_pipeline_fqn_compute(p->project_name, e->path, "__file__"); - if (!file_qn) { - return false; - } - const char *slash = strrchr(e->path, '/'); - const char *basename = slash ? slash + SKIP_ONE : e->path; - const char *ext = strrchr(basename, '.'); - yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); - yyjson_mut_val *root = yyjson_mut_obj(doc); - yyjson_mut_doc_set_root(doc, root); - yyjson_mut_obj_add_str(doc, root, "extension", ext ? ext : ""); - if (partial) { - yyjson_mut_obj_add_str(doc, root, "coverage", CBM_COVERAGE_PARTIAL); - yyjson_mut_obj_add_str(doc, root, "parse_error_ranges", e->reason ? e->reason : ""); - if (strcmp(e->phase, "parse_unusable") == 0) { - yyjson_mut_obj_add_bool(doc, root, "parse_unusable", true); - } - } else { - yyjson_mut_obj_add_str(doc, root, "coverage", CBM_COVERAGE_MISSED); - yyjson_mut_obj_add_str(doc, root, "coverage_phase", e->phase); - yyjson_mut_obj_add_str(doc, root, "coverage_reason", e->reason ? e->reason : ""); - } - char *props = yyjson_mut_write(doc, 0, NULL); - yyjson_mut_doc_free(doc); - bool stamped = props && cbm_gbuf_set_node_props(gbuf, file_qn, props); - free(props); - free(file_qn); - return stamped; -} - -void cbm_pipeline_stamp_coverage(cbm_pipeline_t *p, cbm_gbuf_t *gbuf) { - if (!p || !gbuf) { - return; - } - int partial = 0; - int missed = 0; - /* Partial first, so a file recorded under both keeps the stronger - * "missed" stamp. */ - for (int pass = 0; pass < 2; pass++) { - for (int i = 0; i < p->file_errors_count; i++) { - const cbm_file_error_t *e = &p->file_errors[i]; - if (!e->path || !e->phase) { - continue; - } - bool is_partial = cbm_pipeline_phase_is_partial(e->phase); - if (is_partial != (pass == 0)) { - continue; - } - if (stamp_file_coverage(p, gbuf, e, is_partial)) { - if (is_partial) { - partial++; - } else { - missed++; - } - } - } - } - if (partial > 0 || missed > 0) { - cbm_log_info("index.coverage", "partial", cbm_log_itoa(partial), "missed", - cbm_log_itoa(missed)); - } -} - -void cbm_pipeline_get_committed_counts(const cbm_pipeline_t *p, int *nodes, int *edges) { - if (nodes) { - *nodes = p ? p->committed_nodes : -1; - } - if (edges) { - *edges = p ? p->committed_edges : -1; - } -} - -double cbm_pipeline_get_zova_publish_ms(const cbm_pipeline_t *p) { - return p ? p->zova_publish_ms : 0.0; -} - -cbm_pipeline_route_t cbm_pipeline_get_last_route(const cbm_pipeline_t *p) { - return p ? p->last_route : CBM_PIPELINE_ROUTE_UNKNOWN; -} - -/* ── Main-buffer id counter sync after parallel phases (#50) ─────── */ - -static CBM_TLS void (*g_id_sync_hook)(cbm_gbuf_t *, void *); -static CBM_TLS void *g_id_sync_hook_ctx; -static CBM_TLS void (*g_before_dump_hook)(const cbm_gbuf_t *, void *); -static CBM_TLS void *g_before_dump_hook_ctx; - -void cbm_pipeline_set_id_sync_hook_for_tests(void (*hook)(cbm_gbuf_t *, void *), void *ctx) { - g_id_sync_hook = hook; - g_id_sync_hook_ctx = ctx; -} - -void cbm_pipeline_set_before_dump_hook_for_tests(void (*hook)(const cbm_gbuf_t *, void *), - void *ctx) { - g_before_dump_hook = hook; - g_before_dump_hook_ctx = ctx; -} - -void cbm_pipeline_run_before_dump_hook(const cbm_gbuf_t *gb) { - if (g_before_dump_hook) { - g_before_dump_hook(gb, g_before_dump_hook_ctx); - } -} - -void cbm_pipeline_sync_next_id(cbm_gbuf_t *gb, _Atomic int64_t *shared_ids) { - if (g_id_sync_hook) { - g_id_sync_hook(gb, g_id_sync_hook_ctx); - } - /* Raise only. The merge leaves the main counter above the shared one when - * merged edges take main-counter ids; lowering it to the shared value - * handed those ids out a second time. */ - cbm_gbuf_raise_next_id(gb, atomic_load(shared_ids)); -} - -/* ── Shared id counter raise before parallel resolve (#88) ───────── */ - -static CBM_TLS void (*g_share_ids_hook)(cbm_gbuf_t *, _Atomic int64_t *, void *); -static CBM_TLS void *g_share_ids_hook_ctx; - -void cbm_pipeline_set_share_ids_hook_for_tests(void (*hook)(cbm_gbuf_t *, _Atomic int64_t *, - void *), - void *ctx) { - g_share_ids_hook = hook; - g_share_ids_hook_ctx = ctx; -} - -void cbm_pipeline_share_next_id(cbm_gbuf_t *gb, _Atomic int64_t *shared_ids) { - if (g_share_ids_hook) { - g_share_ids_hook(gb, shared_ids, g_share_ids_hook_ctx); - } - /* No worker runs here, so a plain load and store is enough. Raise only: - * the shared counter may already be above the main one. */ - int64_t main_next = cbm_gbuf_next_id(gb); - if (main_next > atomic_load(shared_ids)) { - atomic_store(shared_ids, main_next); - } -} - -bool cbm_pipeline_had_format_migration(const cbm_pipeline_t *p) { - return p && p->format_migration; -} - -bool cbm_pipeline_get_zova_publish_stats( - const cbm_pipeline_t *p, cbm_pipeline_zova_publish_stats_t *out_stats) { - if (!p || !out_stats || !p->zova_publish_stats.completed) return false; - *out_stats = p->zova_publish_stats; - return true; -} - -void cbm_pipeline_set_committed_counts(cbm_pipeline_t *p, int nodes, int edges) { - if (p) { - p->committed_nodes = nodes; - p->committed_edges = edges; - } -} - -static int check_cancel(const cbm_pipeline_t *p) { - return atomic_load(&p->cancelled) ? CBM_NOT_FOUND : 0; -} - -/* ── Hash table cleanup callback ─────────────────────────────────── */ - -static void free_seen_dir_key(const char *key, void *val, void *ud) { - (void)val; - (void)ud; - free((void *)key); -} - -/* ── Pass 1: Structure ──────────────────────────────────────────── */ - -/* True for languages whose module QN is the CONTAINING DIRECTORY (Java/Go - * package). MUST match cbm_lang_module_is_dir() (internal/cbm/helpers.c). */ -static bool stem_module_is_dir(CBMLanguage lang) { - return lang == CBM_LANG_JAVA || lang == CBM_LANG_GO; -} - -/* Each file's Module QN, computed as extraction computes it, and the Folder - * QN of every directory above it. */ -void cbm_stem_marks_init(cbm_stem_marks_t *m, const char *project, const cbm_file_info_t *files, - int file_count) { - cbm_arena_init(&m->arena); - m->module_qns = cbm_ht_create(CBM_SZ_256); - m->dir_module_qns = cbm_ht_create(CBM_SZ_64); - m->folder_qns = cbm_ht_create(CBM_SZ_256); - for (int i = 0; i < file_count; i++) { - const char *rel = files[i].rel_path; - if (!rel) { - continue; - } - char *qn = cbm_fqn_module_source_lang(&m->arena, project, rel, files[i].language); - if (qn) { - cbm_ht_set(stem_module_is_dir(files[i].language) ? m->dir_module_qns : m->module_qns, - qn, qn); - } - for (const char *s = strchr(rel, '/'); s; s = strchr(s + SKIP_ONE, '/')) { - char *dir = cbm_arena_strndup(&m->arena, rel, (size_t)(s - rel)); - char *fqn = dir ? cbm_fqn_folder(&m->arena, project, dir) : NULL; - if (fqn) { - cbm_ht_set(m->folder_qns, fqn, fqn); - } - } - } -} - -void cbm_stem_marks_free(cbm_stem_marks_t *m) { - cbm_ht_free(m->module_qns); - cbm_ht_free(m->dir_module_qns); - cbm_ht_free(m->folder_qns); - cbm_arena_destroy(&m->arena); -} - -int cbm_stem_marks_side(const cbm_stem_marks_t *m, const char *qn) { - if (!m || !qn || !cbm_ht_get(m->module_qns, qn)) { - return CBM_STEM_NONE; - } - if (cbm_ht_get(m->dir_module_qns, qn)) { - return CBM_STEM_MODULE_YIELDS; - } - return cbm_ht_get(m->folder_qns, qn) ? CBM_STEM_FOLDER_YIELDS : CBM_STEM_NONE; -} - -/* The extension names the yielding Module's file ("sh" for genproto.sh); - * "file" when there is none. */ -static const char *stem_mark_suffix(const char *rel_path) { - const char *base = strrchr(rel_path, '/'); - base = base ? base + SKIP_ONE : rel_path; - const char *dot = strrchr(base, '.'); - return dot && dot != base && dot[SKIP_ONE] ? dot + SKIP_ONE : "file"; -} - -const char *cbm_stem_marks_module_qn(const cbm_stem_marks_t *m, const char *qn, - const char *rel_path, CBMLanguage lang, char *buf, - size_t buf_size) { - if (!m || !qn || !rel_path || stem_module_is_dir(lang) || !cbm_ht_get(m->dir_module_qns, qn)) { - return qn; - } - int n = snprintf(buf, buf_size, "%s#%s", qn, stem_mark_suffix(rel_path)); - return n > 0 && (size_t)n < buf_size ? buf : qn; -} - -const char *cbm_stem_marks_folder_qn(const cbm_stem_marks_t *m, const char *project, - const char *rel_dir, char *buf, size_t buf_size) { - char *qn = cbm_pipeline_fqn_folder(project, rel_dir); - if (!qn) { - return NULL; - } - bool mark = m && cbm_ht_get(m->module_qns, qn) && !cbm_ht_get(m->dir_module_qns, qn); - int n = snprintf(buf, buf_size, "%s%s", qn, mark ? "/" : ""); - free(qn); - return n >= 0 && (size_t)n < buf_size ? buf : NULL; -} - -/* Create Project, Folder/Package, and File nodes in the graph buffer. */ -/* Walk directory chain upward, creating Folder nodes and CONTAINS_FOLDER edges. */ -static void create_folder_chain(cbm_pipeline_t *p, const cbm_stem_marks_t *marks, const char *dir, - CBMHashTable *seen_dirs) { - char *walk = strdup(dir); - while (walk[0] != '\0' && !cbm_ht_get(seen_dirs, walk)) { - cbm_ht_set(seen_dirs, strdup(walk), cbm_intptr_to_ptr(SKIP_ONE)); - char folder_buf[CBM_SZ_8K]; - const char *folder_qn = - cbm_stem_marks_folder_qn(marks, p->project_name, walk, folder_buf, sizeof(folder_buf)); - const char *dir_base = strrchr(walk, '/'); - dir_base = dir_base ? dir_base + SKIP_ONE : walk; - cbm_gbuf_upsert_node(p->gbuf, "Folder", dir_base, folder_qn, walk, 0, 0, "{}"); - - char *pdir = strdup(walk); - char *ps = strrchr(pdir, '/'); - if (ps) { - *ps = '\0'; - } else { - free(pdir); - pdir = strdup(""); - } - const char *pqn; - char pqn_buf[CBM_SZ_8K]; - if (pdir[0] == '\0') { - pqn = p->branch_qn ? p->branch_qn : p->project_name; - } else { - pqn = cbm_stem_marks_folder_qn(marks, p->project_name, pdir, pqn_buf, sizeof(pqn_buf)); - } - const cbm_gbuf_node_t *fn = cbm_gbuf_find_by_qn(p->gbuf, folder_qn); - const cbm_gbuf_node_t *pn = cbm_gbuf_find_by_qn(p->gbuf, pqn); - if (fn && pn) { - cbm_gbuf_insert_edge(p->gbuf, pn->id, fn->id, "CONTAINS_FOLDER", "{}"); - } - char *up = strrchr(walk, '/'); - if (up) { - *up = '\0'; - } else { - walk[0] = '\0'; - } - free(pdir); - } - free(walk); -} - -static int pass_structure(cbm_pipeline_t *p, const cbm_stem_marks_t *marks, - const cbm_file_info_t *files, int file_count) { - cbm_log_info("pass.start", "pass", "structure", "files", cbm_log_itoa(file_count)); - - /* Project node */ - /* coverage_version marks a generation whose File nodes carry coverage - * stamps; incremental runs keep the node, so older generations stay - * unmarked until the next full index. index_format is checked by - * try_incremental before an existing generation is reused. */ - cbm_gbuf_upsert_node(p->gbuf, "Project", p->project_name, p->project_name, NULL, 0, 0, - "{\"coverage_version\":" CBM_COVERAGE_VERSION_STR - ",\"index_format\":" CBM_INDEX_FORMAT_VERSION_STR "}"); - const char *branch_qn = p->branch_qn ? p->branch_qn : p->project_name; - const char *branch_name = p->git_ctx.branch ? p->git_ctx.branch : "working-tree"; - char branch_props[CBM_SZ_2K]; - const char *branch_props_json = "{}"; - if (cbm_git_context_props_json(&p->git_ctx, branch_props, sizeof(branch_props)) > 0) { - branch_props_json = branch_props; - } - if (p->branch_qn) { - int64_t branch_id = cbm_gbuf_upsert_node(p->gbuf, "Branch", branch_name, branch_qn, NULL, 0, - 0, branch_props_json); - const cbm_gbuf_node_t *project_node = cbm_gbuf_find_by_qn(p->gbuf, p->project_name); - if (project_node && branch_id > 0) { - cbm_gbuf_insert_edge(p->gbuf, project_node->id, branch_id, "HAS_BRANCH", - branch_props_json); - } - } - - /* Collect unique directories and create Folder/Package nodes */ - CBMHashTable *seen_dirs = cbm_ht_create(CBM_SZ_256); - - for (int i = 0; i < file_count; i++) { - const char *rel = files[i].rel_path; - if (!rel) { - continue; - } - - /* Create File node */ - char *file_qn = cbm_pipeline_fqn_compute(p->project_name, rel, "__file__"); - /* Extract basename */ - const char *slash = strrchr(rel, '/'); - const char *basename = slash ? slash + SKIP_ONE : rel; - - char props[CBM_SZ_256]; - const char *ext = strrchr(basename, '.'); - snprintf(props, sizeof(props), "{\"extension\":\"%s\"}", ext ? ext : ""); - - const char *qualified_name = file_qn; - const char *file_path = rel; - cbm_gbuf_upsert_node(p->gbuf, "File", basename, qualified_name, file_path, 0, 0, props); - - /* CONTAINS_FILE edge: parent dir -> file */ - char *dir = strdup(rel); - char *last_slash = strrchr(dir, '/'); - if (last_slash) { - { - *last_slash = '\0'; - } - } else { - free(dir); - dir = strdup(""); - } - - const char *parent_qn; - char parent_buf[CBM_SZ_8K]; - if (dir[0] == '\0') { - parent_qn = branch_qn; - } else { - parent_qn = cbm_stem_marks_folder_qn(marks, p->project_name, dir, parent_buf, - sizeof(parent_buf)); - } - - /* Walk up directory chain, creating Folder nodes */ - create_folder_chain(p, marks, dir, seen_dirs); - - /* Now create the CONTAINS_FILE edge */ - const cbm_gbuf_node_t *fnode = cbm_gbuf_find_by_qn(p->gbuf, file_qn); - const cbm_gbuf_node_t *pnode = cbm_gbuf_find_by_qn(p->gbuf, parent_qn); - if (fnode && pnode) { - cbm_gbuf_insert_edge(p->gbuf, pnode->id, fnode->id, "CONTAINS_FILE", "{}"); - } - - free(file_qn); - free(dir); - } - - /* Free seen_dirs keys */ - cbm_ht_foreach(seen_dirs, free_seen_dir_key, NULL); - cbm_ht_free(seen_dirs); - - cbm_log_info("pass.done", "pass", "structure", "nodes", - cbm_log_itoa(cbm_gbuf_node_count(p->gbuf)), "edges", - cbm_log_itoa(cbm_gbuf_edge_count(p->gbuf))); - return 0; -} - -/* ── Githistory compute thread (for fused post-pass parallelism) ─── */ - -typedef struct { - const char *repo_path; - cbm_githistory_result_t *result; -} gh_compute_arg_t; - -static void *gh_compute_thread_fn(void *arg) { - gh_compute_arg_t *a = arg; - cbm_pipeline_githistory_compute(a->repo_path, a->result); - return NULL; -} - -/* Extract Route nodes from URL strings found in config files (YAML, HCL, TOML). - * These are infrastructure-defined endpoints (Cloud Scheduler, Terraform). */ -/* Process infra bindings: topic→URL pairs from IaC configs. - * Creates Route nodes for endpoints and HANDLES edges linking - * topic Routes to endpoint Routes (bridging the gap). */ -/* Process one infra binding: create Route node + INFRA_MAPS edge. */ -static int process_one_infra_binding(cbm_gbuf_t *gbuf, const CBMInfraBinding *ib, - const char *rel_path) { - char url_route_qn[CBM_ROUTE_QN_SIZE]; - snprintf(url_route_qn, sizeof(url_route_qn), "__route__infra__%s", ib->target_url); - int64_t url_route_id = cbm_gbuf_upsert_node(gbuf, "Route", ib->target_url, url_route_qn, - rel_path, 0, 0, "{\"source\":\"infra\"}"); - char topic_route_qn[CBM_ROUTE_QN_SIZE]; - snprintf(topic_route_qn, sizeof(topic_route_qn), "__route__%s__%s", - ib->broker ? ib->broker : "async", ib->source_name); - const cbm_gbuf_node_t *topic_route = cbm_gbuf_find_by_qn(gbuf, topic_route_qn); - int64_t topic_route_id; - if (topic_route) { - topic_route_id = topic_route->id; - } else { - /* The config file IS the declaration that the topic/queue/schedule exists; - * upsert its Route node so the binding maps even when no code-side dispatch - * call created the node first (e.g. a standalone scheduler/subscription - * manifest). */ - topic_route_id = cbm_gbuf_upsert_node(gbuf, "Route", ib->source_name, topic_route_qn, - rel_path, 0, 0, ib->broker ? ib->broker : "async"); - if (topic_route_id <= 0) { - return 0; - } - } - char props[CBM_SZ_512]; - snprintf(props, sizeof(props), "{\"broker\":\"%s\",\"topic\":\"%s\",\"endpoint\":\"%s\"}", - ib->broker ? ib->broker : "async", ib->source_name, ib->target_url); - cbm_gbuf_insert_edge(gbuf, topic_route_id, url_route_id, "INFRA_MAPS", props); - return SKIP_ONE; -} - -void cbm_pipeline_process_infra_bindings(cbm_gbuf_t *gbuf, const cbm_file_info_t *files, - CBMFileResult **result_cache, cbm_result_spill_t *spill, - int file_count) { - int bindings = 0; - for (int i = 0; i < file_count; i++) { - if (!result_cache[i] && cbm_result_spill_infra_binding_count(spill, i) == 0) { - continue; - } - CBMFileResult *result = cbm_result_cache_acquire(result_cache, spill, i); - if (!result) { - continue; - } - for (int bi = 0; bi < result->infra_bindings.count; bi++) { - const CBMInfraBinding *ib = &result->infra_bindings.items[bi]; - if (ib->source_name && ib->target_url) { - bindings += process_one_infra_binding(gbuf, ib, files[i].rel_path); - } - } - cbm_result_cache_release(result_cache, i, result); - } - if (bindings > 0) { - char buf[CBM_SZ_16]; - snprintf(buf, sizeof(buf), "%d", bindings); - cbm_log_info("pass.infra_bindings", "linked", buf); - } -} - -bool cbm_pipeline_is_infra_file(const char *fp) { - return fp != NULL && - (strstr(fp, ".yaml") != NULL || strstr(fp, ".yml") != NULL || - strstr(fp, ".tf") != NULL || strstr(fp, ".hcl") != NULL || strstr(fp, ".toml") != NULL); -} - -/* True when a YAML key path denotes an UPSTREAM dependency, CONFIG value, or - * HEALTHCHECK target rather than an endpoint this service exposes. Such URLs - * (auth JWKS, downstream service base URLs, package-registry URLs, healthcheck - * curl targets) are NOT routes the service serves and must not mint Route nodes - * (#521). Exposed-endpoint keys (push_endpoint, post_url, callback, webhook) - * are intentionally absent here so they still produce infra Route nodes. */ -static bool is_upstream_config_key(const char *key_path) { - if (!key_path) { - /* No key context (e.g. flat string) — keep prior behaviour and mint. */ - return false; - } - static const char *const deny[] = {"jwks", "registry", "registries", "healthcheck", - "upstream", "_service_url", "auth", NULL}; - for (int i = 0; deny[i]; i++) { - if (strstr(key_path, deny[i]) != NULL) { - return true; - } - } - return false; -} - -/* Try to create an infra Route node from one string_ref. */ -static void try_upsert_infra_route(cbm_gbuf_t *gbuf, const CBMStringRef *sr, const char *fp) { - if (sr->kind != CBM_STRREF_URL || !sr->value || !strstr(sr->value, "://")) { - return; - } - /* Skip upstream/config/healthcheck URLs — they are not exposed routes (#521). */ - if (is_upstream_config_key(sr->key_path)) { - return; - } - char route_qn[CBM_ROUTE_QN_SIZE]; - snprintf(route_qn, sizeof(route_qn), "__route__infra__%s", sr->value); - char route_props[CBM_SZ_512]; - if (sr->key_path) { - snprintf(route_props, sizeof(route_props), "{\"source\":\"infra\",\"key_path\":\"%s\"}", - sr->key_path); - } else { - snprintf(route_props, sizeof(route_props), "{\"source\":\"infra\"}"); - } - cbm_gbuf_upsert_node(gbuf, "Route", sr->value, route_qn, fp, 0, 0, route_props); -} - -/* A URL string_ref that does NOT denote a route the service serves: a value - * containing whitespace is a command/sentence with an embedded URL (e.g. a - * Docker healthcheck `curl --fail http://... || exit 1`); a NULL key_path is a - * context-less/duplicate ref; an upstream/config/healthcheck key is an external - * dependency, not an exposed route. (#521) */ -/* True when one dotted key segment names an endpoint the service is called at: - * Pub/Sub push endpoints, Cloud Scheduler/Tasks `uri`, webhooks, callbacks, - * post URLs. Case, '_' and '-' are ignored (pushEndpoint == push_endpoint). */ -static bool is_endpoint_key_segment(const char *segment, size_t len) { - char norm[CBM_SZ_64]; - size_t n = 0; - for (size_t i = 0; i < len && n < sizeof(norm) - SKIP_ONE; i++) { - char c = segment[i]; - if (c == '_' || c == '-') { - continue; - } - norm[n++] = (char)((c >= 'A' && c <= 'Z') ? c - 'A' + 'a' : c); - } - norm[n] = '\0'; - static const char *const contains[] = {"endpoint", "webhook", "callback", "posturl", NULL}; - for (int i = 0; contains[i]; i++) { - if (strstr(norm, contains[i])) { - return true; - } - } - return strcmp(norm, "uri") == 0; -} - -/* Allow-list, not deny-list: generic keys such as `url`, `link` or `avatarUrl` - * hold documentation and data URLs (sponsor lists, profiles, images), so only - * endpoint-named keys mint infra Routes. Upstream/config keys stay denied. (#521) */ -bool cbm_pipeline_infra_route_key_allowed(const char *key_path) { - if (!key_path || is_upstream_config_key(key_path)) { - return false; - } - const char *segment = key_path; - for (;;) { - const char *dot = strchr(segment, '.'); - size_t len = dot ? (size_t)(dot - segment) : strlen(segment); - if (is_endpoint_key_segment(segment, len)) { - return true; - } - if (!dot) { - return false; - } - segment = dot + SKIP_ONE; - } -} - -static bool route_sr_denied(const CBMStringRef *sr) { - if (!sr->value || strchr(sr->value, ' ')) { - return true; - } - return !cbm_pipeline_infra_route_key_allowed(sr->key_path); -} - -void cbm_pipeline_extract_infra_routes(cbm_gbuf_t *gbuf, const cbm_file_info_t *files, - CBMFileResult **result_cache, cbm_result_spill_t *spill, - int file_count) { - /* DENY-WINS-BY-VALUE: the same URL is often extracted as several string_refs - * at different key_path granularities (full path, leaf key, flat). The Route - * node is keyed by VALUE, so it would be minted if ANY granularity passed the - * per-ref guard — e.g. a denied full path `registries.terraform-registry.url` - * is defeated by a sibling leaf `url`. So pass 1 collects every URL value - * denied under ANY of its refs; pass 2 mints only values never denied. (#521) */ - CBMHashTable *denied = cbm_ht_create(16); - /* `denied` borrows its keys; values from a loaded (spilled) result are - * copied here because the result is freed before pass 2 reads them. */ - CBMArena denied_keys; - cbm_arena_init(&denied_keys); - for (int pass = 0; pass < 2; pass++) { - for (int i = 0; i < file_count; i++) { - if (!cbm_pipeline_is_infra_file(files[i].rel_path) || - (!result_cache[i] && cbm_result_spill_string_ref_count(spill, i) == 0)) { - continue; - } - CBMFileResult *result = cbm_result_cache_acquire(result_cache, spill, i); - if (!result) { - continue; - } - const bool loaded = result != result_cache[i]; - for (int si = 0; si < result->string_refs.count; si++) { - const CBMStringRef *sr = &result->string_refs.items[si]; - if (sr->kind != CBM_STRREF_URL || !sr->value || !strstr(sr->value, "://")) { - continue; - } - if (pass == 0) { - if (denied && route_sr_denied(sr) && !cbm_ht_has(denied, sr->value)) { - const char *key = - loaded ? cbm_arena_strdup(&denied_keys, sr->value) : sr->value; - if (key) { - cbm_ht_set(denied, key, (void *)1); - } - } - } else if (!denied || !cbm_ht_has(denied, sr->value)) { - try_upsert_infra_route(gbuf, sr, files[i].rel_path); - } - } - cbm_result_cache_release(result_cache, i, result); - } - } - cbm_ht_free(denied); - cbm_arena_destroy(&denied_keys); -} - -/* Run decorator_tags, configlink, and route matching passes. */ -typedef void (*predump_pass_fn)(cbm_pipeline_ctx_t *); -static void predump_deco(cbm_pipeline_ctx_t *ctx) { - cbm_pipeline_pass_decorator_tags(ctx->gbuf, ctx->project_name); -} -static void predump_route(cbm_pipeline_ctx_t *ctx) { - cbm_pipeline_create_route_nodes(ctx->gbuf); -} -static void predump_sim(cbm_pipeline_ctx_t *ctx) { - cbm_pipeline_pass_similarity(ctx); -} -static void predump_sem(cbm_pipeline_ctx_t *ctx) { - cbm_pipeline_pass_semantic_edges(ctx); -} -static void predump_cfg(cbm_pipeline_ctx_t *ctx) { - cbm_pipeline_pass_configlink(ctx); -} -static void predump_complexity(cbm_pipeline_ctx_t *ctx) { - cbm_pipeline_pass_complexity(ctx); -} -static void predump_importance(cbm_pipeline_ctx_t *ctx) { - cbm_pipeline_pass_importance(ctx); -} - -static int run_predump_passes(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx) { - static const struct { - predump_pass_fn fn; - const char *name; - bool moderate_only; /* true = skip in fast mode */ - } passes[] = { - {predump_deco, "decorator_tags", false}, {predump_cfg, "configlink", false}, - {predump_route, "route_match", false}, {predump_sim, "similarity", true}, - {predump_sem, "semantic_edges", true}, {predump_complexity, "complexity", false}, - /* Importance runs LAST: it reads CALLS/USAGE (extraction) and TESTS - * (pass_tests, run before this loop). */ - {predump_importance, "importance", false}, - }; - /* Derived from the table: a hand-written count that lags an appended - * entry silently skips the last pass. */ - enum { PREDUMP_PASS_COUNT = (int)(sizeof(passes) / sizeof(passes[0])) }; - struct timespec t; - for (int i = 0; i < PREDUMP_PASS_COUNT && !check_cancel(p); i++) { - /* "moderate_only" passes (similarity/semantic edges) run in FULL, - * MODERATE and ADVANCED — they are skipped only in FAST. Compare - * explicitly against FAST rather than `> MODERATE` so ADVANCED - * (numerically 3) is not mistaken for a lighter mode than FULL. */ - if (passes[i].moderate_only && p->mode == CBM_MODE_FAST) { - continue; - } - /* #49: each derived pass allocates whole-graph scratch. Check before - * it starts so an over-budget run stops instead of stacking another - * pass's working set on top. */ - char phase[CBM_SZ_64]; - snprintf(phase, sizeof(phase), "before_%s", passes[i].name); - int gate_rc = cbm_pipeline_budget_gate(phase); - if (gate_rc != 0) { - return gate_rc; - } - cbm_clock_gettime(CLOCK_MONOTONIC, &t); - passes[i].fn(ctx); - cbm_log_info("pass.timing", "pass", passes[i].name, "elapsed_ms", - cbm_log_itoa((int)cbm_elapsed_ms(t))); - cbm_pipeline_log_phase_mem(passes[i].name); - } - return 0; -} - -/* Run the parallel pipeline path: extract, registry, resolve, infra, k8s. */ -static int run_parallel_passes(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, - const cbm_file_info_t *files, int file_count, int worker_count, - struct timespec *t) { - cbm_log_info("pipeline.mode", "mode", "parallel", "workers", cbm_log_itoa(worker_count), - "files", cbm_log_itoa(file_count)); - _Atomic int64_t shared_ids; - atomic_init(&shared_ids, cbm_gbuf_next_id(p->gbuf)); - CBMFileResult **cache = (CBMFileResult **)calloc(file_count, sizeof(CBMFileResult *)); - if (!cache) { - cbm_log_error("pipeline.err", "phase", "cache_alloc"); - return CBM_NOT_FOUND; - } - cbm_clock_gettime(CLOCK_MONOTONIC, t); - int rc = cbm_parallel_extract(ctx, files, file_count, cache, &shared_ids, worker_count); - cbm_log_info("pass.timing", "pass", "parallel_extract", "elapsed_ms", - cbm_log_itoa((int)cbm_elapsed_ms(*t))); - if (rc != 0 || check_cancel(p)) { - free(cache); - return rc != 0 ? rc : CBM_NOT_FOUND; - } - cbm_pipeline_sync_next_id(p->gbuf, &shared_ids); - /* extract -> registry handoff: return the extract phase's freed-but-retained - * allocator pages to the OS before registry_build allocates. On a 2x Linux - * index the extract peak holds ~13 GB of reclaimable pages (peak_mb 20.7 vs - * live rss_mb 7); not returning them pushed the process over the system - * memory-pressure threshold and got it SIGKILLed at registry entry. */ - cbm_mem_collect(); - cbm_log_info("mem.collect", "phase", "post_extract", "rss_mb", - cbm_log_itoa((int)(cbm_mem_rss() / (1024 * 1024)))); - cbm_clock_gettime(CLOCK_MONOTONIC, t); - rc = cbm_build_registry_from_cache(ctx, files, file_count, cache); - cbm_log_info("pass.timing", "pass", "registry_build", "elapsed_ms", - cbm_log_itoa((int)cbm_elapsed_ms(*t))); - cbm_pipeline_log_phase_mem("registry_build"); - /* #49: registry_build is the first post-extract whole-graph allocation. */ - if (rc == 0 && !check_cancel(p)) { - rc = cbm_pipeline_budget_gate("before_lsp_cross_prepare"); - } - if (rc != 0 || check_cancel(p)) { - for (int i = 0; i < file_count; i++) { - if (cache[i]) { - cbm_free_result(cache[i]); - } - } - free(cache); - return rc != 0 ? rc : CBM_NOT_FOUND; - } - /* Resolve workers must not reuse ids of nodes the registry build made. */ - cbm_pipeline_share_next_id(p->gbuf, &shared_ids); - /* Cross-file LSP precondition: build a project-wide CBMLSPDef[] - * once. The fused resolve_worker invokes cbm_pxc_run_one(_ts) per - * file using these defs + the file's IMPORTS map, so cross-file - * type-resolved CALLS land in result->resolved_calls before the - * CALLS-edge emission. This replaces the old sequential - * cbm_pipeline_pass_lsp_cross pass which re-read every source from - * disk and re-parsed every tree on a single thread (~520s on - * kubernetes). Soft-failure: NULL all_defs / NULL def_modules just - * mean cross-file LSP no-ops; per-file LSP already ran during - * extract. */ - cbm_clock_gettime(CLOCK_MONOTONIC, t); - /* Cross-file LSP (type-aware call/usage resolution across files) — the - * most expensive phase. CBM_DISABLE_LSP_CROSS=1 opts out (it can SIGSEGV - * on large TS projects — see #340/#344); with cross-LSP off, all_defs - * stays NULL and the fused resolver simply no-ops cross-file resolution - * (per-file LSP already ran during extract). */ - char cbm_lsp_cross_env[CBM_SZ_16]; - const bool run_cross_lsp = cbm_safe_getenv("CBM_DISABLE_LSP_CROSS", cbm_lsp_cross_env, - sizeof(cbm_lsp_cross_env), NULL) == NULL; - if (!run_cross_lsp) { - cbm_log_info("lsp_cross.skipped", "reason", "CBM_DISABLE_LSP_CROSS env set"); - } - /* Owns the per-language cross registries below and the def strings of - * spilled results, so it outlives all_defs. */ - CBMArena cross_lsp_arena; - cbm_arena_init(&cross_lsp_arena); - char **def_modules = NULL; - int def_count = 0; - CBMLSPDef *all_defs = NULL; - if (run_cross_lsp) { - def_modules = (char **)calloc((size_t)file_count, sizeof(char *)); - all_defs = def_modules ? cbm_pxc_collect_all_defs(ctx, cache, files, file_count, - ctx->project_name, def_modules, - &cross_lsp_arena, &def_count) - : NULL; - } - /* Build inverted index: module_qn → defs. The fused resolve_worker - * uses this to filter the global all_defs[] down to just the defs - * each file actually needs (own_module + imported modules) — the - * gopls "package summary" pattern. Drops per-file registry build - * cost from O(all_defs) to O(relevant_defs), typically 50-100× - * smaller per file. */ - CBMModuleDefIndex *module_def_index = - all_defs ? cbm_pxc_build_module_def_index(all_defs, def_count) : NULL; - /* Tier 2 full: pre-build per-language cross-LSP registries. - * Built ONCE here; shared READ-ONLY across all files of that language - * during resolve. Per-file work is then: parse + AST walk + O(1) lookups - * — no registry build, no Phase 1b mutations. Languages added so far: - * Go, Python. Others (C/C++, TS/JS, PHP, C#) fall back to per-file. */ - CBMCrossLspRegistries cross_registries = {0}; - if (all_defs) { - cross_registries.go = cbm_go_build_cross_registry(&cross_lsp_arena, all_defs, def_count); - cross_registries.python = - cbm_py_build_cross_registry(&cross_lsp_arena, all_defs, def_count); - cross_registries.c = cbm_c_build_cross_registry(&cross_lsp_arena, all_defs, def_count); - cross_registries.cs = cbm_cs_build_cross_registry(&cross_lsp_arena, all_defs, def_count); - cross_registries.ts = cbm_ts_build_cross_registry(&cross_lsp_arena, all_defs, def_count); - cross_registries.zig_args = cbm_pxc_collect_zig_args( - ctx, cache, files, file_count, def_modules, module_def_index, all_defs, def_count, - &cross_lsp_arena, &cross_registries.zig_arg_count); - /* Rust: NOT built here. The shared all_defs registry is built LAZILY on the - * first NULL-filter rust file (the amplifier files) inside cbm_parallel_resolve - * — repos whose rust files all filter to subsets never pay the build/RSS. */ - } - cbm_log_info("pass.timing", "pass", "lsp_cross_prepare", "elapsed_ms", - cbm_log_itoa((int)cbm_elapsed_ms(*t))); - cbm_pipeline_log_phase_mem("lsp_cross_prepare"); - /* #49: the cross-file registries are a whole-graph working set allocated - * after extract. Stop before resolve if they pushed RSS over the budget. */ - if (cbm_pipeline_budget_gate("before_parallel_resolve") != 0) { - rc = CBM_PIPELINE_ABORT_OVER_BUDGET; - } - if (rc == 0) { - cbm_clock_gettime(CLOCK_MONOTONIC, t); - rc = - cbm_parallel_resolve(ctx, files, file_count, cache, &shared_ids, worker_count, all_defs, - def_count, def_modules, module_def_index, &cross_registries); - cbm_log_info("pass.timing", "pass", "parallel_resolve", "elapsed_ms", - cbm_log_itoa((int)cbm_elapsed_ms(*t))); - cbm_pipeline_log_phase_mem("parallel_resolve"); - } - cbm_pxc_free_module_def_index(module_def_index); - free((void *)cross_registries.zig_args); - cbm_arena_destroy(&cross_lsp_arena); /* releases all per-lang registries */ - free(all_defs); - if (def_modules) { - for (int i = 0; i < file_count; i++) { - free(def_modules[i]); - } - free(def_modules); - } - cbm_pipeline_sync_next_id(p->gbuf, &shared_ids); - /* A budget abort means stop allocating: the infra passes would run over - * the same cache only to return the abort below. Other resolve errors keep - * main's behaviour and still run them. */ - if (rc != CBM_PIPELINE_ABORT_OVER_BUDGET) { - cbm_pipeline_extract_infra_routes(p->gbuf, files, cache, ctx->spill, file_count); - cbm_pipeline_process_infra_bindings(p->gbuf, files, cache, ctx->spill, file_count); - } - for (int i = 0; i < file_count; i++) { - if (cache[i]) { - cbm_free_result(cache[i]); - } - } - free(cache); - if (rc != 0) { - return rc; - } - cbm_clock_gettime(CLOCK_MONOTONIC, t); - cbm_pipeline_pass_k8s(ctx, files, file_count); - cbm_log_info("pass.timing", "pass", "k8s", "elapsed_ms", cbm_log_itoa((int)cbm_elapsed_ms(*t))); - return check_cancel(p) ? CBM_NOT_FOUND : 0; -} - -/* run_parallel_passes with a spill store (F1) the extract can park results - * in when it runs over its memory budget. The store's file lives under the - * cache dir and is deleted when the run ends. */ -static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, - const cbm_file_info_t *files, int file_count, int worker_count, - struct timespec *t) { - const char *cache_dir = cbm_resolve_cache_dir(); - char spill_dir[CBM_SZ_1K]; - if (cache_dir && snprintf(spill_dir, sizeof(spill_dir), "%s/spill", cache_dir) < - (int)sizeof(spill_dir)) { - ctx->spill = cbm_result_spill_open(spill_dir, file_count); - } - int rc = run_parallel_passes(p, ctx, files, file_count, worker_count, t); - if (rc == 0 && cbm_result_spill_load_failures(ctx->spill) > 0) { - /* A pass ran without some parked results: never publish that graph. */ - cbm_log_error("mem.spill.failed", "load_failures", - cbm_log_itoa((int)cbm_result_spill_load_failures(ctx->spill))); - rc = CBM_NOT_FOUND; - } - if (ctx->spill) { - int64_t parked = 0; - int64_t bytes = 0; - int64_t loads = 0; - cbm_result_spill_stats(ctx->spill, &parked, &bytes, &loads); - if (parked > 0) { - cbm_log_info("mem.spill.done", "parked", cbm_log_itoa((int)parked), "mb", - cbm_log_itoa((int)(bytes / (1024 * 1024))), "loads", - cbm_log_itoa((int)loads)); - } - cbm_result_spill_close(ctx->spill); - ctx->spill = NULL; - } - return rc; -} - -enum { PIPELINE_INCREMENTAL_FATAL = -2 }; - -/* True when the stored generation's Project node carries the current - * index_format. Unstamped generations predate the stamp. */ -static bool stored_index_format_current(const cbm_gbuf_t *stored, const char *project) { - static const char key[] = "\"index_format\":" CBM_INDEX_FORMAT_VERSION_STR; - const cbm_gbuf_node_t *node = cbm_gbuf_find_by_qn(stored, project); - const char *hit = node && node->properties_json ? strstr(node->properties_json, key) : NULL; - return hit && !isdigit((unsigned char)hit[sizeof(key) - 1]); -} - -/* Try the incremental pipeline against the project's committed cbm.zova - * workspace. Returns >= 0 if incremental was used (the return code), or -1 to - * proceed with full. CBM_PIPELINE_ABORT_OVER_BUDGET passes through unchanged. */ -static int try_incremental(cbm_pipeline_t *p, cbm_file_info_t *files, int file_count) { - char zova_path[CBM_SZ_1K]; - if (cbm_zova_user_database_path(zova_path, sizeof(zova_path)) == 0) { - cbm_zova_repository_t *repo = cbm_zova_repository_open(zova_path, p->project_name); - if (repo) { - cbm_project_summary_export_t summary = {0}; - const char *workspace_id = cbm_zova_repository_workspace_id(repo); - if (cbm_zova_repository_project_summary(repo, workspace_id, &summary) == - CBM_STORE_OK) { - cbm_store_project_summary_export_free(&p->saved_summary); - p->saved_summary = summary; - p->saved_summary_valid = true; - } - cbm_zova_workspace_snapshot_t snapshot = {0}; - cbm_gbuf_t *preloaded = cbm_gbuf_new(p->project_name, p->repo_path); - if (preloaded && cbm_zova_repository_export_incremental_snapshot_to_graph( - zova_path, workspace_id, &snapshot, preloaded) == 0) { - int hash_count = snapshot.file_hash_count; - if (hash_count > 0 && !stored_index_format_current(preloaded, p->project_name)) { - /* Reusing it would mix old-format QNs with new ones. */ - cbm_log_info("pipeline.route", "path", "format_change_reindex", - "required_format", CBM_INDEX_FORMAT_VERSION_STR, - "authority", "zova"); - p->format_migration = true; - } else if (hash_count > 0 && - file_count <= hash_count + (hash_count / PAIR_LEN)) { - cbm_log_info("pipeline.route", "path", "incremental", "stored_hashes", - cbm_log_itoa(hash_count), "authority", "zova"); - /* The exported snapshot owns all of its data. Release the - * read transaction before publishing the next generation, - * otherwise it pins the old WAL and prevents checkpointing. */ - cbm_zova_repository_close(repo); - p->last_route = CBM_PIPELINE_ROUTE_INCREMENTAL; - int rc = cbm_pipeline_run_incremental_zova(p, zova_path, files, file_count, - &snapshot, preloaded); - preloaded = NULL; - cbm_zova_workspace_snapshot_free(&snapshot); - if (rc == CBM_PIPELINE_INCREMENTAL_DECLINED) { - cbm_log_info("pipeline.route", "path", "incremental_declined_reindex", - "authority", "zova"); - return CBM_NOT_FOUND; - } - if (rc == CBM_PIPELINE_ABORT_OVER_BUDGET) { - return rc; - } - return rc == 0 ? 0 : PIPELINE_INCREMENTAL_FATAL; - } - if (hash_count > 0 && !p->format_migration) { - cbm_log_info("pipeline.route", "path", "mode_change_reindex", "stored_hashes", - cbm_log_itoa(hash_count), "discovered", cbm_log_itoa(file_count), - "authority", "zova"); - } - cbm_zova_workspace_snapshot_free(&snapshot); - } - cbm_gbuf_free(preloaded); - cbm_zova_repository_close(repo); - } - } - return CBM_NOT_FOUND; -} - -static int pipeline_publish_zova_user_database( - cbm_pipeline_t *p, cbm_gbuf_t *gbuf, const cbm_file_info_t *files, int file_count, - const cbm_file_hash_t *mode_skipped, int mode_skipped_count, - cbm_zova_workspace_snapshot_t *before) { - if (!p || !gbuf || file_count < 0 || mode_skipped_count < 0 || - (file_count > 0 && !files) || (mode_skipped_count > 0 && !mode_skipped)) { - return CBM_NOT_FOUND; - } - p->zova_publish_ms = 0.0; - memset(&p->zova_publish_stats, 0, sizeof(p->zova_publish_stats)); - struct timespec route_started = {0}; - cbm_clock_gettime(CLOCK_MONOTONIC, &route_started); - int capacity = file_count + mode_skipped_count; - cbm_zova_file_hash_input_t *hashes = - capacity > 0 ? calloc((size_t)capacity, sizeof(*hashes)) : NULL; - if (capacity > 0 && !hashes) return CBM_NOT_FOUND; - int count = 0; - for (int i = 0; i < file_count; i++) { - struct stat fst; - if (stat(files[i].path, &fst) != 0) continue; - hashes[count++] = (cbm_zova_file_hash_input_t){ - .file_path = files[i].rel_path, - .content_hash = "", - .mtime_ns = cbm_stat_mtime_ns(&fst), - .size_bytes = fst.st_size, - }; - } - for (int i = 0; i < mode_skipped_count; i++) { - hashes[count++] = (cbm_zova_file_hash_input_t){ - .file_path = mode_skipped[i].rel_path, - .content_hash = mode_skipped[i].sha256 ? mode_skipped[i].sha256 : "", - .mtime_ns = mode_skipped[i].mtime_ns, - .size_bytes = mode_skipped[i].size, - }; - } - cbm_zova_project_summary_input_t summary = {0}; - if (p->saved_summary_valid) { - summary.present = true; - summary.summary = p->saved_summary.summary; - summary.source_hash = p->saved_summary.source_hash; - summary.created_at = p->saved_summary.created_at; - summary.updated_at = p->saved_summary.updated_at; - } - cbm_zova_workspace_generation_result_t result = {0}; - int rc = before - ? cbm_gbuf_publish_zova_user_database_delta(gbuf, before, hashes, count, &summary, &result) - : cbm_gbuf_publish_zova_user_database(gbuf, hashes, count, &summary, &result); - free(hashes); - if (rc != 0) { - cbm_log_error("zova.single_file_publish", "project", p->project_name, - "reason", "publication_failed"); - return rc; - } - p->zova_publish_ms = result.publish_ms; - p->zova_publish_stats = (cbm_pipeline_zova_publish_stats_t){ - .completed = true, - .delta = result.publication_mode == CBM_ZOVA_PUBLICATION_MODE_DELTA, - .full_clear_count = result.full_clear_count, - .unchanged_rewrite_count = result.unchanged_rewrite_count, - .nodes_inserted = result.nodes_inserted, - .nodes_updated = result.nodes_updated, - .nodes_deleted = result.nodes_deleted, - .edges_inserted = result.edges_inserted, - .edges_deleted = result.edges_deleted, - .node_vectors_upserted = result.node_vectors_upserted, - .node_vectors_deleted = result.node_vectors_deleted, - .token_vectors_upserted = result.token_vectors_upserted, - .token_vectors_deleted = result.token_vectors_deleted, - .route_ms = cbm_elapsed_ms(route_started), - .normalization_ms = result.normalization_ms, - .model_nodes_ms = result.model_nodes_ms, - .model_edges_ms = result.model_edges_ms, - .model_edge_endpoint_ms = result.model_edge_endpoint_ms, - .model_edge_sort_ms = result.model_edge_sort_ms, - .model_edge_group_ms = result.model_edge_group_ms, - .model_edge_payload_ms = result.model_edge_payload_ms, - .model_edge_digest_ms = result.model_edge_digest_ms, - .model_edge_default_payloads = result.model_edge_default_payloads, - .model_edge_payload_scratch_edges = result.model_edge_payload_scratch_edges, - .model_hashes_ms = result.model_hashes_ms, - .model_vectors_ms = result.model_vectors_ms, - .model_digests_ms = result.model_digests_ms, - .writer_guard_ms = result.writer_guard_ms, - .database_init_ms = result.database_init_ms, - .database_open_ms = result.database_open_ms, - .transaction_begin_ms = result.transaction_begin_ms, - .transaction_body_ms = result.transaction_body_ms, - .transaction_commit_ms = result.transaction_commit_ms, - .database_close_ms = result.database_close_ms, - .clear_ms = result.clear_ms, - .finalize_ms = result.finalize_ms, - .canonical_files_ms = result.canonical_files_ms, - .canonical_nodes_ms = result.canonical_nodes_ms, - .canonical_edges_ms = result.canonical_edges_ms, - .canonical_hashes_ms = result.canonical_hashes_ms, - .fts_ms = result.fts_ms, - .token_metadata_ms = result.token_metadata_ms, - .native_graph_ms = result.native_graph_ms, - .native_graph_materialize_ms = result.native_graph_materialize_ms, - .native_graph_reset_ms = result.native_graph_reset_ms, - .native_graph_nodes_ms = result.native_graph_nodes_ms, - .native_graph_edges_ms = result.native_graph_edges_ms, - .native_graph_validate_ms = result.native_graph_validate_ms, - .native_graph_key_generation_ms = result.native_graph_key_generation_ms, - .native_graph_cleanup_ms = result.native_graph_cleanup_ms, - .native_vectors_ms = result.native_vectors_ms, - .fresh_validation_ms = result.fresh_validation_ms, - .fresh_index_ms = result.fresh_index_ms, - .fresh_commit_ms = result.fresh_commit_ms, - .fresh_build_ms = result.fresh_build_ms, - .readback_ms = result.readback_ms, - .snapshot_completed = before != NULL, - .snapshot_base_ms = before ? before->metrics.base_ms : 0.0, - .snapshot_optional_ms = before ? before->metrics.optional_ms : 0.0, - .snapshot_open_ms = before ? before->metrics.open_ms : 0.0, - .snapshot_header_ms = before ? before->metrics.header_ms : 0.0, - .snapshot_integrity_ms = before ? before->metrics.integrity_ms : 0.0, - .snapshot_nodes_sql_ms = before ? before->metrics.nodes_sql_ms : 0.0, - .snapshot_nodes_native_ms = before ? before->metrics.nodes_native_ms : 0.0, - .snapshot_nodes_finalize_ms = before ? before->metrics.nodes_finalize_ms : 0.0, - .snapshot_edges_sql_ms = before ? before->metrics.edges_sql_ms : 0.0, - .snapshot_edges_native_ms = before ? before->metrics.edges_native_ms : 0.0, - .snapshot_edges_finalize_ms = before ? before->metrics.edges_finalize_ms : 0.0, - .snapshot_hashes_summary_ms = before ? before->metrics.hashes_summary_ms : 0.0, - .snapshot_close_ms = before ? before->metrics.close_ms : 0.0, - .snapshot_graph_buffer_ms = before ? before->metrics.graph_buffer_ms : 0.0, - .snapshot_base_phase_mask = before ? before->metrics.base_phase_mask : 0, - .snapshot_node_rows = before ? before->metrics.node_rows : 0, - .snapshot_edge_rows = before ? before->metrics.edge_rows : 0, - .snapshot_file_hash_rows = before ? before->metrics.file_hash_rows : 0, - .snapshot_hydrated_components = before ? before->hydrated_components : 0, - .snapshot_topology_rows = before ? before->metrics.topology_rows : 0, - .snapshot_node_vector_rows = before ? before->metrics.node_vector_rows : 0, - .snapshot_token_vector_rows = before ? before->metrics.token_vector_rows : 0, - .snapshot_generation = before ? before->generation : 0, - }; - char generation_text[32], publish_ms_text[32], inserted_text[32], updated_text[32], deleted_text[32]; - snprintf(generation_text,sizeof(generation_text),"%lld",(long long)result.generation); - snprintf(publish_ms_text,sizeof(publish_ms_text),"%.3f",result.publish_ms); - snprintf(inserted_text,sizeof(inserted_text),"%llu",(unsigned long long)result.inserted_count); - snprintf(updated_text,sizeof(updated_text),"%llu",(unsigned long long)result.updated_count); - snprintf(deleted_text,sizeof(deleted_text),"%llu",(unsigned long long)result.deleted_count); - cbm_log_info("zova.single_file_publish", "project", p->project_name, - "generation", generation_text, - "publish_ms", publish_ms_text, - "publication_mode", - result.publication_mode == CBM_ZOVA_PUBLICATION_MODE_DELTA ? "delta" : "full", - "inserted", inserted_text, - "updated", updated_text, - "deleted", deleted_text); - return 0; -} - -int cbm_pipeline_publish_zova_user_database( - cbm_pipeline_t *p, cbm_gbuf_t *gbuf, const cbm_file_info_t *files, int file_count, - const cbm_file_hash_t *mode_skipped, int mode_skipped_count) { - return pipeline_publish_zova_user_database(p,gbuf,files,file_count,mode_skipped, - mode_skipped_count,NULL); -} - -int cbm_pipeline_publish_zova_user_database_delta( - cbm_pipeline_t *p, cbm_gbuf_t *gbuf, const cbm_file_info_t *files, int file_count, - const cbm_file_hash_t *mode_skipped, int mode_skipped_count, - cbm_zova_workspace_snapshot_t *before) { - if(!before)return CBM_NOT_FOUND; - return pipeline_publish_zova_user_database(p,gbuf,files,file_count,mode_skipped, - mode_skipped_count,before); -} - -/* Publish the graph and file hashes to the shared cbm.zova. */ -static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_info_t *files, int file_count, - struct timespec *t) { - cbm_clock_gettime(CLOCK_MONOTONIC, t); - /* #49: dump + publish materialize the whole graph again (normalization, - * native topology, FTS, vectors). Check before starting; nothing has been - * published yet, so an abort leaves the previous generation serving. */ - cbm_pipeline_log_phase_mem("pre_dump"); - int gate_rc = cbm_pipeline_budget_gate("before_dump"); - if (gate_rc != 0) { - return gate_rc; - } - char zova_dir[CBM_SZ_1K]; - if (cbm_zova_user_database_path(zova_dir, sizeof(zova_dir)) == 0) { - char *last_slash = strrchr(zova_dir, '/'); - if (last_slash) { - *last_slash = '\0'; - cbm_mkdir_p(zova_dir, CBM_DIR_PERMS); - } - } - /* Capture committed counts before preparing the dump, which releases the - * graph buffer indexes (graph_buffer.c), after which cbm_gbuf_node_count() - * returns 0 and the #334 plausibility gate would never fire. */ - cbm_pipeline_run_before_dump_hook(p->gbuf); - p->committed_nodes = cbm_gbuf_node_count(p->gbuf); - p->committed_edges = cbm_gbuf_edge_count(p->gbuf); - int prepare_rc = cbm_gbuf_prepare_zova_dump(p->gbuf); - if (prepare_rc != 0) { - cbm_log_error("pipeline.err", "phase", "zova_prepare"); - return prepare_rc; - } - cbm_pipeline_log_phase_mem("zova_prepare"); - /* Preparing the dump releases the gbuf indexes; if RSS is still over - * budget now, publishing would only raise it further. Abort before the - * atomic publication starts. */ - gate_rc = cbm_pipeline_budget_gate("before_publish"); - if (gate_rc != 0) { - return gate_rc; - } - int publish_rc = cbm_pipeline_publish_zova_user_database(p, p->gbuf, files, file_count, NULL, 0); - if (publish_rc != 0) { - cbm_log_error("pipeline.err", "phase", "zova_single_file"); - return publish_rc; - } - cbm_pipeline_log_phase_mem("zova_publish"); - return 0; -} - -/* Run githistory pass. */ -static int run_githistory(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx) { - struct timespec t_gh; - cbm_clock_gettime(CLOCK_MONOTONIC, &t_gh); - - cbm_githistory_result_t gh_result = {0}; - cbm_thread_t gh_thread; - bool gh_threaded = false; - gh_compute_arg_t gh_arg = {.repo_path = ctx->repo_path, .result = &gh_result}; - - if (p->mode != CBM_MODE_FAST) { - if (cbm_default_worker_count(true) > SKIP_ONE) { - if (cbm_thread_create(&gh_thread, 0, gh_compute_thread_fn, &gh_arg) == 0) { - gh_threaded = true; - } - } - if (!gh_threaded) { - cbm_pipeline_githistory_compute(ctx->repo_path, &gh_result); - cbm_log_info("pass.timing", "pass", "githistory_compute", "elapsed_ms", - cbm_log_itoa((int)cbm_elapsed_ms(t_gh))); - } - } else { - cbm_log_info("pass.skip", "pass", "githistory", "reason", "fast_mode"); - } - - if (gh_threaded) { - cbm_thread_join(&gh_thread); - cbm_log_info("pass.timing", "pass", "githistory_compute", "elapsed_ms", - cbm_log_itoa((int)cbm_elapsed_ms(t_gh))); - } - - int gh_edges = 0; - if (gh_result.count > 0 || gh_result.file_temporal_count > 0) { - gh_edges = cbm_pipeline_githistory_apply(ctx, &gh_result); - } - cbm_log_info("pass.done", "pass", "githistory", "commits", cbm_log_itoa(gh_result.commit_count), - "edges", cbm_log_itoa(gh_edges)); - free(gh_result.couplings); - free(gh_result.file_temporal); - return 0; -} - -/* ── Pipeline run ────────────────────────────────────────────────── */ - -/* Run tests + git history. Returns 0 on success. */ -static int run_tests_and_history(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, - const cbm_file_info_t *files, int file_count) { - struct timespec t; - cbm_clock_gettime(CLOCK_MONOTONIC, &t); - CBM_PROF_START(t_tests); - int rc = cbm_pipeline_pass_tests(ctx, files, file_count); - CBM_PROF_END_N("pipeline", "pass_tests", t_tests, file_count); - cbm_log_info("pass.timing", "pass", "tests", "elapsed_ms", - cbm_log_itoa((int)cbm_elapsed_ms(t))); - if (rc == 0 && !check_cancel(p)) { - CBM_PROF_START(t_gh); - rc = run_githistory(p, ctx); - CBM_PROF_END("pipeline", "pass_githistory", t_gh); - } - if (check_cancel(p)) { - return CBM_NOT_FOUND; - } - return rc; -} - -/* Run tests, git history, predump passes, and dump+persist. */ -static int run_post_extraction(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, - const cbm_file_info_t *files, int file_count) { - /* #49: extract already enforces the budget. Everything from here on works - * on the whole graph, so gate the entry too. */ - cbm_pipeline_log_phase_mem("extract_done"); - int rc = cbm_pipeline_budget_gate("before_tests_and_history"); - if (rc != 0) { - return rc; - } - rc = run_tests_and_history(p, ctx, files, file_count); - if (rc != 0) { - return rc; - } - cbm_pipeline_log_phase_mem("tests_and_history"); - - CBM_PROF_START(t_predump); - rc = run_predump_passes(p, ctx); - CBM_PROF_END("pipeline", "3_predump_passes_total", t_predump); - if (rc != 0) { - return rc; - } - - if (!check_cancel(p)) { - struct timespec t; - CBM_PROF_START(t_dump); - rc = dump_and_persist_hashes(p, files, file_count, &t); - CBM_PROF_END("pipeline", "4_dump_and_persist", t_dump); - } - return rc; -} - -/* Run structure + extraction passes. */ -static int run_structure_and_extraction(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, - const cbm_file_info_t *files, int file_count) { - struct timespec t; - cbm_clock_gettime(CLOCK_MONOTONIC, &t); - CBM_PROF_START(t_struct); - pass_structure(p, ctx->stem_marks, files, file_count); - CBM_PROF_END_N("pipeline", "pass_structure", t_struct, file_count); - cbm_log_info("pass.timing", "pass", "structure", "elapsed_ms", - cbm_log_itoa((int)cbm_elapsed_ms(t))); - if (check_cancel(p)) { - return CBM_NOT_FOUND; - } - - int worker_count = cbm_default_worker_count(true); - CBM_PROF_START(t_extract_total); - int rc = run_parallel_pipeline(p, ctx, files, file_count, worker_count, &t); - CBM_PROF_END_N("pipeline", "2_extraction_total", t_extract_total, file_count); - if (check_cancel(p)) { - return CBM_NOT_FOUND; - } - if (rc == 0) { - cbm_pipeline_stamp_coverage(p, p->gbuf); - } - return rc; -} - -/* The stem collisions decide Folder QNs (structure) and Module QNs - * (extraction); both phases read the one set. */ -static int run_extraction_phase(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, - const cbm_file_info_t *files, int file_count) { - cbm_stem_marks_t marks; - cbm_stem_marks_init(&marks, p->project_name, files, file_count); - ctx->stem_marks = &marks; - int rc = run_structure_and_extraction(p, ctx, files, file_count); - ctx->stem_marks = NULL; - cbm_stem_marks_free(&marks); - return rc; -} - -int cbm_pipeline_run(cbm_pipeline_t *p) { - if (!p) { - return CBM_NOT_FOUND; - } - p->last_route = CBM_PIPELINE_ROUTE_UNKNOWN; - p->format_migration = false; - - CBM_PROF_START(t_pipeline_total); - struct timespec t0; - cbm_clock_gettime(CLOCK_MONOTONIC, &t0); - cbm_path_alias_collection_t *path_aliases = NULL; - cbm_pipeline_ctx_t ctx = {0}; /* zeroed first: cleanup reads ctx.pkgmap */ - - /* C/C++ #define Macro nodes (#375) dominate extraction on macro-dense repos - * (≈49% of nodes on the Linux kernel), so gate them to full mode — moderate - * and fast skip them entirely. Set before any extraction dispatch. */ - cbm_set_macro_extraction(p->mode == CBM_MODE_FULL); - - /* Load user-defined extension overrides (fail-open: NULL on error) */ - CBM_PROF_START(t_userconfig); - p->userconfig = cbm_userconfig_load(p->repo_path); - cbm_set_user_lang_config(p->userconfig); - CBM_PROF_END("pipeline", "0_userconfig_load", t_userconfig); - - /* Phase 1: Discover files */ - CBM_PROF_START(t_discover); - cbm_discover_opts_t opts = { - .mode = p->mode, - .ignore_file = NULL, - .max_file_size = 0, - }; - cbm_file_info_t *files = NULL; - int file_count = 0; - /* Capture skipped subtrees on the pipeline so the MCP layer can report - * which directories were excluded (#411). Replace any prior list (e.g. a - * re-run on the same pipeline) to avoid leaking the previous one. */ - cbm_discover_free_excluded(p->excluded_dirs, p->excluded_count); - p->excluded_dirs = NULL; - p->excluded_count = 0; - bool discover_degraded = false; - int rc = cbm_discover_ex(p->repo_path, &opts, &files, &file_count, &p->excluded_dirs, - &p->excluded_count, &discover_degraded); - p->discover_degraded = discover_degraded; - if (discover_degraded) { - cbm_log_warn("pipeline.discover", "degraded", "true"); - } - if (rc != 0) { - cbm_log_error("pipeline.err", "phase", "discover", "rc", cbm_log_itoa(rc)); - } - CBM_PROF_END_N("pipeline", "1_discover", t_discover, file_count); - cbm_log_info("pipeline.discover", "files", cbm_log_itoa(file_count), "elapsed_ms", - cbm_log_itoa((int)cbm_elapsed_ms(t0))); - if (rc != 0 || check_cancel(p)) { - rc = CBM_NOT_FOUND; - goto cleanup; - } - - /* Check for existing DB → try incremental or delete for reindex */ - rc = try_incremental(p, files, file_count); - if (rc == PIPELINE_INCREMENTAL_FATAL) { - cbm_discover_free(files, file_count); - return CBM_NOT_FOUND; - } - if (rc == CBM_PIPELINE_ABORT_OVER_BUDGET) { - /* A full run would hit the same budget; the served generation stays. */ - cbm_discover_free(files, file_count); - return rc; - } - if (rc >= 0) { - cbm_discover_free(files, file_count); - return rc; - } - cbm_log_info("pipeline.route", "path", "full"); - p->last_route = CBM_PIPELINE_ROUTE_FULL; - - /* Phase 2: Create graph buffer and registry */ - p->gbuf = cbm_gbuf_new(p->project_name, p->repo_path); - p->registry = cbm_registry_new(); - - /* Phase 2b: Load build-tool path aliases (tsconfig/jsconfig today). NULL - * when no usable configs are found — non-TS projects pay nothing. */ - path_aliases = - cbm_load_path_aliases_excluded(p->repo_path, p->excluded_dirs, p->excluded_count); - - /* Build shared context for pass functions */ - ctx = (cbm_pipeline_ctx_t){ - .project_name = p->project_name, - .repo_path = p->repo_path, - .gbuf = p->gbuf, - .registry = p->registry, - .cancelled = &p->cancelled, - .pipeline = p, /* so passes can record per-file skips (Track B) */ - .mode = (int)p->mode, - .path_aliases = path_aliases, - .excluded_dirs = p->excluded_dirs, - .excluded_count = p->excluded_count, - }; - - rc = run_extraction_phase(p, &ctx, files, file_count); - if (rc != 0) { - goto cleanup; - } - - rc = run_post_extraction(p, &ctx, files, file_count); - if (rc != 0) { - goto cleanup; - } - - cbm_pipeline_log_phase_mem("pipeline_done"); - cbm_log_info("pipeline.done", "nodes", cbm_log_itoa(cbm_gbuf_node_count(p->gbuf)), "edges", - cbm_log_itoa(cbm_gbuf_edge_count(p->gbuf)), "elapsed_ms", - cbm_log_itoa((int)cbm_elapsed_ms(t0))); - CBM_PROF_END("pipeline", "TOTAL", t_pipeline_total); - -cleanup: - cbm_pkgmap_free(ctx.pkgmap); - ctx.pkgmap = NULL; - cbm_pipeline_set_pkgmap(NULL); - cbm_discover_free(files, file_count); - cbm_gbuf_free(p->gbuf); - p->gbuf = NULL; - cbm_registry_free(p->registry); - p->registry = NULL; - cbm_path_alias_collection_free(path_aliases); - /* Clear and free user extension config */ - cbm_set_user_lang_config(NULL); - cbm_userconfig_free(p->userconfig); - p->userconfig = NULL; - return rc; -} diff --git a/src/pipeline/pipeline.zig b/src/pipeline/pipeline.zig new file mode 100644 index 00000000..5ce26932 --- /dev/null +++ b/src/pipeline/pipeline.zig @@ -0,0 +1,1740 @@ +//! pipeline.zig — Indexing pipeline orchestrator. +//! +//! Coordinates multi-pass indexing: +//! 1. Discover files +//! 2. Build structure (Project/Folder/Package/File nodes) +//! 3. Bulk load sources (read + LZ4 HC compress) +//! 4. Extract definitions (fused: extract + write nodes + build registry) +//! 5. Resolve imports, calls, usages, semantic edges +//! 6. Post-passes: tests, communities, HTTP links, git history +//! 7. Dump graph buffer to SQLite + +const std = @import("std"); +const c = @import("c"); +const interop = @import("../foundation/interop.zig"); +const util = @import("util.zig"); + +const CStr = util.CStr; +const MutStr = util.MutStr; +const ix = util.ix; +const streq = util.streq; +const ptrDiff = util.ptrDiff; +const HashTable = c.CBMHashTable; +const Gbuf = c.cbm_gbuf_t; +const Node = c.cbm_gbuf_node_t; +const Ctx = c.cbm_pipeline_ctx_t; +const FileInfo = c.cbm_file_info_t; +const FileError = c.cbm_file_error_t; +const FileResult = c.CBMFileResult; +const Spill = c.cbm_result_spill_t; +const StemMarks = c.cbm_stem_marks_t; +const Timespec = c.struct_timespec; + +const CBM_DIR_PERMS = 0o755; +const BYTES_PER_MB = 1024 * 1024; + +// ── Global pkgmap (one active pipeline at a time) ─────────────────── + +var g_pkgmap: ?*HashTable = null; + +pub export fn cbm_pipeline_get_pkgmap() ?*HashTable { + return g_pkgmap; +} + +pub export fn cbm_pipeline_set_pkgmap(map: ?*HashTable) void { + g_pkgmap = map; +} + +// ── Internal state ────────────────────────────────────────────────── + +/// struct cbm_pipeline. The headers only name it as an opaque type. +const Pipeline = extern struct { + repo_path: MutStr, + db_path: MutStr, + project_name: MutStr, + git_ctx: c.cbm_git_context_t, + branch_qn: MutStr, + mode: c.cbm_index_mode_t, + /// atomic_int: read and written through @atomicLoad / @atomicStore. + cancelled: c_int, + + // Indexing state (set during run) + gbuf: ?*Gbuf, + registry: ?*c.cbm_registry_t, + + /// Directory subtrees skipped during discovery (rel paths). Captured from + /// cbm_discover_ex so the MCP layer can report excluded subtrees (#411). + /// Owned by the pipeline; freed in cbm_pipeline_free. + excluded_dirs: [*c]MutStr, + excluded_count: c_int, + + /// Set when discovery could not fully materialize the tree (bounded walk + /// stack full / allocation failure). The MCP layer reports a degraded index + /// instead of a complete-looking partial one (#17, upstream 03dc9c91). + discover_degraded: bool, + + /// Per-file indexing failures (skipped files) surfaced via MCP/CLI/logfile + /// (Stage 2 / Track B). A skip is the expected handled outcome of a bad or + /// oversized file — the run still succeeds ("indexed"). Owned by the + /// pipeline; freed in cbm_pipeline_free. + file_errors: [*c]FileError, + file_errors_count: c_int, + file_errors_cap: c_int, + + /// User-defined extension overrides (loaded once per run) + userconfig: ?*c.cbm_userconfig_t, + + /// Committed graph size at dump time (-1 = dump did not run). #334 gate axis. + committed_nodes: c_int, + committed_edges: c_int, + zova_publish_ms: f64, + last_route: c.cbm_pipeline_route_t, + /// The last run rebuilt a generation stamped with an older index format. + format_migration: bool, + zova_publish_stats: c.cbm_pipeline_zova_publish_stats_t, + + saved_summary: c.cbm_project_summary_export_t, + saved_summary_valid: bool, +}; + +/// (int)x for a size_t, as C converts it. +fn truncInt(v: usize) c_int { + return @bitCast(@as(c_uint, @truncate(v))); +} + +/// cbm_log_itoa of an int-valued expression. +fn itoa(v: c_int) CStr { + return c.cbm_log_itoa(v); +} + +/// cbm_log_itoa of a byte count in whole megabytes. +fn mbText(bytes: usize) CStr { + return c.cbm_log_itoa(truncInt(bytes / BYTES_PER_MB)); +} + +/// cbm_log_itoa of the milliseconds elapsed since `start`. +fn elapsedText(start: Timespec) CStr { + return c.cbm_log_itoa(@intFromFloat(c.cbm_elapsed_ms(start))); +} + +fn logError(msg: interop.Str, kvs: anytype) void { + interop.log(c.CBM_LOG_ERROR, msg, kvs); +} + +fn now(t: *Timespec) void { + _ = c.cbm_clock_gettime(c.CLOCK_MONOTONIC, t); +} + +/// Log current + peak RSS and the budget at a pipeline phase boundary (memory +/// profiling). The process peak is cumulative, so the boundary where peak_mb +/// first rises is the phase that set it (#49). +pub export fn cbm_pipeline_log_phase_mem(phase: CStr) void { + interop.logInfo("mem.phase", .{ + "phase", phase, + "rss_mb", mbText(c.cbm_mem_rss()), + "peak_mb", mbText(c.cbm_mem_peak_rss()), + "budget_mb", mbText(c.cbm_mem_budget()), + }); +} + +/// Post-extraction budget gate. The extract phase enforces the budget with +/// back-pressure and result spill, but registry/resolve/derived passes/dump +/// work on the whole graph at once. Reclaim freed pages, then fail the attempt +/// whole if the resident graph is still over budget, before anything is +/// published, so the previously served generation keeps answering. +/// +/// Only an explicit CBM_MEM_BUDGET_MB hard-fails. The detected RAM-fraction +/// default stays a soft target: it is sized for extraction's spill heuristics, +/// and a run that overshoots it after extraction already finishes today (#49 +/// measurements). Hard-failing it would turn successful indexes into repeatable +/// over_memory_budget errors with no way to configure out but the env var. +/// Returns 0 when under budget, when the budget is implicit, or when none is +/// configured. Every call site is single-threaded: the workers have joined, so +/// there is no transient to wait for. +pub export fn cbm_pipeline_budget_gate(phase: CStr) c_int { + if (c.cbm_mem_budget() == 0 or !c.cbm_mem_budget_explicit()) return 0; + if (!c.cbm_mem_over_budget()) return 0; + c.cbm_mem_collect(); + if (!c.cbm_mem_over_budget()) return 0; + logError("mem.budget.exceeded", .{ + "rss_mb", mbText(c.cbm_mem_rss()), + "budget_mb", mbText(c.cbm_mem_budget()), + "phase", @as(CStr, if (phase != null) phase else "post_extract"), + "action", "fail_whole", + }); + return c.CBM_PIPELINE_ABORT_OVER_BUDGET; +} + +// ── Lifecycle ─────────────────────────────────────────────────────── + +pub export fn cbm_pipeline_new(repo_path: CStr, db_path: CStr, mode: c.cbm_index_mode_t) ?*Pipeline { + if (repo_path == null) return null; + + const p: *Pipeline = @ptrCast(@alignCast(c.calloc(1, @sizeOf(Pipeline)) orelse return null)); + + p.repo_path = c.strdup(repo_path); + p.db_path = if (db_path != null) c.strdup(db_path) else null; + p.project_name = c.cbm_project_name_from_path(repo_path); + _ = c.cbm_git_context_resolve(repo_path, &p.git_ctx); + p.branch_qn = c.cbm_git_context_branch_qn(p.project_name, &p.git_ctx); + p.mode = mode; + p.committed_nodes = -1; + p.committed_edges = -1; + p.cancelled = 0; + + return p; +} + +pub export fn cbm_pipeline_set_project_name(p_c: ?*Pipeline, name: CStr) bool { + const p = p_c orelse return false; + if (name == null or name[0] == 0) return false; + + const normalized: MutStr = c.cbm_project_name_from_path(name); + if (normalized == null) return false; + if (!c.cbm_validate_project_name(normalized)) { + c.free(normalized); + return false; + } + + c.free(p.project_name); + p.project_name = normalized; + c.free(p.branch_qn); + p.branch_qn = c.cbm_git_context_branch_qn(p.project_name, &p.git_ctx); + return true; +} + +pub export fn cbm_pipeline_free(p_c: ?*Pipeline) void { + const p = p_c orelse return; + c.free(p.repo_path); + c.free(p.db_path); + c.free(p.project_name); + c.cbm_discover_free_excluded(p.excluded_dirs, p.excluded_count); + p.excluded_dirs = null; + p.excluded_count = 0; + for (0..ix(p.file_errors_count)) |i| { + c.free(p.file_errors[i].path); + c.free(p.file_errors[i].reason); + c.free(p.file_errors[i].phase); + } + c.free(p.file_errors); + p.file_errors = null; + p.file_errors_count = 0; + p.file_errors_cap = 0; + c.free(p.branch_qn); + c.cbm_store_project_summary_export_free(&p.saved_summary); + p.saved_summary_valid = false; + c.cbm_git_context_free(&p.git_ctx); + // gbuf, store, registry freed during/after run + // Defensively free userconfig in case run() was never called or panicked + if (p.userconfig != null) { + c.cbm_set_user_lang_config(null); + c.cbm_userconfig_free(p.userconfig); + p.userconfig = null; + } + c.free(p); +} + +pub export fn cbm_pipeline_cancel(p: ?*Pipeline) void { + if (p) |pl| @atomicStore(c_int, &pl.cancelled, 1, .seq_cst); +} + +pub export fn cbm_pipeline_project_name(p: ?*const Pipeline) CStr { + return if (p) |pl| pl.project_name else null; +} + +pub export fn cbm_pipeline_repo_path(p: ?*const Pipeline) CStr { + return if (p) |pl| pl.repo_path else null; +} + +pub export fn cbm_pipeline_cancelled_ptr(p: ?*Pipeline) ?*c_int { + return if (p) |pl| &pl.cancelled else null; +} + +pub export fn cbm_pipeline_get_mode(p: ?*const Pipeline) c_int { + return if (p) |pl| @intCast(pl.mode) else 0; +} + +pub export fn cbm_pipeline_get_excluded(p: ?*const Pipeline, out: ?*[*c]MutStr, count: ?*c_int) void { + if (out) |o| o.* = if (p) |pl| pl.excluded_dirs else null; + if (count) |n| n.* = if (p) |pl| pl.excluded_count else 0; +} + +pub export fn cbm_pipeline_discover_degraded(p: ?*const Pipeline) bool { + return if (p) |pl| pl.discover_degraded else false; +} + +/// NULL-safe heap strdup (avoids a strdup dependency + guards NULL inputs). +fn feStrdup(s: CStr) MutStr { + if (s == null) return null; + const n = c.strlen(s) + 1; + const d: MutStr = @ptrCast(c.malloc(n)); + if (d != null) @memcpy(d[0..n], s[0..n]); + return d; +} + +pub export fn cbm_pipeline_phase_is_partial(phase: CStr) bool { + return phase != null and (streq(phase, "parse_partial") or streq(phase, "parse_unusable")); +} + +pub export fn cbm_pipeline_add_file_error(p_c: ?*Pipeline, path: CStr, reason: CStr, phase: CStr) void { + const p = p_c orelse return; + if (p.file_errors_count >= p.file_errors_cap) { + const ncap: c_int = if (p.file_errors_cap != 0) p.file_errors_cap * 2 else 16; + const grown = interop.reallocN(FileError, p.file_errors, ix(ncap)); + // Never abort indexing just to record a skip — drop this record. + if (grown == null) return; + p.file_errors = grown; + p.file_errors_cap = ncap; + } + const e = util.at(p.file_errors, ix(p.file_errors_count)); + e.path = feStrdup(path); + e.reason = feStrdup(reason); + e.phase = feStrdup(phase); + p.file_errors_count += 1; +} + +pub export fn cbm_pipeline_get_file_errors(p: ?*const Pipeline, out: ?*[*c]FileError, count: ?*c_int) void { + if (out) |o| o.* = if (p) |pl| pl.file_errors else null; + if (count) |n| n.* = if (p) |pl| pl.file_errors_count else 0; +} + +fn orEmpty(s: CStr) CStr { + return if (s != null) s else ""; +} + +/// Rebuild a File node's properties with its coverage stamp. The extension +/// key mirrors pass_structure / incr_add_changed_file_structure. +fn stampFileCoverage(p: *Pipeline, gbuf: ?*Gbuf, e: *const FileError, partial: bool) bool { + const file_qn: MutStr = c.cbm_pipeline_fqn_compute(p.project_name, e.path, "__file__"); + if (file_qn == null) return false; + const slash: CStr = c.strrchr(e.path, '/'); + const basename: CStr = if (slash != null) slash + 1 else e.path; + const ext: CStr = c.strrchr(basename, '.'); + const doc = c.yyjson_mut_doc_new(null); + const root = c.yyjson_mut_obj(doc); + c.yyjson_mut_doc_set_root(doc, root); + _ = c.yyjson_mut_obj_add_str(doc, root, "extension", orEmpty(ext)); + if (partial) { + _ = c.yyjson_mut_obj_add_str(doc, root, "coverage", c.CBM_COVERAGE_PARTIAL); + _ = c.yyjson_mut_obj_add_str(doc, root, "parse_error_ranges", orEmpty(e.reason)); + if (streq(e.phase, "parse_unusable")) { + _ = c.yyjson_mut_obj_add_bool(doc, root, "parse_unusable", true); + } + } else { + _ = c.yyjson_mut_obj_add_str(doc, root, "coverage", c.CBM_COVERAGE_MISSED); + _ = c.yyjson_mut_obj_add_str(doc, root, "coverage_phase", e.phase); + _ = c.yyjson_mut_obj_add_str(doc, root, "coverage_reason", orEmpty(e.reason)); + } + const props: MutStr = c.yyjson_mut_write(doc, 0, null); + c.yyjson_mut_doc_free(doc); + const stamped = props != null and c.cbm_gbuf_set_node_props(gbuf, file_qn, props); + c.free(props); + c.free(file_qn); + return stamped; +} + +pub export fn cbm_pipeline_stamp_coverage(p_c: ?*Pipeline, gbuf: ?*Gbuf) void { + const p = p_c orelse return; + if (gbuf == null) return; + var partial: c_int = 0; + var missed: c_int = 0; + // Partial first, so a file recorded under both keeps the stronger + // "missed" stamp. + for (0..2) |pass| { + for (0..ix(p.file_errors_count)) |i| { + const e = util.at(p.file_errors, i); + if (e.path == null or e.phase == null) continue; + const is_partial = cbm_pipeline_phase_is_partial(e.phase); + if (is_partial != (pass == 0)) continue; + if (stampFileCoverage(p, gbuf, e, is_partial)) { + if (is_partial) partial += 1 else missed += 1; + } + } + } + if (partial > 0 or missed > 0) { + interop.logInfo("index.coverage", .{ "partial", itoa(partial), "missed", itoa(missed) }); + } +} + +pub export fn cbm_pipeline_get_committed_counts(p: ?*const Pipeline, nodes: ?*c_int, edges: ?*c_int) void { + if (nodes) |n| n.* = if (p) |pl| pl.committed_nodes else -1; + if (edges) |e| e.* = if (p) |pl| pl.committed_edges else -1; +} + +pub export fn cbm_pipeline_get_zova_publish_ms(p: ?*const Pipeline) f64 { + return if (p) |pl| pl.zova_publish_ms else 0.0; +} + +pub export fn cbm_pipeline_get_last_route(p: ?*const Pipeline) c.cbm_pipeline_route_t { + return if (p) |pl| pl.last_route else c.CBM_PIPELINE_ROUTE_UNKNOWN; +} + +// ── Main-buffer id counter sync after parallel phases (#50) ───────── + +const IdSyncHook = ?*const fn (?*Gbuf, ?*anyopaque) callconv(.c) void; +const BeforeDumpHook = ?*const fn (?*const Gbuf, ?*anyopaque) callconv(.c) void; +const ShareIdsHook = ?*const fn (?*Gbuf, [*c]i64, ?*anyopaque) callconv(.c) void; + +threadlocal var g_id_sync_hook: IdSyncHook = null; +threadlocal var g_id_sync_hook_ctx: ?*anyopaque = null; +threadlocal var g_before_dump_hook: BeforeDumpHook = null; +threadlocal var g_before_dump_hook_ctx: ?*anyopaque = null; + +pub export fn cbm_pipeline_set_id_sync_hook_for_tests(hook: IdSyncHook, ctx: ?*anyopaque) void { + g_id_sync_hook = hook; + g_id_sync_hook_ctx = ctx; +} + +pub export fn cbm_pipeline_set_before_dump_hook_for_tests(hook: BeforeDumpHook, ctx: ?*anyopaque) void { + g_before_dump_hook = hook; + g_before_dump_hook_ctx = ctx; +} + +pub export fn cbm_pipeline_run_before_dump_hook(gb: ?*const Gbuf) void { + if (g_before_dump_hook) |hook| hook(gb, g_before_dump_hook_ctx); +} + +pub export fn cbm_pipeline_sync_next_id(gb: ?*Gbuf, shared_ids: *i64) void { + if (g_id_sync_hook) |hook| hook(gb, g_id_sync_hook_ctx); + // Raise only. The merge leaves the main counter above the shared one when + // merged edges take main-counter ids; lowering it to the shared value + // handed those ids out a second time. + c.cbm_gbuf_raise_next_id(gb, @atomicLoad(i64, shared_ids, .seq_cst)); +} + +// ── Shared id counter raise before parallel resolve (#88) ─────────── + +threadlocal var g_share_ids_hook: ShareIdsHook = null; +threadlocal var g_share_ids_hook_ctx: ?*anyopaque = null; + +pub export fn cbm_pipeline_set_share_ids_hook_for_tests(hook: ShareIdsHook, ctx: ?*anyopaque) void { + g_share_ids_hook = hook; + g_share_ids_hook_ctx = ctx; +} + +pub export fn cbm_pipeline_share_next_id(gb: ?*Gbuf, shared_ids: *i64) void { + if (g_share_ids_hook) |hook| hook(gb, shared_ids, g_share_ids_hook_ctx); + // No worker runs here, so a plain load and store is enough. Raise only: + // the shared counter may already be above the main one. + const main_next = c.cbm_gbuf_next_id(gb); + if (main_next > @atomicLoad(i64, shared_ids, .seq_cst)) { + @atomicStore(i64, shared_ids, main_next, .seq_cst); + } +} + +pub export fn cbm_pipeline_had_format_migration(p: ?*const Pipeline) bool { + return if (p) |pl| pl.format_migration else false; +} + +pub export fn cbm_pipeline_get_zova_publish_stats( + p_c: ?*const Pipeline, + out_stats: ?*c.cbm_pipeline_zova_publish_stats_t, +) bool { + const p = p_c orelse return false; + const out = out_stats orelse return false; + if (!p.zova_publish_stats.completed) return false; + out.* = p.zova_publish_stats; + return true; +} + +pub export fn cbm_pipeline_set_committed_counts(p: ?*Pipeline, nodes: c_int, edges: c_int) void { + if (p) |pl| { + pl.committed_nodes = nodes; + pl.committed_edges = edges; + } +} + +fn cancelled(p: *const Pipeline) bool { + return @atomicLoad(c_int, &p.cancelled, .seq_cst) != 0; +} + +// ── Hash table cleanup callback ───────────────────────────────────── + +fn freeSeenDirKey(key: CStr, val: ?*anyopaque, ud: ?*anyopaque) callconv(.c) void { + _ = val; + _ = ud; + c.free(@constCast(key)); +} + +// ── Pass 1: Structure ─────────────────────────────────────────────── + +/// True for languages whose module QN is the CONTAINING DIRECTORY (Java/Go +/// package). MUST match cbm_lang_module_is_dir() (internal/cbm/helpers.c). +fn stemModuleIsDir(lang: c.CBMLanguage) bool { + return lang == c.CBM_LANG_JAVA or lang == c.CBM_LANG_GO; +} + +/// Each file's Module QN, computed as extraction computes it, and the Folder +/// QN of every directory above it. +pub export fn cbm_stem_marks_init(m: *StemMarks, project: CStr, files: [*c]const FileInfo, file_count: c_int) void { + c.cbm_arena_init(&m.arena); + m.module_qns = c.cbm_ht_create(c.CBM_SZ_256); + m.dir_module_qns = c.cbm_ht_create(c.CBM_SZ_64); + m.folder_qns = c.cbm_ht_create(c.CBM_SZ_256); + for (0..ix(file_count)) |i| { + const rel: CStr = files[i].rel_path; + if (rel == null) continue; + const qn: MutStr = c.cbm_fqn_module_source_lang(&m.arena, project, rel, files[i].language); + if (qn != null) { + const table = if (stemModuleIsDir(files[i].language)) m.dir_module_qns else m.module_qns; + _ = c.cbm_ht_set(table, qn, qn); + } + var s: CStr = c.strchr(rel, '/'); + while (s != null) : (s = c.strchr(s + 1, '/')) { + const dir: MutStr = c.cbm_arena_strndup(&m.arena, rel, ptrDiff(s, rel)); + const fqn: MutStr = if (dir != null) c.cbm_fqn_folder(&m.arena, project, dir) else null; + if (fqn != null) _ = c.cbm_ht_set(m.folder_qns, fqn, fqn); + } + } +} + +pub export fn cbm_stem_marks_free(m: *StemMarks) void { + c.cbm_ht_free(m.module_qns); + c.cbm_ht_free(m.dir_module_qns); + c.cbm_ht_free(m.folder_qns); + c.cbm_arena_destroy(&m.arena); +} + +pub export fn cbm_stem_marks_side(m_c: ?*const StemMarks, qn: CStr) c_int { + const m = m_c orelse return c.CBM_STEM_NONE; + if (qn == null or c.cbm_ht_get(m.module_qns, qn) == null) return c.CBM_STEM_NONE; + if (c.cbm_ht_get(m.dir_module_qns, qn) != null) return c.CBM_STEM_MODULE_YIELDS; + return if (c.cbm_ht_get(m.folder_qns, qn) != null) c.CBM_STEM_FOLDER_YIELDS else c.CBM_STEM_NONE; +} + +/// The extension names the yielding Module's file ("sh" for genproto.sh); +/// "file" when there is none. +fn stemMarkSuffix(rel_path: CStr) CStr { + var base: CStr = c.strrchr(rel_path, '/'); + base = if (base != null) base + 1 else rel_path; + const dot: CStr = c.strrchr(base, '.'); + return if (dot != null and dot != base and dot[1] != 0) dot + 1 else "file"; +} + +pub export fn cbm_stem_marks_module_qn( + m_c: ?*const StemMarks, + qn: CStr, + rel_path: CStr, + lang: c.CBMLanguage, + buf: MutStr, + buf_size: usize, +) CStr { + const m = m_c orelse return qn; + if (qn == null or rel_path == null or stemModuleIsDir(lang) or + c.cbm_ht_get(m.dir_module_qns, qn) == null) + { + return qn; + } + const n = c.snprintf(buf, buf_size, "%s#%s", qn, stemMarkSuffix(rel_path)); + return if (n > 0 and ix(n) < buf_size) buf else qn; +} + +pub export fn cbm_stem_marks_folder_qn( + m: ?*const StemMarks, + project: CStr, + rel_dir: CStr, + buf: MutStr, + buf_size: usize, +) CStr { + const qn: MutStr = c.cbm_pipeline_fqn_folder(project, rel_dir); + if (qn == null) return null; + const mark = if (m) |marks| + c.cbm_ht_get(marks.module_qns, qn) != null and c.cbm_ht_get(marks.dir_module_qns, qn) == null + else + false; + const n = c.snprintf(buf, buf_size, "%s%s", qn, @as(CStr, if (mark) "/" else "")); + c.free(qn); + return if (n >= 0 and ix(n) < buf_size) buf else null; +} + +/// Create Project, Folder/Package, and File nodes in the graph buffer. +/// Walk directory chain upward, creating Folder nodes and CONTAINS_FOLDER edges. +fn createFolderChain(p: *Pipeline, marks: ?*const StemMarks, dir: CStr, seen_dirs: ?*HashTable) void { + const walk: MutStr = c.strdup(dir); + while (walk[0] != 0 and c.cbm_ht_get(seen_dirs, walk) == null) { + _ = c.cbm_ht_set(seen_dirs, c.strdup(walk), @ptrFromInt(1)); + var folder_buf: [c.CBM_SZ_8K]u8 = undefined; + const folder_qn = cbm_stem_marks_folder_qn(marks, p.project_name, walk, &folder_buf, folder_buf.len); + var dir_base: CStr = c.strrchr(walk, '/'); + dir_base = if (dir_base != null) dir_base + 1 else walk; + _ = c.cbm_gbuf_upsert_node(p.gbuf, "Folder", dir_base, folder_qn, walk, 0, 0, "{}"); + + var pdir: MutStr = c.strdup(walk); + const ps: MutStr = c.strrchr(pdir, '/'); + if (ps != null) { + ps[0] = 0; + } else { + c.free(pdir); + pdir = c.strdup(""); + } + var pqn_buf: [c.CBM_SZ_8K]u8 = undefined; + const pqn: CStr = if (pdir[0] == 0) + (if (p.branch_qn != null) p.branch_qn else p.project_name) + else + cbm_stem_marks_folder_qn(marks, p.project_name, pdir, &pqn_buf, pqn_buf.len); + const fnode: ?*const Node = c.cbm_gbuf_find_by_qn(p.gbuf, folder_qn); + const pnode: ?*const Node = c.cbm_gbuf_find_by_qn(p.gbuf, pqn); + if (fnode != null and pnode != null) { + _ = c.cbm_gbuf_insert_edge(p.gbuf, pnode.?.id, fnode.?.id, "CONTAINS_FOLDER", "{}"); + } + const up: MutStr = c.strrchr(walk, '/'); + if (up != null) up[0] = 0 else walk[0] = 0; + c.free(pdir); + } + c.free(walk); +} + +fn passStructure(p: *Pipeline, marks: ?*const StemMarks, files: [*c]const FileInfo, file_count: c_int) c_int { + interop.logInfo("pass.start", .{ "pass", "structure", "files", itoa(file_count) }); + + // Project node + // coverage_version marks a generation whose File nodes carry coverage + // stamps; incremental runs keep the node, so older generations stay + // unmarked until the next full index. index_format is checked by + // try_incremental before an existing generation is reused. + _ = c.cbm_gbuf_upsert_node( + p.gbuf, + "Project", + p.project_name, + p.project_name, + null, + 0, + 0, + "{\"coverage_version\":" ++ c.CBM_COVERAGE_VERSION_STR ++ + ",\"index_format\":" ++ c.CBM_INDEX_FORMAT_VERSION_STR ++ "}", + ); + const branch_qn: CStr = if (p.branch_qn != null) p.branch_qn else p.project_name; + const branch_name: CStr = if (p.git_ctx.branch != null) p.git_ctx.branch else "working-tree"; + var branch_props: [c.CBM_SZ_2K]u8 = undefined; + var branch_props_json: CStr = "{}"; + if (c.cbm_git_context_props_json(&p.git_ctx, &branch_props, branch_props.len) > 0) { + branch_props_json = &branch_props; + } + if (p.branch_qn != null) { + const branch_id = c.cbm_gbuf_upsert_node(p.gbuf, "Branch", branch_name, branch_qn, null, 0, 0, branch_props_json); + const project_node: ?*const Node = c.cbm_gbuf_find_by_qn(p.gbuf, p.project_name); + if (project_node != null and branch_id > 0) { + _ = c.cbm_gbuf_insert_edge(p.gbuf, project_node.?.id, branch_id, "HAS_BRANCH", branch_props_json); + } + } + + // Collect unique directories and create Folder/Package nodes + const seen_dirs = c.cbm_ht_create(c.CBM_SZ_256); + + for (0..ix(file_count)) |i| { + const rel: CStr = files[i].rel_path; + if (rel == null) continue; + + // Create File node + const file_qn: MutStr = c.cbm_pipeline_fqn_compute(p.project_name, rel, "__file__"); + // Extract basename + const slash: CStr = c.strrchr(rel, '/'); + const basename: CStr = if (slash != null) slash + 1 else rel; + + var props: [c.CBM_SZ_256]u8 = undefined; + const ext: CStr = c.strrchr(basename, '.'); + _ = c.snprintf(&props, props.len, "{\"extension\":\"%s\"}", orEmpty(ext)); + + _ = c.cbm_gbuf_upsert_node(p.gbuf, "File", basename, file_qn, rel, 0, 0, &props); + + // CONTAINS_FILE edge: parent dir -> file + var dir: MutStr = c.strdup(rel); + const last_slash: MutStr = c.strrchr(dir, '/'); + if (last_slash != null) { + last_slash[0] = 0; + } else { + c.free(dir); + dir = c.strdup(""); + } + + var parent_buf: [c.CBM_SZ_8K]u8 = undefined; + const parent_qn: CStr = if (dir[0] == 0) + branch_qn + else + cbm_stem_marks_folder_qn(marks, p.project_name, dir, &parent_buf, parent_buf.len); + + // Walk up directory chain, creating Folder nodes + createFolderChain(p, marks, dir, seen_dirs); + + // Now create the CONTAINS_FILE edge + const fnode: ?*const Node = c.cbm_gbuf_find_by_qn(p.gbuf, file_qn); + const pnode: ?*const Node = c.cbm_gbuf_find_by_qn(p.gbuf, parent_qn); + if (fnode != null and pnode != null) { + _ = c.cbm_gbuf_insert_edge(p.gbuf, pnode.?.id, fnode.?.id, "CONTAINS_FILE", "{}"); + } + + c.free(file_qn); + c.free(dir); + } + + // Free seen_dirs keys + c.cbm_ht_foreach(seen_dirs, freeSeenDirKey, null); + c.cbm_ht_free(seen_dirs); + + interop.logInfo("pass.done", .{ + "pass", "structure", + "nodes", itoa(c.cbm_gbuf_node_count(p.gbuf)), + "edges", itoa(c.cbm_gbuf_edge_count(p.gbuf)), + }); + return 0; +} + +// ── Githistory compute thread (for fused post-pass parallelism) ───── + +const GhComputeArg = extern struct { + repo_path: CStr, + result: *c.cbm_githistory_result_t, +}; + +fn ghComputeThreadFn(arg: ?*anyopaque) callconv(.c) ?*anyopaque { + const a: *GhComputeArg = @ptrCast(@alignCast(arg)); + _ = c.cbm_pipeline_githistory_compute(a.repo_path, a.result); + return null; +} + +/// "async" when an infra binding names no broker. +fn brokerOrAsync(ib: *const c.CBMInfraBinding) CStr { + return if (ib.broker != null) ib.broker else "async"; +} + +// Extract Route nodes from URL strings found in config files (YAML, HCL, TOML). +// These are infrastructure-defined endpoints (Cloud Scheduler, Terraform). +// Process infra bindings: topic→URL pairs from IaC configs. +// Creates Route nodes for endpoints and HANDLES edges linking +// topic Routes to endpoint Routes (bridging the gap). +/// Process one infra binding: create Route node + INFRA_MAPS edge. +fn processOneInfraBinding(gbuf: ?*Gbuf, ib: *const c.CBMInfraBinding, rel_path: CStr) c_int { + var url_route_qn: [c.CBM_ROUTE_QN_SIZE]u8 = undefined; + _ = c.snprintf(&url_route_qn, url_route_qn.len, "__route__infra__%s", ib.target_url); + const url_route_id = c.cbm_gbuf_upsert_node(gbuf, "Route", ib.target_url, &url_route_qn, rel_path, 0, 0, "{\"source\":\"infra\"}"); + var topic_route_qn: [c.CBM_ROUTE_QN_SIZE]u8 = undefined; + _ = c.snprintf(&topic_route_qn, topic_route_qn.len, "__route__%s__%s", brokerOrAsync(ib), ib.source_name); + const topic_route: ?*const Node = c.cbm_gbuf_find_by_qn(gbuf, &topic_route_qn); + var topic_route_id: i64 = undefined; + if (topic_route) |route| { + topic_route_id = route.id; + } else { + // The config file IS the declaration that the topic/queue/schedule exists; + // upsert its Route node so the binding maps even when no code-side dispatch + // call created the node first (e.g. a standalone scheduler/subscription + // manifest). + topic_route_id = c.cbm_gbuf_upsert_node(gbuf, "Route", ib.source_name, &topic_route_qn, rel_path, 0, 0, brokerOrAsync(ib)); + if (topic_route_id <= 0) return 0; + } + var props: [c.CBM_SZ_512]u8 = undefined; + _ = c.snprintf( + &props, + props.len, + "{\"broker\":\"%s\",\"topic\":\"%s\",\"endpoint\":\"%s\"}", + brokerOrAsync(ib), + ib.source_name, + ib.target_url, + ); + _ = c.cbm_gbuf_insert_edge(gbuf, topic_route_id, url_route_id, "INFRA_MAPS", &props); + return 1; +} + +pub export fn cbm_pipeline_process_infra_bindings( + gbuf: ?*Gbuf, + files: [*c]const FileInfo, + result_cache: [*c][*c]FileResult, + spill: ?*Spill, + file_count: c_int, +) void { + var bindings: c_int = 0; + for (0..ix(file_count)) |i| { + const fi: c_int = @intCast(i); + if (result_cache[i] == null and c.cbm_result_spill_infra_binding_count(spill, fi) == 0) { + continue; + } + const result: *FileResult = c.cbm_result_cache_acquire(result_cache, spill, fi) orelse continue; + for (0..ix(result.infra_bindings.count)) |bi| { + const ib = util.at(result.infra_bindings.items, bi); + if (ib.source_name != null and ib.target_url != null) { + bindings += processOneInfraBinding(gbuf, ib, files[i].rel_path); + } + } + c.cbm_result_cache_release(result_cache, fi, result); + } + if (bindings > 0) { + var buf: [c.CBM_SZ_16]u8 = undefined; + _ = c.snprintf(&buf, buf.len, "%d", bindings); + interop.logInfo("pass.infra_bindings", .{ "linked", @as(CStr, &buf) }); + } +} + +pub export fn cbm_pipeline_is_infra_file(fp: CStr) bool { + return fp != null and + (c.strstr(fp, ".yaml") != null or c.strstr(fp, ".yml") != null or + c.strstr(fp, ".tf") != null or c.strstr(fp, ".hcl") != null or + c.strstr(fp, ".toml") != null); +} + +/// True when a YAML key path denotes an UPSTREAM dependency, CONFIG value, or +/// HEALTHCHECK target rather than an endpoint this service exposes. Such URLs +/// (auth JWKS, downstream service base URLs, package-registry URLs, healthcheck +/// curl targets) are NOT routes the service serves and must not mint Route nodes +/// (#521). Exposed-endpoint keys (push_endpoint, post_url, callback, webhook) +/// are intentionally absent here so they still produce infra Route nodes. +fn isUpstreamConfigKey(key_path: CStr) bool { + // No key context (e.g. flat string) — keep prior behaviour and mint. + if (key_path == null) return false; + const deny = [_][*:0]const u8{ + "jwks", "registry", "registries", "healthcheck", "upstream", "_service_url", "auth", + }; + for (deny) |d| { + if (c.strstr(key_path, d) != null) return true; + } + return false; +} + +/// Try to create an infra Route node from one string_ref. +fn tryUpsertInfraRoute(gbuf: ?*Gbuf, sr: *const c.CBMStringRef, fp: CStr) void { + if (sr.kind != c.CBM_STRREF_URL or sr.value == null or c.strstr(sr.value, "://") == null) return; + // Skip upstream/config/healthcheck URLs — they are not exposed routes (#521). + if (isUpstreamConfigKey(sr.key_path)) return; + var route_qn: [c.CBM_ROUTE_QN_SIZE]u8 = undefined; + _ = c.snprintf(&route_qn, route_qn.len, "__route__infra__%s", sr.value); + var route_props: [c.CBM_SZ_512]u8 = undefined; + if (sr.key_path != null) { + _ = c.snprintf(&route_props, route_props.len, "{\"source\":\"infra\",\"key_path\":\"%s\"}", sr.key_path); + } else { + _ = c.snprintf(&route_props, route_props.len, "{\"source\":\"infra\"}"); + } + _ = c.cbm_gbuf_upsert_node(gbuf, "Route", sr.value, &route_qn, fp, 0, 0, &route_props); +} + +// A URL string_ref that does NOT denote a route the service serves: a value +// containing whitespace is a command/sentence with an embedded URL (e.g. a +// Docker healthcheck `curl --fail http://... || exit 1`); a NULL key_path is a +// context-less/duplicate ref; an upstream/config/healthcheck key is an external +// dependency, not an exposed route. (#521) +/// True when one dotted key segment names an endpoint the service is called at: +/// Pub/Sub push endpoints, Cloud Scheduler/Tasks `uri`, webhooks, callbacks, +/// post URLs. Case, '_' and '-' are ignored (pushEndpoint == push_endpoint). +fn isEndpointKeySegment(segment: CStr, len: usize) bool { + var norm: [c.CBM_SZ_64]u8 = undefined; + var n: usize = 0; + var i: usize = 0; + while (i < len and n < norm.len - 1) : (i += 1) { + const ch = segment[i]; + if (ch == '_' or ch == '-') continue; + norm[n] = if (ch >= 'A' and ch <= 'Z') ch - 'A' + 'a' else ch; + n += 1; + } + norm[n] = 0; + const contains = [_][*:0]const u8{ "endpoint", "webhook", "callback", "posturl" }; + for (contains) |needle| { + if (c.strstr(&norm, needle) != null) return true; + } + return streq(&norm, "uri"); +} + +/// Allow-list, not deny-list: generic keys such as `url`, `link` or `avatarUrl` +/// hold documentation and data URLs (sponsor lists, profiles, images), so only +/// endpoint-named keys mint infra Routes. Upstream/config keys stay denied. (#521) +pub export fn cbm_pipeline_infra_route_key_allowed(key_path: CStr) bool { + if (key_path == null or isUpstreamConfigKey(key_path)) return false; + var segment = key_path; + while (true) { + const dot: CStr = c.strchr(segment, '.'); + const len = if (dot != null) ptrDiff(dot, segment) else c.strlen(segment); + if (isEndpointKeySegment(segment, len)) return true; + if (dot == null) return false; + segment = dot + 1; + } +} + +fn routeSrDenied(sr: *const c.CBMStringRef) bool { + if (sr.value == null or c.strchr(sr.value, ' ') != null) return true; + return !cbm_pipeline_infra_route_key_allowed(sr.key_path); +} + +pub export fn cbm_pipeline_extract_infra_routes( + gbuf: ?*Gbuf, + files: [*c]const FileInfo, + result_cache: [*c][*c]FileResult, + spill: ?*Spill, + file_count: c_int, +) void { + // DENY-WINS-BY-VALUE: the same URL is often extracted as several string_refs + // at different key_path granularities (full path, leaf key, flat). The Route + // node is keyed by VALUE, so it would be minted if ANY granularity passed the + // per-ref guard — e.g. a denied full path `registries.terraform-registry.url` + // is defeated by a sibling leaf `url`. So pass 1 collects every URL value + // denied under ANY of its refs; pass 2 mints only values never denied. (#521) + const denied = c.cbm_ht_create(16); + // `denied` borrows its keys; values from a loaded (spilled) result are + // copied here because the result is freed before pass 2 reads them. + var denied_keys: c.CBMArena = undefined; + c.cbm_arena_init(&denied_keys); + for (0..2) |pass| { + for (0..ix(file_count)) |i| { + const fi: c_int = @intCast(i); + if (!cbm_pipeline_is_infra_file(files[i].rel_path) or + (result_cache[i] == null and c.cbm_result_spill_string_ref_count(spill, fi) == 0)) + { + continue; + } + const result: *FileResult = c.cbm_result_cache_acquire(result_cache, spill, fi) orelse continue; + const loaded = result != @as(?*FileResult, result_cache[i]); + for (0..ix(result.string_refs.count)) |si| { + const sr = util.at(result.string_refs.items, si); + if (sr.kind != c.CBM_STRREF_URL or sr.value == null or + c.strstr(sr.value, "://") == null) + { + continue; + } + if (pass == 0) { + if (denied != null and routeSrDenied(sr) and !c.cbm_ht_has(denied, sr.value)) { + const key: CStr = if (loaded) + c.cbm_arena_strdup(&denied_keys, sr.value) + else + sr.value; + if (key != null) _ = c.cbm_ht_set(denied, key, @ptrFromInt(1)); + } + } else if (denied == null or !c.cbm_ht_has(denied, sr.value)) { + tryUpsertInfraRoute(gbuf, sr, files[i].rel_path); + } + } + c.cbm_result_cache_release(result_cache, fi, result); + } + } + c.cbm_ht_free(denied); + c.cbm_arena_destroy(&denied_keys); +} + +// Run decorator_tags, configlink, and route matching passes. +fn predumpDeco(ctx: *Ctx) void { + _ = c.cbm_pipeline_pass_decorator_tags(ctx.gbuf, ctx.project_name); +} +fn predumpRoute(ctx: *Ctx) void { + _ = c.cbm_pipeline_create_route_nodes(ctx.gbuf); +} +fn predumpSim(ctx: *Ctx) void { + _ = c.cbm_pipeline_pass_similarity(ctx); +} +fn predumpSem(ctx: *Ctx) void { + _ = c.cbm_pipeline_pass_semantic_edges(ctx); +} +fn predumpCfg(ctx: *Ctx) void { + _ = c.cbm_pipeline_pass_configlink(ctx); +} +fn predumpComplexity(ctx: *Ctx) void { + _ = c.cbm_pipeline_pass_complexity(ctx); +} +fn predumpImportance(ctx: *Ctx) void { + _ = c.cbm_pipeline_pass_importance(ctx); +} + +const PredumpPass = struct { + run: *const fn (*Ctx) void, + name: [*:0]const u8, + /// true = skip in fast mode + moderate_only: bool, +}; + +const predump_passes = [_]PredumpPass{ + .{ .run = predumpDeco, .name = "decorator_tags", .moderate_only = false }, + .{ .run = predumpCfg, .name = "configlink", .moderate_only = false }, + .{ .run = predumpRoute, .name = "route_match", .moderate_only = false }, + .{ .run = predumpSim, .name = "similarity", .moderate_only = true }, + .{ .run = predumpSem, .name = "semantic_edges", .moderate_only = true }, + .{ .run = predumpComplexity, .name = "complexity", .moderate_only = false }, + // Importance runs LAST: it reads CALLS/USAGE (extraction) and TESTS + // (pass_tests, run before this loop). + .{ .run = predumpImportance, .name = "importance", .moderate_only = false }, +}; + +fn runPredumpPasses(p: *Pipeline, ctx: *Ctx) c_int { + var t: Timespec = undefined; + for (predump_passes) |pass| { + if (cancelled(p)) break; + // "moderate_only" passes (similarity/semantic edges) run in FULL, + // MODERATE and ADVANCED — they are skipped only in FAST. Compare + // explicitly against FAST rather than `> MODERATE` so ADVANCED + // (numerically 3) is not mistaken for a lighter mode than FULL. + if (pass.moderate_only and p.mode == c.CBM_MODE_FAST) continue; + // #49: each derived pass allocates whole-graph scratch. Check before + // it starts so an over-budget run stops instead of stacking another + // pass's working set on top. + var phase: [c.CBM_SZ_64]u8 = undefined; + _ = c.snprintf(&phase, phase.len, "before_%s", pass.name); + const gate_rc = cbm_pipeline_budget_gate(&phase); + if (gate_rc != 0) return gate_rc; + now(&t); + pass.run(ctx); + interop.logInfo("pass.timing", .{ "pass", pass.name, "elapsed_ms", elapsedText(t) }); + cbm_pipeline_log_phase_mem(pass.name); + } + return 0; +} + +fn freeResultCache(cache: [*c][*c]FileResult, file_count: c_int) void { + for (0..ix(file_count)) |i| { + if (cache[i] != null) c.cbm_free_result(cache[i]); + } + c.free(@ptrCast(cache)); +} + +/// Run the parallel pipeline path: extract, registry, resolve, infra, k8s. +fn runParallelPasses( + p: *Pipeline, + ctx: *Ctx, + files: [*c]const FileInfo, + file_count: c_int, + worker_count: c_int, + t: *Timespec, +) c_int { + interop.logInfo("pipeline.mode", .{ + "mode", "parallel", + "workers", itoa(worker_count), + "files", itoa(file_count), + }); + var shared_ids: i64 = c.cbm_gbuf_next_id(p.gbuf); + const cache: [*c][*c]FileResult = + @ptrCast(@alignCast(c.calloc(@bitCast(@as(isize, file_count)), @sizeOf([*c]FileResult)))); + if (cache == null) { + logError("pipeline.err", .{ "phase", "cache_alloc" }); + return c.CBM_NOT_FOUND; + } + now(t); + var rc = c.cbm_parallel_extract(ctx, files, file_count, cache, &shared_ids, worker_count); + interop.logInfo("pass.timing", .{ "pass", "parallel_extract", "elapsed_ms", elapsedText(t.*) }); + if (rc != 0 or cancelled(p)) { + c.free(@ptrCast(cache)); + return if (rc != 0) rc else c.CBM_NOT_FOUND; + } + cbm_pipeline_sync_next_id(p.gbuf, &shared_ids); + // extract -> registry handoff: return the extract phase's freed-but-retained + // allocator pages to the OS before registry_build allocates. On a 2x Linux + // index the extract peak holds ~13 GB of reclaimable pages (peak_mb 20.7 vs + // live rss_mb 7); not returning them pushed the process over the system + // memory-pressure threshold and got it SIGKILLed at registry entry. + c.cbm_mem_collect(); + interop.logInfo("mem.collect", .{ "phase", "post_extract", "rss_mb", mbText(c.cbm_mem_rss()) }); + now(t); + rc = c.cbm_build_registry_from_cache(ctx, files, file_count, cache); + interop.logInfo("pass.timing", .{ "pass", "registry_build", "elapsed_ms", elapsedText(t.*) }); + cbm_pipeline_log_phase_mem("registry_build"); + // #49: registry_build is the first post-extract whole-graph allocation. + if (rc == 0 and !cancelled(p)) rc = cbm_pipeline_budget_gate("before_lsp_cross_prepare"); + if (rc != 0 or cancelled(p)) { + freeResultCache(cache, file_count); + return if (rc != 0) rc else c.CBM_NOT_FOUND; + } + // Resolve workers must not reuse ids of nodes the registry build made. + cbm_pipeline_share_next_id(p.gbuf, &shared_ids); + // Cross-file LSP precondition: build a project-wide CBMLSPDef[] + // once. The fused resolve_worker invokes cbm_pxc_run_one(_ts) per + // file using these defs + the file's IMPORTS map, so cross-file + // type-resolved CALLS land in result->resolved_calls before the + // CALLS-edge emission. This replaces the old sequential + // cbm_pipeline_pass_lsp_cross pass which re-read every source from + // disk and re-parsed every tree on a single thread (~520s on + // kubernetes). Soft-failure: NULL all_defs / NULL def_modules just + // mean cross-file LSP no-ops; per-file LSP already ran during + // extract. + now(t); + // Cross-file LSP (type-aware call/usage resolution across files) — the + // most expensive phase. CBM_DISABLE_LSP_CROSS=1 opts out (it can SIGSEGV + // on large TS projects — see #340/#344); with cross-LSP off, all_defs + // stays NULL and the fused resolver simply no-ops cross-file resolution + // (per-file LSP already ran during extract). + var cbm_lsp_cross_env: [c.CBM_SZ_16]u8 = undefined; + const run_cross_lsp = c.cbm_safe_getenv( + "CBM_DISABLE_LSP_CROSS", + &cbm_lsp_cross_env, + cbm_lsp_cross_env.len, + null, + ) == null; + if (!run_cross_lsp) { + interop.logInfo("lsp_cross.skipped", .{ "reason", "CBM_DISABLE_LSP_CROSS env set" }); + } + // Owns the per-language cross registries below and the def strings of + // spilled results, so it outlives all_defs. + var cross_lsp_arena: c.CBMArena = undefined; + c.cbm_arena_init(&cross_lsp_arena); + var def_modules: [*c]MutStr = null; + var def_count: c_int = 0; + var all_defs: [*c]c.CBMLSPDef = null; + if (run_cross_lsp) { + def_modules = @ptrCast(@alignCast(c.calloc(@intCast(file_count), @sizeOf(MutStr)))); + all_defs = if (def_modules != null) + c.cbm_pxc_collect_all_defs(ctx, cache, files, file_count, ctx.project_name, def_modules, &cross_lsp_arena, &def_count) + else + null; + } + // Build inverted index: module_qn → defs. The fused resolve_worker + // uses this to filter the global all_defs[] down to just the defs + // each file actually needs (own_module + imported modules) — the + // gopls "package summary" pattern. Drops per-file registry build + // cost from O(all_defs) to O(relevant_defs), typically 50-100× + // smaller per file. + const module_def_index = + if (all_defs != null) c.cbm_pxc_build_module_def_index(all_defs, def_count) else null; + // Tier 2 full: pre-build per-language cross-LSP registries. + // Built ONCE here; shared READ-ONLY across all files of that language + // during resolve. Per-file work is then: parse + AST walk + O(1) lookups + // — no registry build, no Phase 1b mutations. Languages added so far: + // Go, Python. Others (C/C++, TS/JS, PHP, C#) fall back to per-file. + var cross_registries = std.mem.zeroes(c.CBMCrossLspRegistries); + if (all_defs != null) { + cross_registries.go = c.cbm_go_build_cross_registry(&cross_lsp_arena, all_defs, def_count); + cross_registries.python = c.cbm_py_build_cross_registry(&cross_lsp_arena, all_defs, def_count); + cross_registries.c = c.cbm_c_build_cross_registry(&cross_lsp_arena, all_defs, def_count); + cross_registries.cs = c.cbm_cs_build_cross_registry(&cross_lsp_arena, all_defs, def_count); + cross_registries.ts = c.cbm_ts_build_cross_registry(&cross_lsp_arena, all_defs, def_count); + cross_registries.zig_args = c.cbm_pxc_collect_zig_args( + ctx, + cache, + files, + file_count, + def_modules, + module_def_index, + all_defs, + def_count, + &cross_lsp_arena, + &cross_registries.zig_arg_count, + ); + // Rust: NOT built here. The shared all_defs registry is built LAZILY on the + // first NULL-filter rust file (the amplifier files) inside cbm_parallel_resolve + // — repos whose rust files all filter to subsets never pay the build/RSS. + } + interop.logInfo("pass.timing", .{ "pass", "lsp_cross_prepare", "elapsed_ms", elapsedText(t.*) }); + cbm_pipeline_log_phase_mem("lsp_cross_prepare"); + // #49: the cross-file registries are a whole-graph working set allocated + // after extract. Stop before resolve if they pushed RSS over the budget. + if (cbm_pipeline_budget_gate("before_parallel_resolve") != 0) { + rc = c.CBM_PIPELINE_ABORT_OVER_BUDGET; + } + if (rc == 0) { + now(t); + rc = c.cbm_parallel_resolve( + ctx, + files, + file_count, + cache, + &shared_ids, + worker_count, + all_defs, + def_count, + def_modules, + module_def_index, + &cross_registries, + ); + interop.logInfo("pass.timing", .{ "pass", "parallel_resolve", "elapsed_ms", elapsedText(t.*) }); + cbm_pipeline_log_phase_mem("parallel_resolve"); + } + c.cbm_pxc_free_module_def_index(module_def_index); + c.free(@ptrCast(@constCast(cross_registries.zig_args))); + c.cbm_arena_destroy(&cross_lsp_arena); // releases all per-lang registries + c.free(all_defs); + if (def_modules != null) { + for (0..ix(file_count)) |i| c.free(def_modules[i]); + c.free(@ptrCast(def_modules)); + } + cbm_pipeline_sync_next_id(p.gbuf, &shared_ids); + // A budget abort means stop allocating: the infra passes would run over + // the same cache only to return the abort below. Other resolve errors keep + // main's behaviour and still run them. + if (rc != c.CBM_PIPELINE_ABORT_OVER_BUDGET) { + cbm_pipeline_extract_infra_routes(p.gbuf, files, cache, ctx.spill, file_count); + cbm_pipeline_process_infra_bindings(p.gbuf, files, cache, ctx.spill, file_count); + } + freeResultCache(cache, file_count); + if (rc != 0) return rc; + now(t); + _ = c.cbm_pipeline_pass_k8s(ctx, files, file_count); + interop.logInfo("pass.timing", .{ "pass", "k8s", "elapsed_ms", elapsedText(t.*) }); + return if (cancelled(p)) c.CBM_NOT_FOUND else 0; +} + +/// run_parallel_passes with a spill store (F1) the extract can park results +/// in when it runs over its memory budget. The store's file lives under the +/// cache dir and is deleted when the run ends. +fn runParallelPipeline( + p: *Pipeline, + ctx: *Ctx, + files: [*c]const FileInfo, + file_count: c_int, + worker_count: c_int, + t: *Timespec, +) c_int { + const cache_dir: CStr = c.cbm_resolve_cache_dir(); + var spill_dir: [c.CBM_SZ_1K]u8 = undefined; + if (cache_dir != null and + c.snprintf(&spill_dir, spill_dir.len, "%s/spill", cache_dir) < @as(c_int, spill_dir.len)) + { + ctx.spill = c.cbm_result_spill_open(&spill_dir, file_count); + } + var rc = runParallelPasses(p, ctx, files, file_count, worker_count, t); + if (rc == 0 and c.cbm_result_spill_load_failures(ctx.spill) > 0) { + // A pass ran without some parked results: never publish that graph. + logError("mem.spill.failed", .{ + "load_failures", itoa(@truncate(c.cbm_result_spill_load_failures(ctx.spill))), + }); + rc = c.CBM_NOT_FOUND; + } + if (ctx.spill != null) { + var parked: i64 = 0; + var bytes: i64 = 0; + var loads: i64 = 0; + c.cbm_result_spill_stats(ctx.spill, &parked, &bytes, &loads); + if (parked > 0) { + interop.logInfo("mem.spill.done", .{ + "parked", itoa(@truncate(parked)), + "mb", itoa(@truncate(@divTrunc(bytes, BYTES_PER_MB))), + "loads", itoa(@truncate(loads)), + }); + } + c.cbm_result_spill_close(ctx.spill); + ctx.spill = null; + } + return rc; +} + +const PIPELINE_INCREMENTAL_FATAL = -2; + +/// True when the stored generation's Project node carries the current +/// index_format. Unstamped generations predate the stamp. +fn storedIndexFormatCurrent(stored: ?*const Gbuf, project: CStr) bool { + const key = "\"index_format\":" ++ c.CBM_INDEX_FORMAT_VERSION_STR; + const node: ?*const Node = c.cbm_gbuf_find_by_qn(stored, project); + const n = node orelse return false; + if (n.properties_json == null) return false; + const hit: CStr = c.strstr(n.properties_json, key); + return hit != null and !std.ascii.isDigit(hit[key.len]); +} + +/// Try the incremental pipeline against the project's committed cbm.zova +/// workspace. Returns >= 0 if incremental was used (the return code), or -1 to +/// proceed with full. CBM_PIPELINE_ABORT_OVER_BUDGET passes through unchanged. +fn tryIncremental(p: *Pipeline, files: [*c]FileInfo, file_count: c_int) c_int { + var zova_path: [c.CBM_SZ_1K]u8 = undefined; + if (c.cbm_zova_user_database_path(&zova_path, zova_path.len) != 0) return c.CBM_NOT_FOUND; + const repo = c.cbm_zova_repository_open(&zova_path, p.project_name); + if (repo == null) return c.CBM_NOT_FOUND; + + var summary = std.mem.zeroes(c.cbm_project_summary_export_t); + const workspace_id = c.cbm_zova_repository_workspace_id(repo); + if (c.cbm_zova_repository_project_summary(repo, workspace_id, &summary) == c.CBM_STORE_OK) { + c.cbm_store_project_summary_export_free(&p.saved_summary); + p.saved_summary = summary; + p.saved_summary_valid = true; + } + var snapshot = std.mem.zeroes(c.cbm_zova_workspace_snapshot_t); + var preloaded = c.cbm_gbuf_new(p.project_name, p.repo_path); + if (preloaded != null and c.cbm_zova_repository_export_incremental_snapshot_to_graph( + &zova_path, + workspace_id, + &snapshot, + preloaded, + ) == 0) { + const hash_count: c_int = snapshot.file_hash_count; + if (hash_count > 0 and !storedIndexFormatCurrent(preloaded, p.project_name)) { + // Reusing it would mix old-format QNs with new ones. + interop.logInfo("pipeline.route", .{ + "path", "format_change_reindex", + "required_format", c.CBM_INDEX_FORMAT_VERSION_STR, + "authority", "zova", + }); + p.format_migration = true; + } else if (hash_count > 0 and file_count <= hash_count + @divTrunc(hash_count, 2)) { + interop.logInfo("pipeline.route", .{ + "path", "incremental", + "stored_hashes", itoa(hash_count), + "authority", "zova", + }); + // The exported snapshot owns all of its data. Release the + // read transaction before publishing the next generation, + // otherwise it pins the old WAL and prevents checkpointing. + c.cbm_zova_repository_close(repo); + p.last_route = c.CBM_PIPELINE_ROUTE_INCREMENTAL; + const rc = c.cbm_pipeline_run_incremental_zova(@ptrCast(p), &zova_path, files, file_count, &snapshot, preloaded); + preloaded = null; + c.cbm_zova_workspace_snapshot_free(&snapshot); + if (rc == c.CBM_PIPELINE_INCREMENTAL_DECLINED) { + interop.logInfo("pipeline.route", .{ + "path", "incremental_declined_reindex", + "authority", "zova", + }); + return c.CBM_NOT_FOUND; + } + if (rc == c.CBM_PIPELINE_ABORT_OVER_BUDGET) return rc; + return if (rc == 0) 0 else PIPELINE_INCREMENTAL_FATAL; + } + if (hash_count > 0 and !p.format_migration) { + interop.logInfo("pipeline.route", .{ + "path", "mode_change_reindex", + "stored_hashes", itoa(hash_count), + "discovered", itoa(file_count), + "authority", "zova", + }); + } + c.cbm_zova_workspace_snapshot_free(&snapshot); + } + c.cbm_gbuf_free(preloaded); + c.cbm_zova_repository_close(repo); + return c.CBM_NOT_FOUND; +} + +/// Publication result fields copied to the pipeline's publish stats under the +/// same name. +const publish_result_fields = .{ + "full_clear_count", "unchanged_rewrite_count", "nodes_inserted", + "nodes_updated", "nodes_deleted", "edges_inserted", + "edges_deleted", "node_vectors_upserted", "node_vectors_deleted", + "token_vectors_upserted", "token_vectors_deleted", "normalization_ms", + "model_nodes_ms", "model_edges_ms", "model_edge_endpoint_ms", + "model_edge_sort_ms", "model_edge_group_ms", "model_edge_payload_ms", + "model_edge_digest_ms", "model_edge_default_payloads", "model_edge_payload_scratch_edges", + "model_hashes_ms", "model_vectors_ms", "model_digests_ms", + "writer_guard_ms", "database_init_ms", "database_open_ms", + "transaction_begin_ms", "transaction_body_ms", "transaction_commit_ms", + "database_close_ms", "clear_ms", "finalize_ms", + "canonical_files_ms", "canonical_nodes_ms", "canonical_edges_ms", + "canonical_hashes_ms", "fts_ms", "token_metadata_ms", + "native_graph_ms", "native_graph_materialize_ms", "native_graph_reset_ms", + "native_graph_nodes_ms", "native_graph_edges_ms", "native_graph_validate_ms", + "native_graph_key_generation_ms", "native_graph_cleanup_ms", "native_vectors_ms", + "fresh_validation_ms", "fresh_index_ms", "fresh_commit_ms", + "fresh_build_ms", "readback_ms", +}; + +/// Snapshot metrics copied to the publish stats as `snapshot_`. +const snapshot_metric_fields = .{ + "base_ms", "optional_ms", "open_ms", "header_ms", + "integrity_ms", "nodes_sql_ms", "nodes_native_ms", "nodes_finalize_ms", + "edges_sql_ms", "edges_native_ms", "edges_finalize_ms", "hashes_summary_ms", + "close_ms", "graph_buffer_ms", "base_phase_mask", "node_rows", + "edge_rows", "file_hash_rows", "topology_rows", "node_vector_rows", + "token_vector_rows", +}; + +fn pipelinePublishZovaUserDatabase( + p_c: ?*Pipeline, + gbuf: ?*Gbuf, + files: [*c]const FileInfo, + file_count: c_int, + mode_skipped: [*c]const c.cbm_file_hash_t, + mode_skipped_count: c_int, + before: ?*c.cbm_zova_workspace_snapshot_t, +) c_int { + const p = p_c orelse return c.CBM_NOT_FOUND; + if (gbuf == null or file_count < 0 or mode_skipped_count < 0 or + (file_count > 0 and files == null) or (mode_skipped_count > 0 and mode_skipped == null)) + { + return c.CBM_NOT_FOUND; + } + p.zova_publish_ms = 0.0; + p.zova_publish_stats = std.mem.zeroes(c.cbm_pipeline_zova_publish_stats_t); + var route_started = std.mem.zeroes(Timespec); + now(&route_started); + const capacity = file_count + mode_skipped_count; + const hashes: [*c]c.cbm_zova_file_hash_input_t = + if (capacity > 0) interop.callocN(c.cbm_zova_file_hash_input_t, ix(capacity)) else null; + if (capacity > 0 and hashes == null) return c.CBM_NOT_FOUND; + var count: usize = 0; + for (0..ix(file_count)) |i| { + var fst: c.struct_stat = undefined; + if (c.stat(files[i].path, &fst) != 0) continue; + hashes[count] = std.mem.zeroes(c.cbm_zova_file_hash_input_t); + hashes[count].file_path = files[i].rel_path; + hashes[count].content_hash = ""; + hashes[count].mtime_ns = c.cbm_stat_mtime_ns(&fst); + hashes[count].size_bytes = fst.st_size; + count += 1; + } + for (0..ix(mode_skipped_count)) |i| { + hashes[count] = std.mem.zeroes(c.cbm_zova_file_hash_input_t); + hashes[count].file_path = mode_skipped[i].rel_path; + hashes[count].content_hash = orEmpty(mode_skipped[i].sha256); + hashes[count].mtime_ns = mode_skipped[i].mtime_ns; + hashes[count].size_bytes = mode_skipped[i].size; + count += 1; + } + var summary = std.mem.zeroes(c.cbm_zova_project_summary_input_t); + if (p.saved_summary_valid) { + summary.present = true; + summary.summary = p.saved_summary.summary; + summary.source_hash = p.saved_summary.source_hash; + summary.created_at = p.saved_summary.created_at; + summary.updated_at = p.saved_summary.updated_at; + } + var result = std.mem.zeroes(c.cbm_zova_workspace_generation_result_t); + const rc = if (before != null) + c.cbm_gbuf_publish_zova_user_database_delta(gbuf, before, hashes, @intCast(count), &summary, &result) + else + c.cbm_gbuf_publish_zova_user_database(gbuf, hashes, @intCast(count), &summary, &result); + c.free(hashes); + if (rc != 0) { + logError("zova.single_file_publish", .{ + "project", @as(CStr, p.project_name), + "reason", "publication_failed", + }); + return rc; + } + p.zova_publish_ms = result.publish_ms; + const is_delta = result.publication_mode == c.CBM_ZOVA_PUBLICATION_MODE_DELTA; + var stats = std.mem.zeroes(c.cbm_pipeline_zova_publish_stats_t); + stats.completed = true; + stats.delta = is_delta; + inline for (publish_result_fields) |name| @field(stats, name) = @field(result, name); + stats.route_ms = c.cbm_elapsed_ms(route_started); + stats.snapshot_completed = before != null; + if (before) |b| { + inline for (snapshot_metric_fields) |name| { + @field(stats, "snapshot_" ++ name) = @field(b.metrics, name); + } + stats.snapshot_hydrated_components = b.hydrated_components; + stats.snapshot_generation = b.generation; + } + p.zova_publish_stats = stats; + var generation_text: [32]u8 = undefined; + var publish_ms_text: [32]u8 = undefined; + var inserted_text: [32]u8 = undefined; + var updated_text: [32]u8 = undefined; + var deleted_text: [32]u8 = undefined; + _ = c.snprintf(&generation_text, generation_text.len, "%lld", @as(c_longlong, result.generation)); + _ = c.snprintf(&publish_ms_text, publish_ms_text.len, "%.3f", result.publish_ms); + _ = c.snprintf(&inserted_text, inserted_text.len, "%llu", @as(c_ulonglong, result.inserted_count)); + _ = c.snprintf(&updated_text, updated_text.len, "%llu", @as(c_ulonglong, result.updated_count)); + _ = c.snprintf(&deleted_text, deleted_text.len, "%llu", @as(c_ulonglong, result.deleted_count)); + interop.logInfo("zova.single_file_publish", .{ + "project", @as(CStr, p.project_name), + "generation", @as(CStr, &generation_text), + "publish_ms", @as(CStr, &publish_ms_text), + "publication_mode", @as(CStr, if (is_delta) "delta" else "full"), + "inserted", @as(CStr, &inserted_text), + "updated", @as(CStr, &updated_text), + "deleted", @as(CStr, &deleted_text), + }); + return 0; +} + +pub export fn cbm_pipeline_publish_zova_user_database( + p: ?*Pipeline, + gbuf: ?*Gbuf, + files: [*c]const FileInfo, + file_count: c_int, + mode_skipped: [*c]const c.cbm_file_hash_t, + mode_skipped_count: c_int, +) c_int { + return pipelinePublishZovaUserDatabase(p, gbuf, files, file_count, mode_skipped, mode_skipped_count, null); +} + +pub export fn cbm_pipeline_publish_zova_user_database_delta( + p: ?*Pipeline, + gbuf: ?*Gbuf, + files: [*c]const FileInfo, + file_count: c_int, + mode_skipped: [*c]const c.cbm_file_hash_t, + mode_skipped_count: c_int, + before: ?*c.cbm_zova_workspace_snapshot_t, +) c_int { + if (before == null) return c.CBM_NOT_FOUND; + return pipelinePublishZovaUserDatabase(p, gbuf, files, file_count, mode_skipped, mode_skipped_count, before); +} + +/// Publish the graph and file hashes to the shared cbm.zova. +fn dumpAndPersistHashes(p: *Pipeline, files: [*c]const FileInfo, file_count: c_int, t: *Timespec) c_int { + now(t); + // #49: dump + publish materialize the whole graph again (normalization, + // native topology, FTS, vectors). Check before starting; nothing has been + // published yet, so an abort leaves the previous generation serving. + cbm_pipeline_log_phase_mem("pre_dump"); + var gate_rc = cbm_pipeline_budget_gate("before_dump"); + if (gate_rc != 0) return gate_rc; + var zova_dir: [c.CBM_SZ_1K]u8 = undefined; + if (c.cbm_zova_user_database_path(&zova_dir, zova_dir.len) == 0) { + const last_slash: MutStr = c.strrchr(&zova_dir, '/'); + if (last_slash != null) { + last_slash[0] = 0; + _ = c.cbm_mkdir_p(&zova_dir, CBM_DIR_PERMS); + } + } + // Capture committed counts before preparing the dump, which releases the + // graph buffer indexes (graph_buffer.c), after which cbm_gbuf_node_count() + // returns 0 and the #334 plausibility gate would never fire. + cbm_pipeline_run_before_dump_hook(p.gbuf); + p.committed_nodes = c.cbm_gbuf_node_count(p.gbuf); + p.committed_edges = c.cbm_gbuf_edge_count(p.gbuf); + const prepare_rc = c.cbm_gbuf_prepare_zova_dump(p.gbuf); + if (prepare_rc != 0) { + logError("pipeline.err", .{ "phase", "zova_prepare" }); + return prepare_rc; + } + cbm_pipeline_log_phase_mem("zova_prepare"); + // Preparing the dump releases the gbuf indexes; if RSS is still over + // budget now, publishing would only raise it further. Abort before the + // atomic publication starts. + gate_rc = cbm_pipeline_budget_gate("before_publish"); + if (gate_rc != 0) return gate_rc; + const publish_rc = cbm_pipeline_publish_zova_user_database(p, p.gbuf, files, file_count, null, 0); + if (publish_rc != 0) { + logError("pipeline.err", .{ "phase", "zova_single_file" }); + return publish_rc; + } + cbm_pipeline_log_phase_mem("zova_publish"); + return 0; +} + +/// Run githistory pass. +fn runGithistory(p: *Pipeline, ctx: *Ctx) c_int { + var t_gh: Timespec = undefined; + now(&t_gh); + + var gh_result = std.mem.zeroes(c.cbm_githistory_result_t); + var gh_thread: c.cbm_thread_t = undefined; + var gh_threaded = false; + var gh_arg: GhComputeArg = .{ .repo_path = ctx.repo_path, .result = &gh_result }; + + if (p.mode != c.CBM_MODE_FAST) { + if (c.cbm_default_worker_count(true) > 1) { + if (c.cbm_thread_create(&gh_thread, 0, ghComputeThreadFn, &gh_arg) == 0) { + gh_threaded = true; + } + } + if (!gh_threaded) { + _ = c.cbm_pipeline_githistory_compute(ctx.repo_path, &gh_result); + interop.logInfo("pass.timing", .{ + "pass", "githistory_compute", + "elapsed_ms", elapsedText(t_gh), + }); + } + } else { + interop.logInfo("pass.skip", .{ "pass", "githistory", "reason", "fast_mode" }); + } + + if (gh_threaded) { + _ = c.cbm_thread_join(&gh_thread); + interop.logInfo("pass.timing", .{ + "pass", "githistory_compute", + "elapsed_ms", elapsedText(t_gh), + }); + } + + var gh_edges: c_int = 0; + if (gh_result.count > 0 or gh_result.file_temporal_count > 0) { + gh_edges = c.cbm_pipeline_githistory_apply(ctx, &gh_result); + } + interop.logInfo("pass.done", .{ + "pass", "githistory", + "commits", itoa(gh_result.commit_count), + "edges", itoa(gh_edges), + }); + c.free(gh_result.couplings); + c.free(gh_result.file_temporal); + return 0; +} + +// ── Pipeline run ──────────────────────────────────────────────────── + +/// Run tests + git history. Returns 0 on success. +fn runTestsAndHistory(p: *Pipeline, ctx: *Ctx, files: [*c]const FileInfo, file_count: c_int) c_int { + var t: Timespec = undefined; + now(&t); + const t_tests = util.profStart(); + var rc = c.cbm_pipeline_pass_tests(ctx, files, file_count); + util.profEnd("pipeline", "pass_tests", &t_tests, file_count); + interop.logInfo("pass.timing", .{ "pass", "tests", "elapsed_ms", elapsedText(t) }); + if (rc == 0 and !cancelled(p)) { + const t_gh = util.profStart(); + rc = runGithistory(p, ctx); + util.profEnd("pipeline", "pass_githistory", &t_gh, 0); + } + if (cancelled(p)) return c.CBM_NOT_FOUND; + return rc; +} + +/// Run tests, git history, predump passes, and dump+persist. +fn runPostExtraction(p: *Pipeline, ctx: *Ctx, files: [*c]const FileInfo, file_count: c_int) c_int { + // #49: extract already enforces the budget. Everything from here on works + // on the whole graph, so gate the entry too. + cbm_pipeline_log_phase_mem("extract_done"); + var rc = cbm_pipeline_budget_gate("before_tests_and_history"); + if (rc != 0) return rc; + rc = runTestsAndHistory(p, ctx, files, file_count); + if (rc != 0) return rc; + cbm_pipeline_log_phase_mem("tests_and_history"); + + const t_predump = util.profStart(); + rc = runPredumpPasses(p, ctx); + util.profEnd("pipeline", "3_predump_passes_total", &t_predump, 0); + if (rc != 0) return rc; + + if (!cancelled(p)) { + var t: Timespec = undefined; + const t_dump = util.profStart(); + rc = dumpAndPersistHashes(p, files, file_count, &t); + util.profEnd("pipeline", "4_dump_and_persist", &t_dump, 0); + } + return rc; +} + +/// Run structure + extraction passes. +fn runStructureAndExtraction(p: *Pipeline, ctx: *Ctx, files: [*c]const FileInfo, file_count: c_int) c_int { + var t: Timespec = undefined; + now(&t); + const t_struct = util.profStart(); + _ = passStructure(p, ctx.stem_marks, files, file_count); + util.profEnd("pipeline", "pass_structure", &t_struct, file_count); + interop.logInfo("pass.timing", .{ "pass", "structure", "elapsed_ms", elapsedText(t) }); + if (cancelled(p)) return c.CBM_NOT_FOUND; + + const worker_count = c.cbm_default_worker_count(true); + const t_extract_total = util.profStart(); + const rc = runParallelPipeline(p, ctx, files, file_count, worker_count, &t); + util.profEnd("pipeline", "2_extraction_total", &t_extract_total, file_count); + if (cancelled(p)) return c.CBM_NOT_FOUND; + if (rc == 0) cbm_pipeline_stamp_coverage(p, p.gbuf); + return rc; +} + +/// The stem collisions decide Folder QNs (structure) and Module QNs +/// (extraction); both phases read the one set. +fn runExtractionPhase(p: *Pipeline, ctx: *Ctx, files: [*c]const FileInfo, file_count: c_int) c_int { + var marks: StemMarks = undefined; + cbm_stem_marks_init(&marks, p.project_name, files, file_count); + ctx.stem_marks = &marks; + const rc = runStructureAndExtraction(p, ctx, files, file_count); + ctx.stem_marks = null; + cbm_stem_marks_free(&marks); + return rc; +} + +/// The full-index part of cbm_pipeline_run, from graph buffer creation to the +/// publication. The caller releases everything it sets up on `p` and `ctx`. +fn runFull( + p: *Pipeline, + ctx: *Ctx, + path_aliases: *?*c.cbm_path_alias_collection_t, + files: [*c]const FileInfo, + file_count: c_int, + t0: Timespec, + t_pipeline_total: *const Timespec, +) c_int { + // Phase 2: Create graph buffer and registry + p.gbuf = c.cbm_gbuf_new(p.project_name, p.repo_path); + p.registry = c.cbm_registry_new(); + + // Phase 2b: Load build-tool path aliases (tsconfig/jsconfig today). NULL + // when no usable configs are found — non-TS projects pay nothing. + path_aliases.* = c.cbm_load_path_aliases_excluded(p.repo_path, p.excluded_dirs, p.excluded_count); + + // Build shared context for pass functions + ctx.* = std.mem.zeroes(Ctx); + ctx.project_name = p.project_name; + ctx.repo_path = p.repo_path; + ctx.gbuf = p.gbuf; + ctx.registry = p.registry; + ctx.cancelled = &p.cancelled; + ctx.pipeline = @ptrCast(p); // so passes can record per-file skips (Track B) + ctx.mode = @intCast(p.mode); + ctx.path_aliases = path_aliases.*; + ctx.excluded_dirs = p.excluded_dirs; + ctx.excluded_count = p.excluded_count; + + var rc = runExtractionPhase(p, ctx, files, file_count); + if (rc != 0) return rc; + + rc = runPostExtraction(p, ctx, files, file_count); + if (rc != 0) return rc; + + cbm_pipeline_log_phase_mem("pipeline_done"); + interop.logInfo("pipeline.done", .{ + "nodes", itoa(c.cbm_gbuf_node_count(p.gbuf)), + "edges", itoa(c.cbm_gbuf_edge_count(p.gbuf)), + "elapsed_ms", elapsedText(t0), + }); + util.profEnd("pipeline", "TOTAL", t_pipeline_total, 0); + return 0; +} + +pub export fn cbm_pipeline_run(p_c: ?*Pipeline) c_int { + const p = p_c orelse return c.CBM_NOT_FOUND; + p.last_route = c.CBM_PIPELINE_ROUTE_UNKNOWN; + p.format_migration = false; + + const t_pipeline_total = util.profStart(); + var t0: Timespec = undefined; + now(&t0); + var path_aliases: ?*c.cbm_path_alias_collection_t = null; + var ctx = std.mem.zeroes(Ctx); // zeroed first: cleanup reads ctx.pkgmap + + // C/C++ #define Macro nodes (#375) dominate extraction on macro-dense repos + // (≈49% of nodes on the Linux kernel), so gate them to full mode — moderate + // and fast skip them entirely. Set before any extraction dispatch. + c.cbm_set_macro_extraction(@intFromBool(p.mode == c.CBM_MODE_FULL)); + + // Load user-defined extension overrides (fail-open: NULL on error) + const t_userconfig = util.profStart(); + p.userconfig = c.cbm_userconfig_load(p.repo_path); + c.cbm_set_user_lang_config(p.userconfig); + util.profEnd("pipeline", "0_userconfig_load", &t_userconfig, 0); + + // Phase 1: Discover files + const t_discover = util.profStart(); + var opts = std.mem.zeroes(c.cbm_discover_opts_t); + opts.mode = p.mode; + opts.ignore_file = null; + opts.max_file_size = 0; + var files: [*c]FileInfo = null; + var file_count: c_int = 0; + // Capture skipped subtrees on the pipeline so the MCP layer can report + // which directories were excluded (#411). Replace any prior list (e.g. a + // re-run on the same pipeline) to avoid leaking the previous one. + c.cbm_discover_free_excluded(p.excluded_dirs, p.excluded_count); + p.excluded_dirs = null; + p.excluded_count = 0; + var discover_degraded = false; + var rc = c.cbm_discover_ex(p.repo_path, &opts, &files, &file_count, &p.excluded_dirs, &p.excluded_count, &discover_degraded); + p.discover_degraded = discover_degraded; + if (discover_degraded) interop.logWarn("pipeline.discover", .{ "degraded", "true" }); + if (rc != 0) logError("pipeline.err", .{ "phase", "discover", "rc", itoa(rc) }); + util.profEnd("pipeline", "1_discover", &t_discover, file_count); + interop.logInfo("pipeline.discover", .{ + "files", itoa(file_count), + "elapsed_ms", elapsedText(t0), + }); + if (rc != 0 or cancelled(p)) { + rc = c.CBM_NOT_FOUND; + } else { + // Check for existing DB → try incremental or delete for reindex + rc = tryIncremental(p, files, file_count); + if (rc == PIPELINE_INCREMENTAL_FATAL) { + c.cbm_discover_free(files, file_count); + return c.CBM_NOT_FOUND; + } + if (rc == c.CBM_PIPELINE_ABORT_OVER_BUDGET) { + // A full run would hit the same budget; the served generation stays. + c.cbm_discover_free(files, file_count); + return rc; + } + if (rc >= 0) { + c.cbm_discover_free(files, file_count); + return rc; + } + interop.logInfo("pipeline.route", .{ "path", "full" }); + p.last_route = c.CBM_PIPELINE_ROUTE_FULL; + + rc = runFull(p, &ctx, &path_aliases, files, file_count, t0, &t_pipeline_total); + } + + // cleanup: + c.cbm_pkgmap_free(ctx.pkgmap); + ctx.pkgmap = null; + cbm_pipeline_set_pkgmap(null); + c.cbm_discover_free(files, file_count); + c.cbm_gbuf_free(p.gbuf); + p.gbuf = null; + c.cbm_registry_free(p.registry); + p.registry = null; + c.cbm_path_alias_collection_free(path_aliases); + // Clear and free user extension config + c.cbm_set_user_lang_config(null); + c.cbm_userconfig_free(p.userconfig); + p.userconfig = null; + return rc; +} diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c deleted file mode 100644 index 5010f8bd..00000000 --- a/src/pipeline/pipeline_incremental.c +++ /dev/null @@ -1,1580 +0,0 @@ -/* - * pipeline_incremental.c — Disk-based incremental re-indexing. - * - * Operates on the existing SQLite DB directly (not RAM-first graph buffer). - * Compares file mtime+size against stored hashes to classify changed/unchanged. - * Deletes changed files' nodes (edges cascade via ON DELETE CASCADE), - * re-parses only changed files through passes into a temp graph buffer, - * then merges new nodes/edges into the disk DB. Persists updated hashes. - * - * Called from pipeline.c when a DB with stored hashes already exists. - */ -#include "foundation/constants.h" - -#include "pipeline/pipeline.h" -#include -#include -#include "pipeline/pipeline_internal.h" -#include "pipeline/lsp_resolve.h" /* cbm_pipeline_node_is_callable_target */ -#include "store/store.h" -#include "zova/cbm_zova.h" -#include "zova/cbm_zova_repository.h" -#include "graph_buffer/graph_buffer.h" -#include "discover/discover.h" -#include "arena.h" -#include "helpers.h" /* cbm_fqn_module, cbm_fqn_folder */ -#include "foundation/log.h" -#include "foundation/hash_table.h" -#include "foundation/compat.h" -#include "foundation/compat_fs.h" -#include "foundation/limits.h" -#include "foundation/platform.h" -#include "foundation/sha256_backend.h" - -#include -#include -#include -#include -#include -#include - -typedef struct { - uint64_t xor_words[4]; - uint64_t sum_words[4]; - uint64_t count; -} cbm_incremental_multiset_digest_t; - -typedef struct { - cbm_incremental_multiset_digest_t input; - cbm_incremental_multiset_digest_t config; - cbm_incremental_multiset_digest_t similarity; - cbm_incremental_multiset_digest_t semantic; -} cbm_incremental_derived_state_t; - -static void derived_hash_text(cbm_sha256_backend_ctx *hash, const char *text) { - const char *value = text ? text : ""; - cbm_sha256_backend_update(hash, value, strlen(value)); - cbm_sha256_backend_update(hash, "\0", 1); -} - -static void derived_digest_add(cbm_incremental_multiset_digest_t *set, - cbm_sha256_backend_ctx *hash) { - uint8_t bytes[CBM_SHA256_DIGEST_LEN]; - uint64_t words[4]; - cbm_sha256_backend_final(hash, bytes); - memcpy(words, bytes, sizeof(words)); - for (int i = 0; i < 4; i++) { - set->xor_words[i] ^= words[i]; - set->sum_words[i] += words[i]; - } - set->count++; -} - -static bool derived_digest_equal(const cbm_incremental_multiset_digest_t *left, - const cbm_incremental_multiset_digest_t *right) { - return left->count == right->count && - memcmp(left->xor_words, right->xor_words, sizeof(left->xor_words)) == 0 && - memcmp(left->sum_words, right->sum_words, sizeof(left->sum_words)) == 0; -} - -typedef struct { - const cbm_gbuf_t *gbuf; - cbm_incremental_derived_state_t *state; -} cbm_incremental_derived_scan_t; - -static void derived_node_visitor(const cbm_gbuf_node_t *node, void *userdata) { - cbm_incremental_derived_scan_t *scan = userdata; - cbm_sha256_backend_ctx hash; - cbm_sha256_backend_init(&hash); - derived_hash_text(&hash, "node"); - derived_hash_text(&hash, node->label); - derived_hash_text(&hash, node->name); - derived_hash_text(&hash, node->qualified_name); - derived_hash_text(&hash, node->file_path); - cbm_sha256_backend_update(&hash, &node->start_line, sizeof(node->start_line)); - cbm_sha256_backend_update(&hash, &node->end_line, sizeof(node->end_line)); - derived_hash_text(&hash, node->properties_json); - derived_digest_add(&scan->state->input, &hash); -} - -static void derived_edge_visitor(const cbm_gbuf_edge_t *edge, void *userdata) { - cbm_incremental_derived_scan_t *scan = userdata; - cbm_incremental_multiset_digest_t *target = &scan->state->input; - if (strcmp(edge->type, "CONFIGURES") == 0) target = &scan->state->config; - else if (strcmp(edge->type, "SIMILAR_TO") == 0) target = &scan->state->similarity; - else if (strcmp(edge->type, "SEMANTICALLY_RELATED") == 0) - target = &scan->state->semantic; - const cbm_gbuf_node_t *source = cbm_gbuf_find_by_id(scan->gbuf, edge->source_id); - const cbm_gbuf_node_t *destination = cbm_gbuf_find_by_id(scan->gbuf, edge->target_id); - cbm_sha256_backend_ctx hash; - cbm_sha256_backend_init(&hash); - derived_hash_text(&hash, "edge"); - derived_hash_text(&hash, source ? source->qualified_name : NULL); - derived_hash_text(&hash, edge->type); - derived_hash_text(&hash, destination ? destination->qualified_name : NULL); - derived_hash_text(&hash, edge->properties_json); - derived_digest_add(target, &hash); -} - -static cbm_incremental_derived_state_t derived_state(const cbm_gbuf_t *gbuf) { - cbm_incremental_derived_state_t state = {0}; - cbm_incremental_derived_scan_t scan = {.gbuf = gbuf, .state = &state}; - cbm_gbuf_foreach_node(gbuf, derived_node_visitor, &scan); - cbm_gbuf_foreach_edge(gbuf, derived_edge_visitor, &scan); - return state; -} - -/* ── File classification ─────────────────────────────────────────── */ - -/* Classify discovered files against stored hashes using mtime+size. - * Returns a boolean array: changed[i] = true if files[i] needs re-parsing. - * Caller must free the returned array. */ -static bool *classify_files(cbm_file_info_t *files, int file_count, cbm_file_hash_t *stored, - int stored_count, int *out_changed, int *out_unchanged) { - bool *changed = calloc((size_t)file_count, sizeof(bool)); - if (!changed) { - return NULL; - } - - int n_changed = 0; - int n_unchanged = 0; - - /* Build lookup: rel_path -> stored hash */ - CBMHashTable *ht = - cbm_ht_create(stored_count > 0 ? (size_t)stored_count * PAIR_LEN : CBM_SZ_64); - for (int i = 0; i < stored_count; i++) { - cbm_ht_set(ht, stored[i].rel_path, &stored[i]); - } - - for (int i = 0; i < file_count; i++) { - cbm_file_hash_t *h = cbm_ht_get(ht, files[i].rel_path); - if (!h) { - /* New file */ - changed[i] = true; - n_changed++; - continue; - } - - struct stat st; - if (stat(files[i].path, &st) != 0) { - changed[i] = true; - n_changed++; - continue; - } - - if (cbm_stat_mtime_ns(&st) != h->mtime_ns || st.st_size != h->size) { - changed[i] = true; - n_changed++; - } else { - n_unchanged++; - } - } - - cbm_ht_free(ht); - *out_changed = n_changed; - *out_unchanged = n_unchanged; - return changed; -} - -/* Classify stored files that are absent from current discovery. Returns the - * count of truly-deleted files (output via out_deleted) and ALSO collects - * mode-skipped files into out_mode_skipped (caller frees both). - * - * A stored file is classified as: - * - "deleted" — `stat()` returns ENOENT or ENOTDIR. Its nodes will - * be purged and its hash row dropped. - * - "mode-skipped" — `stat()` succeeds. The file exists on disk but the - * current discovery pass didn't visit it (e.g. excluded - * by FAST_SKIP_DIRS in fast/moderate mode). Its nodes - * must be preserved AND its hash row must be carried - * forward into the new DB so subsequent reindexes can - * still see it as "known" rather than treating it as - * new-or-deleted. - * - * Without this distinction, a fast-mode reindex after a full-mode index - * would silently purge every file under `tools/`, `scripts/`, `bin/`, - * `build/`, `docs/`, `__tests__/`, etc. — see task - * claude-connectors/codebase-memory-index-repository-is-destructive-... - * and the 2026-04-13 Skyline incident (packages/mcp/src/tools/ vanished - * from a live graph mid-session). - * - * Mode-skipped hash preservation is the second half of the additive-merge - * contract: dump_and_persist re-upserts these hash rows so the next reindex - * can correctly detect a real on-disk deletion of a mode-skipped file (as - * opposed to seeing it as "never existed" → noop → orphaned graph nodes). - * - * Fail-safe rules (preserve nodes on uncertainty): - * - repo_path NULL → log error and preserve everything (return 0 - * deletions, empty mode_skipped). The caller contract is that - * repo_path is required; a NULL means a misconfigured pipeline, - * not a deletion signal. - * - snprintf truncation (combined path ≥ CBM_SZ_4K) → preserve. We can't - * reliably stat a truncated path. Treat as mode-skipped. - * - stat() errno != ENOENT/ENOTDIR (EACCES, EIO, ELOOP, transient NFS, - * etc.) → preserve. The file may exist; we just can't see it right now. - * Treat as mode-skipped. - * - * Note: we use stat() (not lstat()) on purpose. A symlink whose target was - * deleted should be classified as deleted from the indexer's perspective - * because the indexer follows symlinks during discovery — a stale symlink - * has no source to parse. */ -static int find_deleted_files(const char *repo_path, cbm_file_info_t *files, int file_count, - cbm_file_hash_t *stored, int stored_count, char ***out_deleted, - cbm_file_hash_t **out_mode_skipped, int *out_mode_skipped_count) { - *out_deleted = NULL; - *out_mode_skipped = NULL; - *out_mode_skipped_count = 0; - - if (!repo_path) { - /* Misconfigured pipeline. Preserve everything rather than risk - * silently re-introducing the destructive overwrite this function - * was rewritten to prevent. */ - cbm_log_error("incremental.err", "msg", "find_deleted_files_null_repo_path"); - return 0; - } - - CBMHashTable *current = cbm_ht_create((size_t)file_count * PAIR_LEN); - for (int i = 0; i < file_count; i++) { - cbm_ht_set(current, files[i].rel_path, &files[i]); - } - - int del_count = 0; - int del_cap = CBM_SZ_64; - char **deleted = malloc((size_t)del_cap * sizeof(char *)); - if (!deleted) { - cbm_log_error("incremental.err", "msg", "find_deleted_files_oom"); - cbm_ht_free(current); - return 0; - } - - int ms_count = 0; - int ms_cap = CBM_SZ_64; - cbm_file_hash_t *mode_skipped = malloc((size_t)ms_cap * sizeof(cbm_file_hash_t)); - if (!mode_skipped) { - cbm_log_error("incremental.err", "msg", "find_deleted_files_oom_ms"); - free(deleted); - cbm_ht_free(current); - return 0; - } - - for (int i = 0; i < stored_count; i++) { - if (cbm_ht_get(current, stored[i].rel_path)) { - continue; /* still visited by current pass */ - } - /* Not in current discovery — check if it's truly deleted or just - * mode-skipped (excluded by FAST_SKIP_DIRS etc.). */ - bool preserve = false; - char abs_path[CBM_SZ_4K]; - int n = snprintf(abs_path, sizeof(abs_path), "%s/%s", repo_path, stored[i].rel_path); - if (n < 0 || n >= (int)sizeof(abs_path)) { - /* Truncation or encoding error — can't reliably stat. Preserve. */ - cbm_log_warn("incremental.path_truncated", "rel_path", stored[i].rel_path); - preserve = true; - } else { - struct stat st; - if (stat(abs_path, &st) == 0) { - /* File exists on disk — mode-skipped, not deleted. */ - preserve = true; - } else if (errno != ENOENT && errno != ENOTDIR) { - /* Transient or permission error — fail safe by preserving. - * EACCES, EIO, ELOOP, ENAMETOOLONG, etc. */ - cbm_log_warn("incremental.stat_uncertain", "rel_path", stored[i].rel_path, "errno", - cbm_log_itoa(errno)); - preserve = true; - } - } - - if (preserve) { - /* Carry forward the existing hash row so subsequent reindexes - * can correctly classify this file. */ - if (ms_count >= ms_cap) { - ms_cap *= PAIR_LEN; - cbm_file_hash_t *tmp = realloc(mode_skipped, (size_t)ms_cap * sizeof(*tmp)); - if (!tmp) { - cbm_log_error("incremental.err", "msg", "find_deleted_files_realloc_oom_ms"); - break; - } - mode_skipped = tmp; - } - char *rp = strdup(stored[i].rel_path); - char *sh = stored[i].sha256 ? strdup(stored[i].sha256) : NULL; - if (!rp || (stored[i].sha256 && !sh)) { - /* OOM mid-record. Drop this entry rather than persist a - * row with a NULL rel_path that would silently fail the - * NOT NULL constraint in upsert and reintroduce the - * orphaned-node bug. */ - cbm_log_error("incremental.err", "msg", "find_deleted_files_strdup_oom", "rel_path", - stored[i].rel_path); - free(rp); - free(sh); - break; - } - mode_skipped[ms_count].project = NULL; /* unused by upsert API */ - mode_skipped[ms_count].rel_path = rp; - mode_skipped[ms_count].sha256 = sh; - mode_skipped[ms_count].mtime_ns = stored[i].mtime_ns; - mode_skipped[ms_count].size = stored[i].size; - ms_count++; - continue; - } - - /* File is truly gone — record for purge. */ - if (del_count >= del_cap) { - del_cap *= PAIR_LEN; - char **tmp = realloc(deleted, (size_t)del_cap * sizeof(char *)); - if (!tmp) { - cbm_log_error("incremental.err", "msg", "find_deleted_files_realloc_oom"); - break; - } - deleted = tmp; - } - deleted[del_count++] = strdup(stored[i].rel_path); - } - - cbm_ht_free(current); - *out_deleted = deleted; - *out_mode_skipped = mode_skipped; - *out_mode_skipped_count = ms_count; - return del_count; -} - -/* Free a mode_skipped array allocated by find_deleted_files. */ -static void free_mode_skipped(cbm_file_hash_t *ms, int count) { - if (!ms) { - return; - } - for (int i = 0; i < count; i++) { - free((void *)ms[i].rel_path); - free((void *)ms[i].sha256); - } - free(ms); -} - -/* Add the structural nodes for a newly discovered path before definition - * extraction. This mirrors the relevant part of the full structure pass: a - * Folder must exist before a directory-based Module definition arrives, or - * the Module wins the shared qualified-name slot and the graph diverges. */ -static void incr_add_changed_file_structure(cbm_gbuf_t *gbuf, const char *project, - const cbm_stem_marks_t *marks, - const cbm_file_info_t *file) { - if (!gbuf || !project || !file || !file->rel_path) { - return; - } - const char *rel_path = file->rel_path; - const char *slash = strrchr(rel_path, '/'); - const char *basename = slash ? slash + 1 : rel_path; - const char *extension = strrchr(basename, '.'); - char properties[CBM_SZ_256]; - snprintf(properties, sizeof(properties), "{\"extension\":\"%s\"}", - extension ? extension : ""); - char *file_qn = cbm_pipeline_fqn_compute(project, rel_path, "__file__"); - if (!file_qn) { - return; - } - cbm_gbuf_upsert_node(gbuf, "File", basename, file_qn, rel_path, 0, 0, properties); - - const cbm_gbuf_node_t **branches = NULL; - int branch_count = 0; - const char *branch_qn = project; - if (cbm_gbuf_find_by_label(gbuf, "Branch", &branches, &branch_count) == 0 && - branch_count > 0 && branches[0]->qualified_name) { - branch_qn = branches[0]->qualified_name; - } - - char *dir = strdup(rel_path); - if (!dir) { - free(file_qn); - return; - } - char *last_slash = strrchr(dir, '/'); - if (last_slash) { - *last_slash = '\0'; - } else { - dir[0] = '\0'; - } - char file_parent_buf[CBM_SZ_8K]; - const char *file_parent_qn = - dir[0] ? cbm_stem_marks_folder_qn(marks, project, dir, file_parent_buf, - sizeof(file_parent_buf)) - : NULL; - - for (char *walk = dir; walk[0];) { - char folder_buf[CBM_SZ_8K]; - const char *folder_qn = - cbm_stem_marks_folder_qn(marks, project, walk, folder_buf, sizeof(folder_buf)); - if (!folder_qn) { - break; - } - const char *folder_name = strrchr(walk, '/'); - folder_name = folder_name ? folder_name + 1 : walk; - cbm_gbuf_upsert_node(gbuf, "Folder", folder_name, folder_qn, walk, 0, 0, "{}"); - - char *parent_dir = strdup(walk); - if (!parent_dir) { - break; - } - char *parent_slash = strrchr(parent_dir, '/'); - if (parent_slash) { - *parent_slash = '\0'; - } else { - parent_dir[0] = '\0'; - } - char parent_buf[CBM_SZ_8K]; - const char *parent_qn = parent_dir[0] - ? cbm_stem_marks_folder_qn(marks, project, parent_dir, - parent_buf, sizeof(parent_buf)) - : branch_qn; - const cbm_gbuf_node_t *folder = cbm_gbuf_find_by_qn(gbuf, folder_qn); - const cbm_gbuf_node_t *parent = cbm_gbuf_find_by_qn(gbuf, parent_qn); - if (folder && parent) { - cbm_gbuf_insert_edge(gbuf, parent->id, folder->id, "CONTAINS_FOLDER", "{}"); - } - free(parent_dir); - - char *up = strrchr(walk, '/'); - if (up) { - *up = '\0'; - } else { - walk[0] = '\0'; - } - } - - const cbm_gbuf_node_t *file_node = cbm_gbuf_find_by_qn(gbuf, file_qn); - const char *parent_qn = file_parent_qn ? file_parent_qn : branch_qn; - const cbm_gbuf_node_t *parent = cbm_gbuf_find_by_qn(gbuf, parent_qn); - if (file_node && parent) { - cbm_gbuf_insert_edge(gbuf, parent->id, file_node->id, "CONTAINS_FILE", "{}"); - } - free(dir); - free(file_qn); -} - -/* True when the stem QN or a directory QN of rel yields on a different side - * (CBM_STEM_*) in the two file sets. */ -static bool incr_stem_side_differs(const cbm_stem_marks_t *now, const cbm_stem_marks_t *before, - CBMArena *a, const char *project, const char *rel) { - char *stem = cbm_fqn_module(a, project, rel); - if (stem && cbm_stem_marks_side(now, stem) != cbm_stem_marks_side(before, stem)) { - return true; - } - for (const char *s = strchr(rel, '/'); s; s = strchr(s + SKIP_ONE, '/')) { - char *dir = cbm_arena_strndup(a, rel, (size_t)(s - rel)); - char *fqn = dir ? cbm_fqn_folder(a, project, dir) : NULL; - if (fqn && cbm_stem_marks_side(now, fqn) != cbm_stem_marks_side(before, fqn)) { - return true; - } - } - return false; -} - -/* The file set of the stored generation: the discovered files it already - * held, plus the deleted ones, which take the language their name gives. - * Counts the discovered files it did not hold in *added. */ -static cbm_file_info_t *incr_previous_files(const cbm_file_info_t *files, int file_count, - CBMHashTable *stored_paths, char **deleted, - int deleted_count, int *out_count, int *added) { - cbm_file_info_t *before = - calloc((size_t)file_count + (size_t)deleted_count + SKIP_ONE, sizeof(*before)); - int n = 0; - for (int i = 0; i < file_count; i++) { - if (files[i].rel_path && cbm_ht_get(stored_paths, files[i].rel_path)) { - if (before) { - before[n++] = files[i]; - } - } else { - (*added)++; - } - } - for (int i = 0; before && i < deleted_count; i++) { - const char *base = strrchr(deleted[i], '/'); - before[n++] = (cbm_file_info_t){ - .rel_path = deleted[i], - .language = cbm_language_for_filename(base ? base + SKIP_ONE : deleted[i]), - }; - } - *out_count = n; - return before; -} - -/* Whether an added or deleted file yields on a different side of a stem - * collision in `now` than in `prev`. */ -static bool incr_stem_sides_moved(const char *project, const cbm_file_info_t *files, int file_count, - CBMHashTable *stored_paths, const cbm_stem_marks_t *now, - const cbm_stem_marks_t *prev, char **deleted, int deleted_count) { - CBMArena a; - cbm_arena_init(&a); - bool moved = false; - for (int i = 0; !moved && i < file_count; i++) { - if (files[i].rel_path && !cbm_ht_get(stored_paths, files[i].rel_path)) { - moved = incr_stem_side_differs(now, prev, &a, project, files[i].rel_path); - } - } - for (int i = 0; !moved && i < deleted_count; i++) { - moved = incr_stem_side_differs(now, prev, &a, project, deleted[i]); - } - cbm_arena_destroy(&a); - return moved; -} - -/* Whether the added and deleted files change a stem collision - * (cbm_stem_marks_t): one appears or disappears, or the yielding side flips - * because a Java/Go package gains its first or loses its last file. The - * unchanged files keep the spelling of the stored generation, so only a full - * index re-spells them. `now` holds the discovered files; the previous set is - * the stored paths, where a deleted file takes the language its name gives. - * A changed file can only move the sides of its own stem QN and of the - * directories above it, so only those are compared. */ -static bool incr_stem_marks_changed(const char *project, const cbm_file_info_t *files, - int file_count, const cbm_stem_marks_t *now, - cbm_file_hash_t *stored, int stored_count, char **deleted, - int deleted_count) { - CBMHashTable *stored_paths = - cbm_ht_create(stored_count > 0 ? (size_t)stored_count * PAIR_LEN : CBM_SZ_64); - for (int i = 0; i < stored_count; i++) { - if (stored[i].rel_path) { - cbm_ht_set(stored_paths, stored[i].rel_path, &stored[i]); - } - } - int before_count = 0; - int added = 0; - cbm_file_info_t *before = incr_previous_files(files, file_count, stored_paths, deleted, - deleted_count, &before_count, &added); - bool changed = false; - if (added > 0 || deleted_count > 0) { - /* Without the previous set the comparison cannot be made: rebuild. */ - changed = true; - if (before) { - cbm_stem_marks_t prev; - cbm_stem_marks_init(&prev, project, before, before_count); - changed = incr_stem_sides_moved(project, files, file_count, stored_paths, now, &prev, - deleted, deleted_count); - cbm_stem_marks_free(&prev); - } - } - free(before); - cbm_ht_free(stored_paths); - return changed; -} - -/* ── Inbound cross-file edge preservation (incremental correctness) ── - * - * The purge step (cbm_gbuf_delete_by_file) removes a changed file's nodes, - * and the cascade then drops every edge referencing them — INCLUDING inbound - * edges whose source lives in an UNCHANGED file (e.g. StudyService.grade -> - * SM2.review, or a Folder -> File containment edge). Because incremental only - * re-parses the changed files, the resolution passes never regenerate those - * inbound edges, so the graph silently loses cross-file CALLS / USAGE / - * CONTAINS_FILE / INHERITS / ... edges on every edit and diverges from a - * clean full reindex (which resolves every file). - * - * Fix: snapshot inbound cross-file edges into changed or deleted files BEFORE - * the purge. Re-run CALLS resolution for unchanged callers after re-extraction, - * then re-link non-CALLS edges by exact qualified-name target. Notes: - * - Only edges whose target is in a changed file and whose source is NOT - * are snapshotted; edges out of a changed file are regenerated when that - * file is re-resolved. - * - Edge types recomputed wholesale by post-passes (SIMILAR_TO, - * SEMANTICALLY_RELATED) are skipped — re-linking a stale snapshot could - * add edges a full reindex would not produce. - * - cbm_gbuf_insert_edge dedups, so re-linking an edge the resolver already - * recreated is a harmless no-op. - * - A deleted target is dropped. A renamed CALLS edge is rebuilt from its - * unchanged caller source, preserving the resolver's current confidence - * and strategy rather than copying stale edge properties. */ - -typedef struct { - char *source_qn; - char *target_qn; - char *target_name; - char *target_label; - char *type; - char *props; -} cbm_saved_edge_t; - -typedef struct { - cbm_gbuf_t *gbuf; - CBMHashTable *changed_paths; /* rel_path -> non-NULL sentinel (membership set) */ - cbm_saved_edge_t *items; - int count; - int cap; -} cbm_edge_capture_t; - -/* Edge types that must NOT be re-linked from the pre-purge snapshot, because a - * full reindex (re)computes them via a pass whose result can differ from the - * snapshot — restoring a stale copy could leave wrong properties or even an - * edge a full reindex would not produce: - * - SIMILAR_TO / SEMANTICALLY_RELATED / FILE_CHANGES_WITH / DATA_FLOWS: - * rebuilt wholesale by the incremental post-passes (similarity / - * semantic_edges / githistory / route matching) over a drifting corpus. - * Structural containment edges are regenerated from changed paths before - * extraction, so snapshots omit them rather than retaining a pre-rename - * parent relationship. */ -static bool incr_edge_type_is_recomputed(const char *type) { - return type && (strcmp(type, "SIMILAR_TO") == 0 || strcmp(type, "SEMANTICALLY_RELATED") == 0 || - strcmp(type, "FILE_CHANGES_WITH") == 0 || strcmp(type, "DATA_FLOWS") == 0 || - strcmp(type, "CONTAINS_FILE") == 0 || strcmp(type, "CONTAINS_FOLDER") == 0); -} - -/* cbm_gbuf_foreach_edge visitor: snapshot inbound cross-file edges into - * changed files so they survive the purge and can be re-linked afterward. */ -static void incr_capture_inbound_edge(const cbm_gbuf_edge_t *edge, void *userdata) { - cbm_edge_capture_t *cap = (cbm_edge_capture_t *)userdata; - if (incr_edge_type_is_recomputed(edge->type)) { - return; - } - const cbm_gbuf_node_t *src = cbm_gbuf_find_by_id(cap->gbuf, edge->source_id); - const cbm_gbuf_node_t *tgt = cbm_gbuf_find_by_id(cap->gbuf, edge->target_id); - if (!src || !tgt || !src->qualified_name || !tgt->qualified_name || !src->file_path || - !tgt->file_path) { - return; - } - /* Keep only edges that the purge would orphan permanently: target is in a - * changed file (its node is deleted + re-created), source is NOT (its file - * is never re-parsed, so the resolver won't regenerate the edge). */ - if (!cbm_ht_get(cap->changed_paths, tgt->file_path) || - cbm_ht_get(cap->changed_paths, src->file_path)) { - return; - } - if (cap->count >= cap->cap) { - int ncap = (cap->cap > 0) ? cap->cap * PAIR_LEN : CBM_SZ_64; - cbm_saved_edge_t *tmp = realloc(cap->items, (size_t)ncap * sizeof(*tmp)); - if (!tmp) { - cbm_log_warn("incremental.edge_snapshot_oom", "captured", cbm_log_itoa(cap->count)); - return; /* best-effort: stop capturing, keep what we have */ - } - cap->items = tmp; - cap->cap = ncap; - } - cbm_saved_edge_t *s = &cap->items[cap->count]; - s->source_qn = strdup(src->qualified_name); - s->target_qn = strdup(tgt->qualified_name); - s->target_name = strdup(tgt->name ? tgt->name : ""); - s->target_label = strdup(tgt->label ? tgt->label : ""); - s->type = strdup(edge->type); - s->props = strdup(edge->properties_json ? edge->properties_json : "{}"); - if (!s->source_qn || !s->target_qn || !s->target_name || !s->target_label || !s->type || - !s->props) { - free(s->source_qn); - free(s->target_qn); - free(s->target_name); - free(s->target_label); - free(s->type); - free(s->props); - return; - } - cap->count++; -} - -static const cbm_gbuf_node_t *incr_find_unique_renamed_target(const cbm_gbuf_t *gbuf, - const cbm_saved_edge_t *saved) { - if (!saved->target_name || !saved->target_name[0] || !saved->target_label || - !saved->target_label[0]) { - return NULL; - } - const cbm_gbuf_node_t **nodes = NULL; - int count = 0; - if (cbm_gbuf_find_by_name(gbuf, saved->target_name, &nodes, &count) != 0) { - return NULL; - } - const cbm_gbuf_node_t *candidate = NULL; - for (int i = 0; i < count; i++) { - const cbm_gbuf_node_t *node = nodes[i]; - if (!node->qualified_name || !node->label || strcmp(node->label, saved->target_label) != 0) { - continue; - } - if (candidate) { - return NULL; /* ambiguous: do not guess a rename target */ - } - candidate = node; - } - return candidate; -} - -static int incr_cmp_id(const void *a, const void *b) { - int64_t x = *(const int64_t *)a; - int64_t y = *(const int64_t *)b; - return (x > y) - (x < y); -} - -typedef struct { - const int64_t *ids; - int count; -} incr_id_set_t; - -static bool incr_keep_unless_source_in_set(const cbm_gbuf_t *gb, const cbm_gbuf_edge_t *edge, - void *userdata) { - (void)gb; - const incr_id_set_t *set = userdata; - if (edge->properties_json && strstr(edge->properties_json, "\"env_access\"") != NULL) { - return true; - } - return !bsearch(&edge->source_id, set->ids, (size_t)set->count, sizeof(*set->ids), - incr_cmp_id); -} - -/* A path rename changes the callee's qualified name. Re-run the CALLS pass - * for only unchanged source files that previously called a purged target, so - * the normal resolver recalculates strategy/confidence against the new - * registry instead of preserving stale edge properties. */ -static void incr_rebuild_snapshot_callers(cbm_pipeline_ctx_t *ctx, const cbm_edge_capture_t *cap, - const cbm_file_info_t *files, int file_count) { - if (!ctx || !cap || !files || file_count <= 0) { - return; - } - cbm_file_info_t *callers = calloc((size_t)cap->count, sizeof(*callers)); - int64_t *source_ids = malloc((size_t)cap->count * sizeof(*source_ids)); - if (!callers || !source_ids) { - free(callers); - free(source_ids); - return; - } - int caller_count = 0; - int source_count = 0; - for (int i = 0; i < cap->count; i++) { - const cbm_saved_edge_t *saved = &cap->items[i]; - /* Re-resolve unchanged sources that previously emitted either CALLS - * or resolver-owned CONFIGURES. CONFIGURES may be the only derived - * edge from a source (for example an environment accessor), so - * restricting this list to CALLS leaves those edges missing after a - * changed-file purge. */ - if (!saved->type || (strcmp(saved->type, "CALLS") != 0 && - strcmp(saved->type, "CONFIGURES") != 0)) { - continue; - } - const cbm_gbuf_node_t *source = cbm_gbuf_find_by_qn(ctx->gbuf, saved->source_qn); - if (!source || !source->file_path) { - continue; - } - source_ids[source_count++] = source->id; - for (int f = 0; f < file_count; f++) { - if (strcmp(files[f].rel_path, source->file_path) != 0) { - continue; - } - bool already_added = false; - for (int existing = 0; existing < caller_count; existing++) { - if (strcmp(callers[existing].rel_path, files[f].rel_path) == 0) { - already_added = true; - break; - } - } - if (!already_added) { - callers[caller_count++] = files[f]; - } - break; - } - } - /* Drop these sources' CONFIGURES edges in one pass over the edge list: - * deleting per captured edge rescanned every edge and rebuilt the edge - * indexes each time, which made a one-file reindex slower than a full one. */ - if (source_count > 0) { - qsort(source_ids, (size_t)source_count, sizeof(*source_ids), incr_cmp_id); - incr_id_set_t set = {source_ids, source_count}; - cbm_gbuf_delete_edges_by_type_unless(ctx->gbuf, "CONFIGURES", - incr_keep_unless_source_in_set, &set); - } - if (caller_count > 0) { - cbm_parallel_resolve_calls(ctx, callers, caller_count); - } - free(source_ids); - free(callers); -} - -/* Re-link snapshotted inbound edges to the freshly re-created target nodes. - * Returns the number of edges re-linked. */ -static int incr_restore_inbound_edges(cbm_gbuf_t *gbuf, cbm_edge_capture_t *cap) { - int restored = 0; - for (int i = 0; i < cap->count; i++) { - cbm_saved_edge_t *s = &cap->items[i]; - const cbm_gbuf_node_t *src = cbm_gbuf_find_by_qn(gbuf, s->source_qn); - const cbm_gbuf_node_t *tgt = cbm_gbuf_find_by_qn(gbuf, s->target_qn); - if (!tgt && s->type && strcmp(s->type, "CALLS") == 0) { - continue; /* rebuilt from source so resolver owns its properties */ - } - /* Incremental runs skip cross-file LSP, so an unchanged source cannot - * re-prove its CALL_REFERENCE. Keep the snapshot edge only while the - * same QN is still a callable; never guess a rename. */ - if (s->type && strcmp(s->type, "CALL_REFERENCE") == 0 && - !cbm_pipeline_node_is_callable_target(tgt)) { - continue; - } - if (!tgt) { - tgt = incr_find_unique_renamed_target(gbuf, s); - } - if (src && tgt) { - cbm_gbuf_insert_edge(gbuf, src->id, tgt->id, s->type, s->props); - restored++; - } - } - return restored; -} - -static void incr_free_edge_capture(cbm_edge_capture_t *cap) { - for (int i = 0; i < cap->count; i++) { - free(cap->items[i].source_qn); - free(cap->items[i].target_qn); - free(cap->items[i].target_name); - free(cap->items[i].target_label); - free(cap->items[i].type); - free(cap->items[i].props); - } - free(cap->items); - cap->items = NULL; - cap->count = 0; - cap->cap = 0; -} - -/* ── Registry seed visitor ────────────────────────────────────────── */ - -/* Labels the full index seeds into the registry - * (pass_parallel.c register_and_link_def — KEEP IN SYNC). Incremental re-resolution must see the - * SAME symbol set, or it diverges from a clean full reindex: seeding extra - * container nodes (File / Module / Folder / ...) lets a type usage like `Word` - * resolve to the same-named Module node instead of the Class node. Only - * callable / declared symbols belong in the registry. */ -static bool incr_label_is_registry_symbol(const char *label) { - /* Mirror pass_parallel.c registry seeding EXACTLY: - * callables + every type-like container (Class/Struct/Interface/Enum/Type/ - * Trait) + Variable/Field. Struct included so an incremental re-resolve seeds - * the same struct type nodes a full reindex would. */ - return label && (strcmp(label, "Function") == 0 || strcmp(label, "Method") == 0 || - cbm_label_is_type_like(label) || strcmp(label, "Variable") == 0 || - strcmp(label, "Field") == 0); -} - -/* Callback for cbm_gbuf_foreach_node: seed the registry with the existing - * project's definition symbols so the resolver can match cross-file symbols - * during incremental. Mirrors the full-index registry contents exactly so an - * incremental re-resolve picks the same nodes a full reindex would. */ -static void registry_visitor(const cbm_gbuf_node_t *node, void *userdata) { - cbm_registry_t *r = (cbm_registry_t *)userdata; - if (!incr_label_is_registry_symbol(node->label)) { - return; - } - cbm_registry_add(r, node->name, node->qualified_name, node->label); -} - -/* Extract + resolve the changed files. */ -static int run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed_files, int ci) { - struct timespec t; - - /* Per-file LSP always runs (every mode). Cross-file LSP stays disabled in - * incremental regardless (cbm_parallel_resolve is called with NULL - * cross_registries below). */ - int worker_count = cbm_default_worker_count(true); - cbm_log_info("incremental.mode", "mode", "parallel", "workers", cbm_log_itoa(worker_count), - "changed", cbm_log_itoa(ci)); - - _Atomic int64_t shared_ids; - atomic_init(&shared_ids, cbm_gbuf_next_id(ctx->gbuf)); - - CBMFileResult **cache = (CBMFileResult **)calloc(ci, sizeof(CBMFileResult *)); - if (cache) { - cbm_clock_gettime(CLOCK_MONOTONIC, &t); - int rc = cbm_parallel_extract(ctx, changed_files, ci, cache, &shared_ids, worker_count); - if (rc == CBM_PIPELINE_ABORT_OVER_BUDGET) { - /* Over the memory budget: stop before publication so the - * served generation stays as it was. */ - for (int j = 0; j < ci; j++) { - if (cache[j]) { - cbm_free_result(cache[j]); - } - } - free(cache); - return rc; - } - cbm_pipeline_sync_next_id(ctx->gbuf, &shared_ids); - cbm_log_info("pass.timing", "pass", "incr_extract", "elapsed_ms", - cbm_log_itoa((int)cbm_elapsed_ms(t))); - - cbm_clock_gettime(CLOCK_MONOTONIC, &t); - cbm_build_registry_from_cache(ctx, changed_files, ci, cache); - cbm_pipeline_share_next_id(ctx->gbuf, &shared_ids); - cbm_log_info("pass.timing", "pass", "incr_registry", "elapsed_ms", - cbm_log_itoa((int)cbm_elapsed_ms(t))); - - /* Incremental skips cross-file LSP precondition build — it - * would need all_defs from the full project, not just the - * changed slice. Per-file LSP (run inside cbm_extract_file) - * still fires; cross-file resolution is deferred to the - * next full re-index. Pass NULL/0/NULL to make the fused - * step in resolve_worker a no-op. */ - cbm_clock_gettime(CLOCK_MONOTONIC, &t); - cbm_parallel_resolve(ctx, changed_files, ci, cache, &shared_ids, worker_count, NULL, 0, - NULL, NULL /* module_def_index */, - NULL /* cross_registries — incremental skips Tier 2 prebuild */); - cbm_pipeline_sync_next_id(ctx->gbuf, &shared_ids); - cbm_log_info("pass.timing", "pass", "incr_resolve", "elapsed_ms", - cbm_log_itoa((int)cbm_elapsed_ms(t))); - - cbm_pipeline_extract_infra_routes(ctx->gbuf, changed_files, cache, NULL, ci); - cbm_pipeline_process_infra_bindings(ctx->gbuf, changed_files, cache, NULL, ci); - for (int j = 0; j < ci; j++) { - if (cache[j]) { - cbm_free_result(cache[j]); - } - } - free(cache); - } - return 0; -} - -/* Infra Route nodes are keyed by URL across config files: a URL becomes a - * Route only when no config file lists it under a denied key, and the node - * belongs to one of the files that list it. A change to one config file can - * therefore change what another contributes, so when any config file changed - * or was deleted, every config file is re-extracted. Returns files added. */ -static int incr_expand_infra_changes(const cbm_file_info_t *files, int file_count, - bool *is_changed, char *const *deleted, int deleted_count) { - bool infra_touched = false; - for (int i = 0; i < file_count && !infra_touched; i++) { - infra_touched = is_changed[i] && cbm_pipeline_is_infra_file(files[i].rel_path); - } - for (int i = 0; i < deleted_count && !infra_touched; i++) { - infra_touched = cbm_pipeline_is_infra_file(deleted[i]); - } - if (!infra_touched) { - return 0; - } - int added = 0; - for (int i = 0; i < file_count; i++) { - if (!is_changed[i] && cbm_pipeline_is_infra_file(files[i].rel_path)) { - is_changed[i] = true; - added++; - } - } - return added; -} - -typedef struct { - const CBMHashTable *changed_paths; - CBMHashTable *client_files; -} incr_client_scan_ctx_t; - -static void incr_find_existing_client_nodes(const cbm_gbuf_node_t *n, void *userdata) { - incr_client_scan_ctx_t *ctx = (incr_client_scan_ctx_t *)userdata; - if (!n || !n->file_path || !n->label || !n->properties_json) { - return; - } - if (strcmp(n->label, "Module") != 0 && strcmp(n->label, "Variable") != 0) { - return; - } - if (!cbm_ht_get(ctx->changed_paths, n->file_path)) { - return; - } - if (strstr(n->properties_json, "\"http_client\":") != NULL) { - if (!cbm_ht_get(ctx->client_files, n->file_path)) { - char *k = strdup(n->file_path); - if (k) { - cbm_ht_set(ctx->client_files, k, k); - } - } - } -} - -static bool incr_is_client_file(const cbm_file_info_t *fi) { - if (!fi || !fi->path) { - return false; - } - if (fi->language != CBM_LANG_JAVASCRIPT && fi->language != CBM_LANG_TYPESCRIPT && - fi->language != CBM_LANG_TSX) { - return false; - } - FILE *f = cbm_fopen(fi->path, "rb"); - if (!f) { - return false; - } - (void)fseek(f, 0, SEEK_END); - long size = ftell(f); - (void)fseek(f, 0, SEEK_SET); - if (size <= 0 || size > cbm_max_file_bytes()) { - (void)fclose(f); - return false; - } - char *buf = malloc((size_t)size + 1); - if (!buf) { - (void)fclose(f); - return false; - } - size_t r = fread(buf, 1, (size_t)size, f); - (void)fclose(f); - buf[r] = '\0'; - - if (strstr(buf, "axios.create") == NULL) { - free(buf); - return false; - } - - CBMFileResult *fr = - cbm_extract_file(buf, (int)r, fi->language, "incr_probe", fi->rel_path, 0, NULL, NULL); - free(buf); - bool has_client = false; - if (fr) { - for (int i = 0; i < fr->defs.count; i++) { - if (fr->defs.items[i].http_client != NULL) { - has_client = true; - break; - } - } - cbm_free_result(fr); - } - return has_client; -} - -static void free_ht_keys_cb(const char *key, void *value, void *userdata) { - (void)value; - (void)userdata; - free((char *)key); -} - -static void free_string_hash_table(CBMHashTable *ht) { - if (!ht) { - return; - } - cbm_ht_foreach(ht, free_ht_keys_cb, NULL); - cbm_ht_free(ht); -} - -/* An axios client instance's baseURL is composed into the HTTP_CALLS edges of - * files that import it. When any file that defines an http_client changes or was - * deleted (before or after the edit), treat the files that import it as changed - * files so they go through full re-extraction. Returns files added. (#75, #1916) */ -static int incr_expand_client_importer_changes(const cbm_gbuf_t *existing, - const cbm_file_info_t *files, int file_count, - bool *is_changed, char *const *deleted, - int deleted_count) { - if (!existing || !files || file_count <= 0 || !is_changed) { - return 0; - } - CBMHashTable *changed_set = cbm_ht_create((uint32_t)(file_count + deleted_count)); - if (!changed_set) { - return 0; - } - for (int i = 0; i < file_count; i++) { - if (is_changed[i]) { - cbm_ht_set(changed_set, files[i].rel_path, (void *)1); - } - } - for (int i = 0; i < deleted_count; i++) { - cbm_ht_set(changed_set, deleted[i], (void *)1); - } - - CBMHashTable *client_files = cbm_ht_create(16); - if (!client_files) { - cbm_ht_free(changed_set); - return 0; - } - - /* 1. Check existing graph for files that defined an http_client before the edit */ - incr_client_scan_ctx_t scan_ctx = { - .changed_paths = changed_set, - .client_files = client_files, - }; - cbm_gbuf_foreach_node(existing, incr_find_existing_client_nodes, &scan_ctx); - - /* 2. Check changed files on disk to detect newly added client instances */ - for (int i = 0; i < file_count; i++) { - if (!is_changed[i]) { - continue; - } - if (cbm_ht_get(client_files, files[i].rel_path)) { - continue; - } - if (incr_is_client_file(&files[i])) { - char *k = strdup(files[i].rel_path); - if (k) { - cbm_ht_set(client_files, k, k); - } - } - } - cbm_ht_free(changed_set); - - if (cbm_ht_count(client_files) == 0) { - free_string_hash_table(client_files); - return 0; - } - - /* 3. Find files in existing graph buffer that import any client file */ - const cbm_gbuf_edge_t **edges = NULL; - int edge_count = 0; - int added = 0; - if (cbm_gbuf_find_edges_by_type(existing, "IMPORTS", &edges, &edge_count) == 0 && edges) { - for (int e = 0; e < edge_count; e++) { - const cbm_gbuf_edge_t *edge = edges[e]; - const cbm_gbuf_node_t *tgt = cbm_gbuf_find_by_id(existing, edge->target_id); - if (!tgt || !tgt->file_path || !cbm_ht_get(client_files, tgt->file_path)) { - continue; - } - const cbm_gbuf_node_t *src = cbm_gbuf_find_by_id(existing, edge->source_id); - if (!src || !src->file_path) { - continue; - } - for (int f = 0; f < file_count; f++) { - if (strcmp(files[f].rel_path, src->file_path) == 0) { - if (!is_changed[f]) { - is_changed[f] = true; - added++; - } - break; - } - } - } - } - free_string_hash_table(client_files); - return added; -} - -/* HANDLES edges into infra Routes come only from route matching. */ -static bool incr_handles_edge_is_not_infra_match(const cbm_gbuf_t *gbuf, - const cbm_gbuf_edge_t *edge, void *userdata) { - (void)userdata; - static const char infra_prefix[] = "__route__infra__"; - const cbm_gbuf_node_t *target = cbm_gbuf_find_by_id(gbuf, edge->target_id); - return !target || !target->qualified_name || - strncmp(target->qualified_name, infra_prefix, sizeof(infra_prefix) - 1) != 0; -} - -/* Run post-extraction passes (tests, decorator tags, configlink). */ -static void run_postpasses(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed_files, int ci, - const char *project, - const cbm_incremental_derived_state_t *before) { - struct timespec t; - - cbm_clock_gettime(CLOCK_MONOTONIC, &t); - cbm_pipeline_pass_tests(ctx, changed_files, ci); - cbm_log_info("pass.timing", "pass", "incr_tests", "elapsed_ms", - cbm_log_itoa((int)cbm_elapsed_ms(t))); - - /* File nodes for changed paths are deleted and recreated above, so their - * git-history properties disappear unless the full-repository derived - * pass is reapplied. Rebuild the derived edges as well to match a fresh - * full index instead of accumulating stale FILE_CHANGES_WITH rows. */ - cbm_gbuf_delete_edges_by_type(ctx->gbuf, "FILE_CHANGES_WITH"); - cbm_clock_gettime(CLOCK_MONOTONIC, &t); - int history_rc = cbm_pipeline_pass_githistory(ctx); - cbm_log_info("pass.timing", "pass", "incr_githistory", "elapsed_ms", - cbm_log_itoa((int)cbm_elapsed_ms(t)), "rc", cbm_log_itoa(history_rc)); - - cbm_clock_gettime(CLOCK_MONOTONIC, &t); - cbm_pipeline_pass_decorator_tags(ctx->gbuf, project); - cbm_log_info("pass.timing", "pass", "incr_decorator_tags", "elapsed_ms", - cbm_log_itoa((int)cbm_elapsed_ms(t))); - - /* Route matching is global: an infra Route in an unchanged config file can - * match a handler in a changed file, and DATA_FLOWS follow those matches. - * Remove the previous matches and data flows, then rerun the pass a full - * index runs, before the derived digest reads the graph. */ - cbm_clock_gettime(CLOCK_MONOTONIC, &t); - cbm_gbuf_delete_edges_by_type_unless(ctx->gbuf, "HANDLES", - incr_handles_edge_is_not_infra_match, NULL); - cbm_gbuf_delete_edges_by_type(ctx->gbuf, "DATA_FLOWS"); - cbm_pipeline_create_route_nodes(ctx->gbuf); - cbm_log_info("pass.timing", "pass", "incr_route_match", "elapsed_ms", - cbm_log_itoa((int)cbm_elapsed_ms(t))); - - /* Restore the final derived node properties before comparing pass inputs. - * The semantic and similarity passes ignore these complexity fields, but - * reproducing them makes the conservative whole-graph input digest exact. */ - cbm_clock_gettime(CLOCK_MONOTONIC, &t); - cbm_pipeline_pass_complexity(ctx); - cbm_log_info("pass.timing", "pass", "incr_complexity", "elapsed_ms", - cbm_log_itoa((int)cbm_elapsed_ms(t))); - - /* Importance reads CALLS/USAGE (re-extraction plus relinked inbound - * edges) and TESTS (pass_tests, first in this function). It must run - * before the digest below, which hashes properties_json. Nodes from - * unchanged files already carry the key; the write-back overwrites it. */ - cbm_clock_gettime(CLOCK_MONOTONIC, &t); - cbm_pipeline_pass_importance(ctx); - cbm_log_info("pass.timing", "pass", "incr_importance", "elapsed_ms", - cbm_log_itoa((int)cbm_elapsed_ms(t))); - - cbm_clock_gettime(CLOCK_MONOTONIC, &t); - cbm_incremental_derived_state_t after = derived_state(ctx->gbuf); - bool input_unchanged = before && derived_digest_equal(&before->input, &after.input); - cbm_log_info("incremental.derived_digest", "unchanged", input_unchanged ? "true" : "false", - "elapsed_ms", cbm_log_itoa((int)cbm_elapsed_ms(t))); - - bool config_unchanged = input_unchanged && - derived_digest_equal(&before->config, &after.config); - if (!config_unchanged) { - cbm_clock_gettime(CLOCK_MONOTONIC, &t); - cbm_pipeline_pass_configlink(ctx); - cbm_log_info("pass.timing", "pass", "incr_configlink", "elapsed_ms", - cbm_log_itoa((int)cbm_elapsed_ms(t))); - } else { - cbm_log_info("incremental.derived_gate", "component", "configlink", "decision", "reuse"); - } - - /* SIMILAR_TO + SEMANTICALLY_RELATED edges only in moderate/full modes */ - if (ctx->mode <= CBM_MODE_MODERATE) { - /* These are global derived relations. The old graph was loaded before - * this incremental update, so rerunning either pass without first - * removing its prior output leaves relationships that a clean full - * index would no longer select after the corpus changed. */ - bool similarity_unchanged = input_unchanged && - derived_digest_equal(&before->similarity, &after.similarity); - if (!similarity_unchanged) { - cbm_gbuf_delete_edges_by_type(ctx->gbuf, "SIMILAR_TO"); - cbm_clock_gettime(CLOCK_MONOTONIC, &t); - cbm_pipeline_pass_similarity(ctx); - cbm_log_info("pass.timing", "pass", "incr_similarity", "elapsed_ms", - cbm_log_itoa((int)cbm_elapsed_ms(t))); - } else { - cbm_log_info("incremental.derived_gate", "component", "similarity", "decision", "reuse"); - } - - bool semantic_unchanged = input_unchanged && - derived_digest_equal(&before->semantic, &after.semantic) && - cbm_gbuf_reuse_zova_semantic_vectors(ctx->gbuf) == 0; - if (!semantic_unchanged) { - cbm_gbuf_delete_edges_by_type(ctx->gbuf, "SEMANTICALLY_RELATED"); - cbm_gbuf_clear_semantic_vectors(ctx->gbuf); - cbm_clock_gettime(CLOCK_MONOTONIC, &t); - cbm_pipeline_pass_semantic_edges(ctx); - cbm_log_info("pass.timing", "pass", "incr_semantic_edges", "elapsed_ms", - cbm_log_itoa((int)cbm_elapsed_ms(t))); - } else { - cbm_log_info("incremental.derived_gate", "component", "semantic", "decision", "reuse"); - } - } -} -/* Publish the merged graph and hashes to cbm.zova as a delta generation. - * Mode-skipped hash rows are preserved so subsequent reindexes can correctly - * distinguish "never indexed" from "indexed but not visited this pass". */ -static int dump_and_persist(cbm_pipeline_t *pipeline, cbm_gbuf_t *gbuf, cbm_file_info_t *files, - int file_count, const cbm_file_hash_t *mode_skipped, - int mode_skipped_count, cbm_zova_workspace_snapshot_t *before) { - /* #49: same post-extraction gates as the full path. Nothing is published - * until the delta commit, so an abort leaves the served generation. */ - cbm_pipeline_log_phase_mem("incr_pre_dump"); - int gate_rc = cbm_pipeline_budget_gate("before_incr_dump"); - if (gate_rc != 0) { - return gate_rc; - } - cbm_pipeline_run_before_dump_hook(gbuf); - if (cbm_gbuf_prepare_zova_dump_delta(gbuf) != 0) { - cbm_log_error("incremental.err", "phase", "zova_prepare"); - return CBM_NOT_FOUND; - } - cbm_pipeline_log_phase_mem("incr_zova_prepare"); - gate_rc = cbm_pipeline_budget_gate("before_incr_publish"); - if (gate_rc != 0) { - return gate_rc; - } - if (cbm_pipeline_publish_zova_user_database_delta(pipeline, gbuf, files, file_count, - mode_skipped, mode_skipped_count, - before) != 0) { - cbm_log_error("incremental.err", "phase", "zova_single_file"); - return CBM_NOT_FOUND; - } - cbm_pipeline_log_phase_mem("incr_zova_publish"); - return 0; -} - -/* ── Incremental pipeline entry point ────────────────────────────── */ - -static int copy_zova_snapshot_hashes(const char *project, - const cbm_zova_workspace_snapshot_t *snapshot, - cbm_file_hash_t **out_hashes, int *out_count) { - *out_hashes = NULL; - *out_count = 0; - if (snapshot->file_hash_count <= 0) { - return 0; - } - cbm_file_hash_t *hashes = calloc((size_t)snapshot->file_hash_count, sizeof(*hashes)); - if (!hashes) { - return CBM_NOT_FOUND; - } - for (int i = 0; i < snapshot->file_hash_count; i++) { - const cbm_zova_file_hash_input_t *source = &snapshot->file_hashes[i]; - hashes[i].project = strdup(project); - hashes[i].rel_path = strdup(source->file_path); - hashes[i].sha256 = strdup(source->content_hash); - hashes[i].mtime_ns = source->mtime_ns; - hashes[i].size = source->size_bytes; - if (!hashes[i].project || !hashes[i].rel_path || !hashes[i].sha256) { - cbm_store_free_file_hashes(hashes, snapshot->file_hash_count); - return CBM_NOT_FOUND; - } - } - *out_hashes = hashes; - *out_count = snapshot->file_hash_count; - return 0; -} - -static int run_incremental(cbm_pipeline_t *p, cbm_file_info_t *files, int file_count, - cbm_zova_workspace_snapshot_t *snapshot, cbm_gbuf_t *existing) { - struct timespec t0; - cbm_clock_gettime(CLOCK_MONOTONIC, &t0); - - const char *project = cbm_pipeline_project_name(p); - - cbm_file_hash_t *stored = NULL; - int stored_count = 0; - if (copy_zova_snapshot_hashes(project, snapshot, &stored, &stored_count) != 0) { - cbm_log_error("incremental.err", "msg", "load_zova_hashes_failed"); - cbm_gbuf_free(existing); - return CBM_NOT_FOUND; - } - - /* Classify files */ - int n_changed = 0; - int n_unchanged = 0; - bool *is_changed = - classify_files(files, file_count, stored, stored_count, &n_changed, &n_unchanged); - - /* Classify stored files absent from current discovery: truly-deleted - * (purge) vs mode-skipped (preserve nodes AND hash rows). */ - char **deleted = NULL; - cbm_file_hash_t *mode_skipped = NULL; - int mode_skipped_count = 0; - int deleted_count = - find_deleted_files(cbm_pipeline_repo_path(p), files, file_count, stored, stored_count, - &deleted, &mode_skipped, &mode_skipped_count); - - if (is_changed) { - int infra_added = - incr_expand_infra_changes(files, file_count, is_changed, deleted, deleted_count); - n_changed += infra_added; - n_unchanged -= infra_added; - - int client_added = - incr_expand_client_importer_changes(existing, files, file_count, is_changed, deleted, - deleted_count); - n_changed += client_added; - n_unchanged -= client_added; - } - - cbm_log_info("incremental.classify", "changed", cbm_log_itoa(n_changed), "unchanged", - cbm_log_itoa(n_unchanged), "deleted", cbm_log_itoa(deleted_count), "mode_skipped", - cbm_log_itoa(mode_skipped_count)); - - /* Fast path: nothing changed → skip. The committed generation is left - * untouched, which means existing hash rows (including for any - * mode-skipped files that were already preserved by an earlier run) - * remain intact. */ - if (n_changed == 0 && deleted_count == 0) { - cbm_log_info("incremental.noop", "reason", "no_changes"); - free(is_changed); - free(deleted); - free_mode_skipped(mode_skipped, mode_skipped_count); - cbm_store_free_file_hashes(stored, stored_count); - cbm_gbuf_free(existing); - return 0; - } - - /* Module and Folder spellings follow the stem collisions of the whole - * discovered set, as in a full run. */ - cbm_stem_marks_t stem_marks; - cbm_stem_marks_init(&stem_marks, project, files, file_count); - if (incr_stem_marks_changed(project, files, file_count, &stem_marks, stored, stored_count, - deleted, deleted_count)) { - cbm_log_info("incremental.declined", "reason", "stem_collision_changed"); - cbm_stem_marks_free(&stem_marks); - free(is_changed); - for (int i = 0; i < deleted_count; i++) { - free(deleted[i]); - } - free(deleted); - free_mode_skipped(mode_skipped, mode_skipped_count); - cbm_store_free_file_hashes(stored, stored_count); - cbm_gbuf_free(existing); - return CBM_PIPELINE_INCREMENTAL_DECLINED; - } - - cbm_store_free_file_hashes(stored, stored_count); - - /* Build list of changed files */ - cbm_file_info_t *changed_files = - (n_changed > 0) ? malloc((size_t)n_changed * sizeof(cbm_file_info_t)) : NULL; - int ci = 0; - for (int i = 0; i < file_count; i++) { - if (is_changed[i]) { - changed_files[ci++] = files[i]; - } - } - free(is_changed); - - cbm_log_info("incremental.reparse", "files", cbm_log_itoa(ci)); - - /* Step 1: the existing graph was exported from the committed generation */ - cbm_log_info("incremental.load_zova", "nodes", cbm_log_itoa(cbm_gbuf_node_count(existing)), - "edges", cbm_log_itoa(cbm_gbuf_edge_count(existing))); - - struct timespec t; - cbm_clock_gettime(CLOCK_MONOTONIC, &t); - cbm_incremental_derived_state_t derived_before = derived_state(existing); - cbm_log_info("incremental.derived_digest", "phase", "baseline", "elapsed_ms", - cbm_log_itoa((int)cbm_elapsed_ms(t))); - - /* Snapshot inbound cross-file edges into changed files BEFORE purging, so - * the cascade delete doesn't permanently drop edges whose source lives in - * an unchanged (never-re-parsed) file. Re-linked after re-resolution. */ - cbm_edge_capture_t edge_cap = {0}; - edge_cap.gbuf = existing; - { - CBMHashTable *changed_paths = cbm_ht_create(ci > 0 ? (size_t)ci * PAIR_LEN : CBM_SZ_64); - for (int i = 0; i < ci; i++) { - cbm_ht_set(changed_paths, changed_files[i].rel_path, &changed_files[i]); - } - for (int i = 0; i < deleted_count; i++) { - cbm_ht_set(changed_paths, deleted[i], deleted[i]); - } - edge_cap.changed_paths = changed_paths; - cbm_clock_gettime(CLOCK_MONOTONIC, &t); - cbm_gbuf_foreach_edge(existing, incr_capture_inbound_edge, &edge_cap); - edge_cap.changed_paths = NULL; - cbm_ht_free(changed_paths); /* keys borrowed from changed_files; not freed here */ - } - cbm_log_info("incremental.edge_snapshot", "captured", cbm_log_itoa(edge_cap.count), - "elapsed_ms", cbm_log_itoa((int)cbm_elapsed_ms(t))); - - /* Step 2: Purge stale nodes */ - cbm_clock_gettime(CLOCK_MONOTONIC, &t); - for (int i = 0; i < ci; i++) { - cbm_gbuf_delete_by_file(existing, changed_files[i].rel_path); - } - for (int i = 0; i < deleted_count; i++) { - cbm_gbuf_delete_by_file(existing, deleted[i]); - } - - int affected_count = ci + deleted_count; - const char **affected_paths = NULL; - if (affected_count > 0) { - affected_paths = calloc((size_t)affected_count, sizeof(*affected_paths)); - if (affected_paths) { - for (int i = 0; i < ci; i++) { - affected_paths[i] = changed_files[i].rel_path; - } - for (int i = 0; i < deleted_count; i++) { - affected_paths[ci + i] = deleted[i]; - } - } - } - if (affected_paths) { - cbm_gbuf_delete_empty_folders_under(existing, affected_paths, affected_count); - free(affected_paths); - } else if (affected_count > 0) { - cbm_log_warn("incremental.folder_prune_oom", "affected_paths", - cbm_log_itoa(affected_count)); - } - for (int i = 0; i < deleted_count; i++) { - free(deleted[i]); - } - free(deleted); - cbm_log_info("incremental.purge", "elapsed_ms", cbm_log_itoa((int)cbm_elapsed_ms(t))); - - /* Step 3-5: Registry + extract + resolve */ - cbm_registry_t *registry = cbm_registry_new(); - cbm_clock_gettime(CLOCK_MONOTONIC, &t); - cbm_gbuf_foreach_node(existing, registry_visitor, registry); - cbm_log_info("incremental.registry_seed", "symbols", cbm_log_itoa(cbm_registry_size(registry)), - "elapsed_ms", cbm_log_itoa((int)cbm_elapsed_ms(t))); - - /* Discovery exclusions (gitignore + skip dirs) captured by the run that - * routed here. Borrowed from the pipeline so the auxiliary repo walks - * (pkgmap via merge_pkg_entries, path aliases) skip excluded subtrees on - * incremental runs too — same borrow as the full path (#792/#804). */ - char **excluded_dirs = NULL; - int excluded_count = 0; - cbm_pipeline_get_excluded(p, &excluded_dirs, &excluded_count); - - cbm_path_alias_collection_t *path_aliases = - cbm_load_path_aliases_excluded(cbm_pipeline_repo_path(p), excluded_dirs, excluded_count); - - cbm_pipeline_ctx_t ctx = { - .project_name = project, - .repo_path = cbm_pipeline_repo_path(p), - .gbuf = existing, - .registry = registry, - .cancelled = cbm_pipeline_cancelled_ptr(p), - .pipeline = p, /* so passes can record per-file skips (Track B) */ - .mode = cbm_pipeline_get_mode(p), - .path_aliases = path_aliases, - .excluded_dirs = excluded_dirs, - .excluded_count = excluded_count, - .stem_marks = &stem_marks, - }; - - for (int i = 0; i < ci; i++) { - incr_add_changed_file_structure(existing, project, &stem_marks, &changed_files[i]); - } - - int extract_rc = run_extract_resolve(&ctx, changed_files, ci); - ctx.stem_marks = NULL; - cbm_stem_marks_free(&stem_marks); - /* #49: extract + resolve work sets are done; gate before the derived - * passes and publication so an explicitly over-budget run stops here. - * RSS includes the preloaded served graph. That is deliberate: a user who - * capped memory below the graph should be told indexing cannot proceed, - * while the detected RAM-fraction default never gates this path. */ - if (extract_rc == 0) { - extract_rc = cbm_pipeline_budget_gate("before_incr_passes"); - } - if (extract_rc != 0) { - incr_free_edge_capture(&edge_cap); - free(changed_files); - cbm_registry_free(registry); - cbm_path_alias_collection_free(path_aliases); - cbm_pkgmap_free(ctx.pkgmap); - ctx.pkgmap = NULL; - cbm_pipeline_set_pkgmap(NULL); - free_mode_skipped(mode_skipped, mode_skipped_count); - cbm_gbuf_free(existing); - return extract_rc; - } - /* Changed files got fresh File props above, so a stale coverage stamp - * cannot survive; unchanged files keep theirs from the generation. */ - cbm_pipeline_stamp_coverage(p, existing); - cbm_pipeline_pass_k8s(&ctx, changed_files, ci); - - incr_rebuild_snapshot_callers(&ctx, &edge_cap, files, file_count); - - /* Restore inbound cross-file edges before any derived pass reads the - * mixed graph. Semantic vectors and complexity both consume CALLS - * topology, so restoring only after those passes made their output differ - * from a clean full index even though the final SQLite edges matched. */ - cbm_clock_gettime(CLOCK_MONOTONIC, &t); - int relinked = incr_restore_inbound_edges(existing, &edge_cap); - cbm_log_info("incremental.edge_relink", "relinked", cbm_log_itoa(relinked), "captured", - cbm_log_itoa(edge_cap.count), "elapsed_ms", cbm_log_itoa((int)cbm_elapsed_ms(t))); - incr_free_edge_capture(&edge_cap); - - run_postpasses(&ctx, changed_files, ci, project, &derived_before); - - free(changed_files); - cbm_registry_free(registry); - cbm_path_alias_collection_free(path_aliases); - cbm_pkgmap_free(ctx.pkgmap); /* built by the parallel extract, if it ran */ - ctx.pkgmap = NULL; - cbm_pipeline_set_pkgmap(NULL); - - /* Step 7: Publish (preserves mode-skipped hash rows so the next reindex - * can correctly classify those files instead of seeing them as - * never-existed). */ - /* Record committed counts before dump_and_persist (whose dump frees the - * gbuf node index, zeroing the count) so the #334 plausibility gate also - * covers incremental reindexes, not just full ones. */ - cbm_pipeline_set_committed_counts(p, cbm_gbuf_node_count(existing), - cbm_gbuf_edge_count(existing)); - int persist_rc = dump_and_persist(p, existing, files, file_count, mode_skipped, - mode_skipped_count, snapshot); - free_mode_skipped(mode_skipped, mode_skipped_count); - cbm_gbuf_free(existing); - if (persist_rc != 0) { - return persist_rc; - } - - cbm_log_info("incremental.done", "elapsed_ms", cbm_log_itoa((int)cbm_elapsed_ms(t0))); - return 0; -} - -int cbm_pipeline_run_incremental_zova(cbm_pipeline_t *p, const char *zova_path, - cbm_file_info_t *files, int file_count, - cbm_zova_workspace_snapshot_t *snapshot, - cbm_gbuf_t *preloaded_graph) { - if (!snapshot || !preloaded_graph) { - return CBM_NOT_FOUND; - } - return run_incremental(p, files, file_count, snapshot, preloaded_graph); -} diff --git a/src/pipeline/pipeline_incremental.zig b/src/pipeline/pipeline_incremental.zig new file mode 100644 index 00000000..2448a027 --- /dev/null +++ b/src/pipeline/pipeline_incremental.zig @@ -0,0 +1,1598 @@ +//! pipeline_incremental.zig — Disk-based incremental re-indexing. +//! +//! Operates on the existing SQLite DB directly (not RAM-first graph buffer). +//! Compares file mtime+size against stored hashes to classify changed/unchanged. +//! Deletes changed files' nodes (edges cascade via ON DELETE CASCADE), +//! re-parses only changed files through passes into a temp graph buffer, +//! then merges new nodes/edges into the disk DB. Persists updated hashes. +//! +//! Called from pipeline.zig when a DB with stored hashes already exists. + +const std = @import("std"); +const c = @import("c"); +const interop = @import("../foundation/interop.zig"); +const util = @import("util.zig"); + +const CStr = util.CStr; +const MutStr = util.MutStr; +const ix = util.ix; +const streq = util.streq; +const ptrDiff = util.ptrDiff; +const HashTable = c.CBMHashTable; +const Gbuf = c.cbm_gbuf_t; +const Node = c.cbm_gbuf_node_t; +const Edge = c.cbm_gbuf_edge_t; +const Ctx = c.cbm_pipeline_ctx_t; +const FileInfo = c.cbm_file_info_t; +const FileHash = c.cbm_file_hash_t; +const StemMarks = c.cbm_stem_marks_t; +const Sha256 = c.cbm_sha256_backend_ctx; +const Timespec = c.struct_timespec; + +fn logError(msg: interop.Str, kvs: anytype) void { + interop.log(c.CBM_LOG_ERROR, msg, kvs); +} + +/// (size_t)n * 2 as a hash table capacity, which cbm_ht_create takes as uint32_t. +fn htCap(n: c_int) u32 { + return @truncate(@as(usize, @bitCast(@as(isize, n))) *% 2); +} + +fn itoa(v: c_int) CStr { + return c.cbm_log_itoa(v); +} + +fn now(t: *Timespec) void { + _ = c.cbm_clock_gettime(c.CLOCK_MONOTONIC, t); +} + +/// cbm_log_itoa of the milliseconds elapsed since `start`. +fn elapsedText(start: Timespec) CStr { + return c.cbm_log_itoa(@intFromFloat(c.cbm_elapsed_ms(start))); +} + +const MultisetDigest = extern struct { + xor_words: [4]u64 = @splat(0), + sum_words: [4]u64 = @splat(0), + count: u64 = 0, +}; + +const DerivedState = extern struct { + input: MultisetDigest = .{}, + config: MultisetDigest = .{}, + similarity: MultisetDigest = .{}, + semantic: MultisetDigest = .{}, +}; + +fn derivedHashText(hash: *Sha256, text: CStr) void { + const value: CStr = if (text != null) text else ""; + c.cbm_sha256_backend_update(hash, value, c.strlen(value)); + c.cbm_sha256_backend_update(hash, "\x00", 1); +} + +fn derivedDigestAdd(set: *MultisetDigest, hash: *Sha256) void { + var bytes: [c.CBM_SHA256_DIGEST_LEN]u8 = undefined; + var words: [4]u64 = undefined; + c.cbm_sha256_backend_final(hash, &bytes); + @memcpy(std.mem.asBytes(&words), bytes[0..@sizeOf([4]u64)]); + for (0..4) |i| { + set.xor_words[i] ^= words[i]; + set.sum_words[i] +%= words[i]; + } + set.count +%= 1; +} + +fn derivedDigestEqual(left: *const MultisetDigest, right: *const MultisetDigest) bool { + return left.count == right.count and + std.mem.eql(u64, &left.xor_words, &right.xor_words) and + std.mem.eql(u64, &left.sum_words, &right.sum_words); +} + +const DerivedScan = struct { + gbuf: ?*const Gbuf, + state: *DerivedState, +}; + +fn derivedNodeVisitor(node: [*c]const Node, userdata: ?*anyopaque) callconv(.c) void { + const scan: *DerivedScan = @ptrCast(@alignCast(userdata)); + var hash: Sha256 = undefined; + c.cbm_sha256_backend_init(&hash); + derivedHashText(&hash, "node"); + derivedHashText(&hash, node.*.label); + derivedHashText(&hash, node.*.name); + derivedHashText(&hash, node.*.qualified_name); + derivedHashText(&hash, node.*.file_path); + c.cbm_sha256_backend_update(&hash, &node.*.start_line, @sizeOf(@TypeOf(node.*.start_line))); + c.cbm_sha256_backend_update(&hash, &node.*.end_line, @sizeOf(@TypeOf(node.*.end_line))); + derivedHashText(&hash, node.*.properties_json); + derivedDigestAdd(&scan.state.input, &hash); +} + +fn derivedEdgeVisitor(edge: [*c]const Edge, userdata: ?*anyopaque) callconv(.c) void { + const scan: *DerivedScan = @ptrCast(@alignCast(userdata)); + var target = &scan.state.input; + if (streq(edge.*.type, "CONFIGURES")) { + target = &scan.state.config; + } else if (streq(edge.*.type, "SIMILAR_TO")) { + target = &scan.state.similarity; + } else if (streq(edge.*.type, "SEMANTICALLY_RELATED")) { + target = &scan.state.semantic; + } + const source: ?*const Node = c.cbm_gbuf_find_by_id(scan.gbuf, edge.*.source_id); + const destination: ?*const Node = c.cbm_gbuf_find_by_id(scan.gbuf, edge.*.target_id); + var hash: Sha256 = undefined; + c.cbm_sha256_backend_init(&hash); + derivedHashText(&hash, "edge"); + derivedHashText(&hash, if (source) |s| s.qualified_name else null); + derivedHashText(&hash, edge.*.type); + derivedHashText(&hash, if (destination) |d| d.qualified_name else null); + derivedHashText(&hash, edge.*.properties_json); + derivedDigestAdd(target, &hash); +} + +fn derivedState(gbuf: ?*const Gbuf) DerivedState { + var state: DerivedState = .{}; + var scan: DerivedScan = .{ .gbuf = gbuf, .state = &state }; + c.cbm_gbuf_foreach_node(gbuf, derivedNodeVisitor, &scan); + c.cbm_gbuf_foreach_edge(gbuf, derivedEdgeVisitor, &scan); + return state; +} + +// ── File classification ───────────────────────────────────────────── + +/// Classify discovered files against stored hashes using mtime+size. +/// Returns a boolean array: changed[i] = true if files[i] needs re-parsing. +/// Caller must free the returned array. +fn classifyFiles( + files: [*c]FileInfo, + file_count: c_int, + stored: [*c]FileHash, + stored_count: c_int, + out_changed: *c_int, + out_unchanged: *c_int, +) [*c]bool { + const changed = interop.callocN(bool, ix(file_count)); + if (changed == null) return null; + + var n_changed: c_int = 0; + var n_unchanged: c_int = 0; + + // Build lookup: rel_path -> stored hash + const ht = c.cbm_ht_create(if (stored_count > 0) htCap(stored_count) else c.CBM_SZ_64); + for (0..ix(stored_count)) |i| { + _ = c.cbm_ht_set(ht, stored[i].rel_path, util.at(stored, i)); + } + + for (0..ix(file_count)) |i| { + const h: ?*FileHash = @ptrCast(@alignCast(c.cbm_ht_get(ht, files[i].rel_path))); + if (h == null) { + // New file + changed[i] = true; + n_changed += 1; + continue; + } + + var st: c.struct_stat = undefined; + if (c.stat(files[i].path, &st) != 0) { + changed[i] = true; + n_changed += 1; + continue; + } + + if (c.cbm_stat_mtime_ns(&st) != h.?.mtime_ns or st.st_size != h.?.size) { + changed[i] = true; + n_changed += 1; + } else { + n_unchanged += 1; + } + } + + c.cbm_ht_free(ht); + out_changed.* = n_changed; + out_unchanged.* = n_unchanged; + return changed; +} + +/// Classify stored files that are absent from current discovery. Returns the +/// count of truly-deleted files (output via out_deleted) and ALSO collects +/// mode-skipped files into out_mode_skipped (caller frees both). +/// +/// A stored file is classified as: +/// - "deleted" — `stat()` returns ENOENT or ENOTDIR. Its nodes will +/// be purged and its hash row dropped. +/// - "mode-skipped" — `stat()` succeeds. The file exists on disk but the +/// current discovery pass didn't visit it (e.g. excluded +/// by FAST_SKIP_DIRS in fast/moderate mode). Its nodes +/// must be preserved AND its hash row must be carried +/// forward into the new DB so subsequent reindexes can +/// still see it as "known" rather than treating it as +/// new-or-deleted. +/// +/// Without this distinction, a fast-mode reindex after a full-mode index +/// would silently purge every file under `tools/`, `scripts/`, `bin/`, +/// `build/`, `docs/`, `__tests__/`, etc. — see task +/// claude-connectors/codebase-memory-index-repository-is-destructive-... +/// and the 2026-04-13 Skyline incident (packages/mcp/src/tools/ vanished +/// from a live graph mid-session). +/// +/// Mode-skipped hash preservation is the second half of the additive-merge +/// contract: dump_and_persist re-upserts these hash rows so the next reindex +/// can correctly detect a real on-disk deletion of a mode-skipped file (as +/// opposed to seeing it as "never existed" → noop → orphaned graph nodes). +/// +/// Fail-safe rules (preserve nodes on uncertainty): +/// - repo_path NULL → log error and preserve everything (return 0 +/// deletions, empty mode_skipped). The caller contract is that +/// repo_path is required; a NULL means a misconfigured pipeline, +/// not a deletion signal. +/// - snprintf truncation (combined path ≥ CBM_SZ_4K) → preserve. We can't +/// reliably stat a truncated path. Treat as mode-skipped. +/// - stat() errno != ENOENT/ENOTDIR (EACCES, EIO, ELOOP, transient NFS, +/// etc.) → preserve. The file may exist; we just can't see it right now. +/// Treat as mode-skipped. +/// +/// Note: we use stat() (not lstat()) on purpose. A symlink whose target was +/// deleted should be classified as deleted from the indexer's perspective +/// because the indexer follows symlinks during discovery — a stale symlink +/// has no source to parse. +fn findDeletedFiles( + repo_path: CStr, + files: [*c]FileInfo, + file_count: c_int, + stored: [*c]FileHash, + stored_count: c_int, + out_deleted: *[*c]MutStr, + out_mode_skipped: *[*c]FileHash, + out_mode_skipped_count: *c_int, +) c_int { + out_deleted.* = null; + out_mode_skipped.* = null; + out_mode_skipped_count.* = 0; + + if (repo_path == null) { + // Misconfigured pipeline. Preserve everything rather than risk + // silently re-introducing the destructive overwrite this function + // was rewritten to prevent. + logError("incremental.err", .{ "msg", "find_deleted_files_null_repo_path" }); + return 0; + } + + const current = c.cbm_ht_create(htCap(file_count)); + for (0..ix(file_count)) |i| { + _ = c.cbm_ht_set(current, files[i].rel_path, util.at(files, i)); + } + + var del_count: c_int = 0; + var del_cap: c_int = c.CBM_SZ_64; + var deleted = interop.mallocN(MutStr, ix(del_cap)); + if (deleted == null) { + logError("incremental.err", .{ "msg", "find_deleted_files_oom" }); + c.cbm_ht_free(current); + return 0; + } + + var ms_count: c_int = 0; + var ms_cap: c_int = c.CBM_SZ_64; + var mode_skipped = interop.mallocN(FileHash, ix(ms_cap)); + if (mode_skipped == null) { + logError("incremental.err", .{ "msg", "find_deleted_files_oom_ms" }); + c.free(@ptrCast(deleted)); + c.cbm_ht_free(current); + return 0; + } + + for (0..ix(stored_count)) |i| { + if (c.cbm_ht_get(current, stored[i].rel_path) != null) continue; // still visited by current pass + // Not in current discovery — check if it's truly deleted or just + // mode-skipped (excluded by FAST_SKIP_DIRS etc.). + var preserve = false; + var abs_path: [c.CBM_SZ_4K]u8 = undefined; + const n = c.snprintf(&abs_path, abs_path.len, "%s/%s", repo_path, stored[i].rel_path); + if (n < 0 or n >= @as(c_int, abs_path.len)) { + // Truncation or encoding error — can't reliably stat. Preserve. + interop.logWarn("incremental.path_truncated", .{ "rel_path", stored[i].rel_path }); + preserve = true; + } else { + var st: c.struct_stat = undefined; + if (c.stat(&abs_path, &st) == 0) { + // File exists on disk — mode-skipped, not deleted. + preserve = true; + } else { + const err = std.c._errno().*; + if (err != c.ENOENT and err != c.ENOTDIR) { + // Transient or permission error — fail safe by preserving. + // EACCES, EIO, ELOOP, ENAMETOOLONG, etc. + interop.logWarn("incremental.stat_uncertain", .{ + "rel_path", stored[i].rel_path, + "errno", itoa(std.c._errno().*), + }); + preserve = true; + } + } + } + + if (preserve) { + // Carry forward the existing hash row so subsequent reindexes + // can correctly classify this file. + if (ms_count >= ms_cap) { + ms_cap *= 2; + const tmp = interop.reallocN(FileHash, mode_skipped, ix(ms_cap)); + if (tmp == null) { + logError("incremental.err", .{ "msg", "find_deleted_files_realloc_oom_ms" }); + break; + } + mode_skipped = tmp; + } + const rp: MutStr = c.strdup(stored[i].rel_path); + const sh: MutStr = if (stored[i].sha256 != null) c.strdup(stored[i].sha256) else null; + if (rp == null or (stored[i].sha256 != null and sh == null)) { + // OOM mid-record. Drop this entry rather than persist a + // row with a NULL rel_path that would silently fail the + // NOT NULL constraint in upsert and reintroduce the + // orphaned-node bug. + logError("incremental.err", .{ + "msg", "find_deleted_files_strdup_oom", + "rel_path", stored[i].rel_path, + }); + c.free(rp); + c.free(sh); + break; + } + const ms = &mode_skipped[ix(ms_count)]; + ms.project = null; // unused by upsert API + ms.rel_path = rp; + ms.sha256 = sh; + ms.mtime_ns = stored[i].mtime_ns; + ms.size = stored[i].size; + ms_count += 1; + continue; + } + + // File is truly gone — record for purge. + if (del_count >= del_cap) { + del_cap *= 2; + const tmp = interop.reallocN(MutStr, deleted, ix(del_cap)); + if (tmp == null) { + logError("incremental.err", .{ "msg", "find_deleted_files_realloc_oom" }); + break; + } + deleted = tmp; + } + deleted[ix(del_count)] = c.strdup(stored[i].rel_path); + del_count += 1; + } + + c.cbm_ht_free(current); + out_deleted.* = deleted; + out_mode_skipped.* = mode_skipped; + out_mode_skipped_count.* = ms_count; + return del_count; +} + +/// Free a mode_skipped array allocated by findDeletedFiles. +fn freeModeSkipped(ms: [*c]FileHash, count: c_int) void { + if (ms == null) return; + for (0..ix(count)) |i| { + c.free(@constCast(ms[i].rel_path)); + c.free(@constCast(ms[i].sha256)); + } + c.free(ms); +} + +/// Cuts a path at its last '/', or empties it when it has none. +fn cutLastSegment(path: MutStr) void { + const slash: MutStr = c.strrchr(path, '/'); + if (slash != null) slash[0] = 0 else path[0] = 0; +} + +/// Add the structural nodes for a newly discovered path before definition +/// extraction. This mirrors the relevant part of the full structure pass: a +/// Folder must exist before a directory-based Module definition arrives, or +/// the Module wins the shared qualified-name slot and the graph diverges. +fn incrAddChangedFileStructure(gbuf: ?*Gbuf, project: CStr, marks: ?*const StemMarks, file: ?*const FileInfo) void { + const f = file orelse return; + if (gbuf == null or project == null or f.rel_path == null) return; + const rel_path: CStr = f.rel_path; + const slash: CStr = c.strrchr(rel_path, '/'); + const basename: CStr = if (slash != null) slash + 1 else rel_path; + const extension: CStr = c.strrchr(basename, '.'); + var properties: [c.CBM_SZ_256]u8 = undefined; + _ = c.snprintf( + &properties, + properties.len, + "{\"extension\":\"%s\"}", + @as(CStr, if (extension != null) extension else ""), + ); + const file_qn: MutStr = c.cbm_pipeline_fqn_compute(project, rel_path, "__file__"); + if (file_qn == null) return; + _ = c.cbm_gbuf_upsert_node(gbuf, "File", basename, file_qn, rel_path, 0, 0, &properties); + + var branches: [*c][*c]const Node = null; + var branch_count: c_int = 0; + var branch_qn: CStr = project; + if (c.cbm_gbuf_find_by_label(gbuf, "Branch", &branches, &branch_count) == 0 and + branch_count > 0 and branches[0].*.qualified_name != null) + { + branch_qn = branches[0].*.qualified_name; + } + + const dir: MutStr = c.strdup(rel_path); + if (dir == null) { + c.free(file_qn); + return; + } + cutLastSegment(dir); + var file_parent_buf: [c.CBM_SZ_8K]u8 = undefined; + const file_parent_qn: CStr = if (dir[0] != 0) + c.cbm_stem_marks_folder_qn(marks, project, dir, &file_parent_buf, file_parent_buf.len) + else + null; + + const walk = dir; + while (walk[0] != 0) { + var folder_buf: [c.CBM_SZ_8K]u8 = undefined; + const folder_qn: CStr = c.cbm_stem_marks_folder_qn(marks, project, walk, &folder_buf, folder_buf.len); + if (folder_qn == null) break; + var folder_name: CStr = c.strrchr(walk, '/'); + folder_name = if (folder_name != null) folder_name + 1 else walk; + _ = c.cbm_gbuf_upsert_node(gbuf, "Folder", folder_name, folder_qn, walk, 0, 0, "{}"); + + const parent_dir: MutStr = c.strdup(walk); + if (parent_dir == null) break; + cutLastSegment(parent_dir); + var parent_buf: [c.CBM_SZ_8K]u8 = undefined; + const parent_qn: CStr = if (parent_dir[0] != 0) + c.cbm_stem_marks_folder_qn(marks, project, parent_dir, &parent_buf, parent_buf.len) + else + branch_qn; + const folder: ?*const Node = c.cbm_gbuf_find_by_qn(gbuf, folder_qn); + const parent: ?*const Node = c.cbm_gbuf_find_by_qn(gbuf, parent_qn); + if (folder != null and parent != null) { + _ = c.cbm_gbuf_insert_edge(gbuf, parent.?.id, folder.?.id, "CONTAINS_FOLDER", "{}"); + } + c.free(parent_dir); + + cutLastSegment(walk); + } + + const file_node: ?*const Node = c.cbm_gbuf_find_by_qn(gbuf, file_qn); + const parent_qn: CStr = if (file_parent_qn != null) file_parent_qn else branch_qn; + const parent: ?*const Node = c.cbm_gbuf_find_by_qn(gbuf, parent_qn); + if (file_node != null and parent != null) { + _ = c.cbm_gbuf_insert_edge(gbuf, parent.?.id, file_node.?.id, "CONTAINS_FILE", "{}"); + } + c.free(dir); + c.free(file_qn); +} + +/// True when the stem QN or a directory QN of rel yields on a different side +/// (CBM_STEM_*) in the two file sets. +fn incrStemSideDiffers(now_marks: *const StemMarks, before: *const StemMarks, a: *c.CBMArena, project: CStr, rel: CStr) bool { + const stem: MutStr = c.cbm_fqn_module(a, project, rel); + if (stem != null and c.cbm_stem_marks_side(now_marks, stem) != c.cbm_stem_marks_side(before, stem)) { + return true; + } + var s: CStr = c.strchr(rel, '/'); + while (s != null) : (s = c.strchr(s + 1, '/')) { + const dir: MutStr = c.cbm_arena_strndup(a, rel, ptrDiff(s, rel)); + const fqn: MutStr = if (dir != null) c.cbm_fqn_folder(a, project, dir) else null; + if (fqn != null and c.cbm_stem_marks_side(now_marks, fqn) != c.cbm_stem_marks_side(before, fqn)) { + return true; + } + } + return false; +} + +/// The file set of the stored generation: the discovered files it already +/// held, plus the deleted ones, which take the language their name gives. +/// Counts the discovered files it did not hold in *added. +fn incrPreviousFiles( + files: [*c]const FileInfo, + file_count: c_int, + stored_paths: ?*HashTable, + deleted: [*c]MutStr, + deleted_count: c_int, + out_count: *c_int, + added: *c_int, +) [*c]FileInfo { + const before = interop.callocN(FileInfo, ix(file_count) + ix(deleted_count) + 1); + var n: usize = 0; + for (0..ix(file_count)) |i| { + if (files[i].rel_path != null and c.cbm_ht_get(stored_paths, files[i].rel_path) != null) { + if (before != null) { + before[n] = files[i]; + n += 1; + } + } else { + added.* += 1; + } + } + if (before != null) for (0..ix(deleted_count)) |i| { + const base: CStr = c.strrchr(deleted[i], '/'); + before[n] = std.mem.zeroes(FileInfo); + before[n].rel_path = deleted[i]; + before[n].language = c.cbm_language_for_filename(if (base != null) base + 1 else deleted[i]); + n += 1; + }; + out_count.* = @intCast(n); + return before; +} + +/// Whether an added or deleted file yields on a different side of a stem +/// collision in `now` than in `prev`. +fn incrStemSidesMoved( + project: CStr, + files: [*c]const FileInfo, + file_count: c_int, + stored_paths: ?*HashTable, + now_marks: *const StemMarks, + prev: *const StemMarks, + deleted: [*c]MutStr, + deleted_count: c_int, +) bool { + var a: c.CBMArena = undefined; + c.cbm_arena_init(&a); + var moved = false; + for (0..ix(file_count)) |i| { + if (moved) break; + if (files[i].rel_path != null and c.cbm_ht_get(stored_paths, files[i].rel_path) == null) { + moved = incrStemSideDiffers(now_marks, prev, &a, project, files[i].rel_path); + } + } + for (0..ix(deleted_count)) |i| { + if (moved) break; + moved = incrStemSideDiffers(now_marks, prev, &a, project, deleted[i]); + } + c.cbm_arena_destroy(&a); + return moved; +} + +/// Whether the added and deleted files change a stem collision +/// (cbm_stem_marks_t): one appears or disappears, or the yielding side flips +/// because a Java/Go package gains its first or loses its last file. The +/// unchanged files keep the spelling of the stored generation, so only a full +/// index re-spells them. `now` holds the discovered files; the previous set is +/// the stored paths, where a deleted file takes the language its name gives. +/// A changed file can only move the sides of its own stem QN and of the +/// directories above it, so only those are compared. +fn incrStemMarksChanged( + project: CStr, + files: [*c]const FileInfo, + file_count: c_int, + now_marks: *const StemMarks, + stored: [*c]FileHash, + stored_count: c_int, + deleted: [*c]MutStr, + deleted_count: c_int, +) bool { + const stored_paths = c.cbm_ht_create(if (stored_count > 0) htCap(stored_count) else c.CBM_SZ_64); + for (0..ix(stored_count)) |i| { + if (stored[i].rel_path != null) _ = c.cbm_ht_set(stored_paths, stored[i].rel_path, util.at(stored, i)); + } + var before_count: c_int = 0; + var added: c_int = 0; + const before = incrPreviousFiles(files, file_count, stored_paths, deleted, deleted_count, &before_count, &added); + var changed = false; + if (added > 0 or deleted_count > 0) { + // Without the previous set the comparison cannot be made: rebuild. + changed = true; + if (before != null) { + var prev: StemMarks = undefined; + c.cbm_stem_marks_init(&prev, project, before, before_count); + changed = incrStemSidesMoved(project, files, file_count, stored_paths, now_marks, &prev, deleted, deleted_count); + c.cbm_stem_marks_free(&prev); + } + } + c.free(before); + c.cbm_ht_free(stored_paths); + return changed; +} + +// ── Inbound cross-file edge preservation (incremental correctness) ── +// +// The purge step (cbm_gbuf_delete_by_file) removes a changed file's nodes, +// and the cascade then drops every edge referencing them — INCLUDING inbound +// edges whose source lives in an UNCHANGED file (e.g. StudyService.grade -> +// SM2.review, or a Folder -> File containment edge). Because incremental only +// re-parses the changed files, the resolution passes never regenerate those +// inbound edges, so the graph silently loses cross-file CALLS / USAGE / +// CONTAINS_FILE / INHERITS / ... edges on every edit and diverges from a +// clean full reindex (which resolves every file). +// +// Fix: snapshot inbound cross-file edges into changed or deleted files BEFORE +// the purge. Re-run CALLS resolution for unchanged callers after re-extraction, +// then re-link non-CALLS edges by exact qualified-name target. Notes: +// - Only edges whose target is in a changed file and whose source is NOT +// are snapshotted; edges out of a changed file are regenerated when that +// file is re-resolved. +// - Edge types recomputed wholesale by post-passes (SIMILAR_TO, +// SEMANTICALLY_RELATED) are skipped — re-linking a stale snapshot could +// add edges a full reindex would not produce. +// - cbm_gbuf_insert_edge dedups, so re-linking an edge the resolver already +// recreated is a harmless no-op. +// - A deleted target is dropped. A renamed CALLS edge is rebuilt from its +// unchanged caller source, preserving the resolver's current confidence +// and strategy rather than copying stale edge properties. + +const SavedEdge = extern struct { + source_qn: MutStr, + target_qn: MutStr, + target_name: MutStr, + target_label: MutStr, + type: MutStr, + props: MutStr, +}; + +const EdgeCapture = struct { + gbuf: ?*Gbuf, + /// rel_path -> non-NULL sentinel (membership set) + changed_paths: ?*HashTable, + items: [*c]SavedEdge = null, + count: c_int = 0, + cap: c_int = 0, +}; + +/// Edge types that must NOT be re-linked from the pre-purge snapshot, because a +/// full reindex (re)computes them via a pass whose result can differ from the +/// snapshot — restoring a stale copy could leave wrong properties or even an +/// edge a full reindex would not produce: +/// - SIMILAR_TO / SEMANTICALLY_RELATED / FILE_CHANGES_WITH / DATA_FLOWS: +/// rebuilt wholesale by the incremental post-passes (similarity / +/// semantic_edges / githistory / route matching) over a drifting corpus. +/// Structural containment edges are regenerated from changed paths before +/// extraction, so snapshots omit them rather than retaining a pre-rename +/// parent relationship. +fn incrEdgeTypeIsRecomputed(edge_type: CStr) bool { + return edge_type != null and (streq(edge_type, "SIMILAR_TO") or + streq(edge_type, "SEMANTICALLY_RELATED") or streq(edge_type, "FILE_CHANGES_WITH") or + streq(edge_type, "DATA_FLOWS") or streq(edge_type, "CONTAINS_FILE") or + streq(edge_type, "CONTAINS_FOLDER")); +} + +/// cbm_gbuf_foreach_edge visitor: snapshot inbound cross-file edges into +/// changed files so they survive the purge and can be re-linked afterward. +fn incrCaptureInboundEdge(edge: [*c]const Edge, userdata: ?*anyopaque) callconv(.c) void { + const cap: *EdgeCapture = @ptrCast(@alignCast(userdata)); + if (incrEdgeTypeIsRecomputed(edge.*.type)) return; + const src: *const Node = c.cbm_gbuf_find_by_id(cap.gbuf, edge.*.source_id) orelse return; + const tgt: *const Node = c.cbm_gbuf_find_by_id(cap.gbuf, edge.*.target_id) orelse return; + if (src.qualified_name == null or tgt.qualified_name == null or src.file_path == null or + tgt.file_path == null) + { + return; + } + // Keep only edges that the purge would orphan permanently: target is in a + // changed file (its node is deleted + re-created), source is NOT (its file + // is never re-parsed, so the resolver won't regenerate the edge). + if (c.cbm_ht_get(cap.changed_paths, tgt.file_path) == null or + c.cbm_ht_get(cap.changed_paths, src.file_path) != null) + { + return; + } + if (cap.count >= cap.cap) { + const ncap: c_int = if (cap.cap > 0) cap.cap * 2 else c.CBM_SZ_64; + const tmp = interop.reallocN(SavedEdge, cap.items, ix(ncap)); + if (tmp == null) { + interop.logWarn("incremental.edge_snapshot_oom", .{ "captured", itoa(cap.count) }); + return; // best-effort: stop capturing, keep what we have + } + cap.items = tmp; + cap.cap = ncap; + } + const s = util.at(cap.items, ix(cap.count)); + s.source_qn = c.strdup(src.qualified_name); + s.target_qn = c.strdup(tgt.qualified_name); + s.target_name = c.strdup(if (tgt.name != null) tgt.name else ""); + s.target_label = c.strdup(if (tgt.label != null) tgt.label else ""); + s.type = c.strdup(edge.*.type); + s.props = c.strdup(if (edge.*.properties_json != null) edge.*.properties_json else "{}"); + if (s.source_qn == null or s.target_qn == null or s.target_name == null or + s.target_label == null or s.type == null or s.props == null) + { + c.free(s.source_qn); + c.free(s.target_qn); + c.free(s.target_name); + c.free(s.target_label); + c.free(s.type); + c.free(s.props); + return; + } + cap.count += 1; +} + +fn incrFindUniqueRenamedTarget(gbuf: ?*const Gbuf, saved: *const SavedEdge) ?*const Node { + if (saved.target_name == null or saved.target_name[0] == 0 or saved.target_label == null or + saved.target_label[0] == 0) + { + return null; + } + var nodes: [*c][*c]const Node = null; + var count: c_int = 0; + if (c.cbm_gbuf_find_by_name(gbuf, saved.target_name, &nodes, &count) != 0) return null; + var candidate: ?*const Node = null; + for (0..ix(count)) |i| { + const node: *const Node = nodes[i]; + if (node.qualified_name == null or node.label == null or + !streq(node.label, saved.target_label)) + { + continue; + } + if (candidate != null) return null; // ambiguous: do not guess a rename target + candidate = node; + } + return candidate; +} + +fn incrCmpId(a: ?*const anyopaque, b: ?*const anyopaque) callconv(.c) c_int { + const x = @as(*const i64, @ptrCast(@alignCast(a))).*; + const y = @as(*const i64, @ptrCast(@alignCast(b))).*; + return @as(c_int, @intFromBool(x > y)) - @as(c_int, @intFromBool(x < y)); +} + +const IncrIdSet = extern struct { + ids: [*c]const i64, + count: c_int, +}; + +fn incrKeepUnlessSourceInSet(gb: ?*const Gbuf, edge: [*c]const Edge, userdata: ?*anyopaque) callconv(.c) bool { + _ = gb; + const set: *const IncrIdSet = @ptrCast(@alignCast(userdata)); + if (edge.*.properties_json != null and + c.strstr(edge.*.properties_json, "\"env_access\"") != null) + { + return true; + } + return c.bsearch(&edge.*.source_id, set.ids, ix(set.count), @sizeOf(i64), incrCmpId) == null; +} + +/// A path rename changes the callee's qualified name. Re-run the CALLS pass +/// for only unchanged source files that previously called a purged target, so +/// the normal resolver recalculates strategy/confidence against the new +/// registry instead of preserving stale edge properties. +fn incrRebuildSnapshotCallers(ctx: *Ctx, cap: *const EdgeCapture, files: [*c]const FileInfo, file_count: c_int) void { + if (files == null or file_count <= 0) return; + const callers = interop.callocN(FileInfo, ix(cap.count)); + const source_ids = interop.mallocN(i64, ix(cap.count)); + if (callers == null or source_ids == null) { + c.free(callers); + c.free(source_ids); + return; + } + var caller_count: usize = 0; + var source_count: usize = 0; + for (0..ix(cap.count)) |i| { + const saved = util.at(cap.items, i); + // Re-resolve unchanged sources that previously emitted either CALLS + // or resolver-owned CONFIGURES. CONFIGURES may be the only derived + // edge from a source (for example an environment accessor), so + // restricting this list to CALLS leaves those edges missing after a + // changed-file purge. + if (saved.type == null or (!streq(saved.type, "CALLS") and !streq(saved.type, "CONFIGURES"))) { + continue; + } + const source: *const Node = c.cbm_gbuf_find_by_qn(ctx.gbuf, saved.source_qn) orelse continue; + if (source.file_path == null) continue; + source_ids[source_count] = source.id; + source_count += 1; + for (0..ix(file_count)) |f| { + if (!streq(files[f].rel_path, source.file_path)) continue; + var already_added = false; + for (0..caller_count) |existing| { + if (streq(callers[existing].rel_path, files[f].rel_path)) { + already_added = true; + break; + } + } + if (!already_added) { + callers[caller_count] = files[f]; + caller_count += 1; + } + break; + } + } + // Drop these sources' CONFIGURES edges in one pass over the edge list: + // deleting per captured edge rescanned every edge and rebuilt the edge + // indexes each time, which made a one-file reindex slower than a full one. + if (source_count > 0) { + c.qsort(source_ids, source_count, @sizeOf(i64), incrCmpId); + var set: IncrIdSet = .{ .ids = source_ids, .count = @intCast(source_count) }; + _ = c.cbm_gbuf_delete_edges_by_type_unless(ctx.gbuf, "CONFIGURES", incrKeepUnlessSourceInSet, &set); + } + if (caller_count > 0) { + _ = c.cbm_parallel_resolve_calls(ctx, callers, @intCast(caller_count)); + } + c.free(source_ids); + c.free(callers); +} + +/// Re-link snapshotted inbound edges to the freshly re-created target nodes. +/// Returns the number of edges re-linked. +fn incrRestoreInboundEdges(gbuf: ?*Gbuf, cap: *EdgeCapture) c_int { + var restored: c_int = 0; + for (0..ix(cap.count)) |i| { + const s = util.at(cap.items, i); + const src: ?*const Node = c.cbm_gbuf_find_by_qn(gbuf, s.source_qn); + var tgt: ?*const Node = c.cbm_gbuf_find_by_qn(gbuf, s.target_qn); + if (tgt == null and s.type != null and streq(s.type, "CALLS")) { + continue; // rebuilt from source so resolver owns its properties + } + // Incremental runs skip cross-file LSP, so an unchanged source cannot + // re-prove its CALL_REFERENCE. Keep the snapshot edge only while the + // same QN is still a callable; never guess a rename. + if (s.type != null and streq(s.type, "CALL_REFERENCE") and + !c.cbm_pipeline_node_is_callable_target(tgt)) + { + continue; + } + if (tgt == null) tgt = incrFindUniqueRenamedTarget(gbuf, s); + if (src != null and tgt != null) { + _ = c.cbm_gbuf_insert_edge(gbuf, src.?.id, tgt.?.id, s.type, s.props); + restored += 1; + } + } + return restored; +} + +fn incrFreeEdgeCapture(cap: *EdgeCapture) void { + for (0..ix(cap.count)) |i| { + c.free(cap.items[i].source_qn); + c.free(cap.items[i].target_qn); + c.free(cap.items[i].target_name); + c.free(cap.items[i].target_label); + c.free(cap.items[i].type); + c.free(cap.items[i].props); + } + c.free(cap.items); + cap.items = null; + cap.count = 0; + cap.cap = 0; +} + +// ── Registry seed visitor ─────────────────────────────────────────── + +/// Labels the full index seeds into the registry +/// (pass_parallel register_and_link_def — KEEP IN SYNC). Incremental re-resolution must see the +/// SAME symbol set, or it diverges from a clean full reindex: seeding extra +/// container nodes (File / Module / Folder / ...) lets a type usage like `Word` +/// resolve to the same-named Module node instead of the Class node. Only +/// callable / declared symbols belong in the registry. +fn incrLabelIsRegistrySymbol(label: CStr) bool { + // Mirror pass_parallel registry seeding EXACTLY: + // callables + every type-like container (Class/Struct/Interface/Enum/Type/ + // Trait) + Variable/Field. Struct included so an incremental re-resolve seeds + // the same struct type nodes a full reindex would. + return label != null and (streq(label, "Function") or streq(label, "Method") or + c.cbm_label_is_type_like(label) or streq(label, "Variable") or streq(label, "Field")); +} + +/// Callback for cbm_gbuf_foreach_node: seed the registry with the existing +/// project's definition symbols so the resolver can match cross-file symbols +/// during incremental. Mirrors the full-index registry contents exactly so an +/// incremental re-resolve picks the same nodes a full reindex would. +fn registryVisitor(node: [*c]const Node, userdata: ?*anyopaque) callconv(.c) void { + const r: ?*c.cbm_registry_t = @ptrCast(@alignCast(userdata)); + if (!incrLabelIsRegistrySymbol(node.*.label)) return; + c.cbm_registry_add(r, node.*.name, node.*.qualified_name, node.*.label); +} + +fn freeResultCache(cache: [*c][*c]c.CBMFileResult, count: c_int) void { + for (0..ix(count)) |j| { + if (cache[j] != null) c.cbm_free_result(cache[j]); + } + c.free(@ptrCast(cache)); +} + +/// Extract + resolve the changed files. +fn runExtractResolve(ctx: *Ctx, changed_files: [*c]FileInfo, ci: c_int) c_int { + var t: Timespec = undefined; + + // Per-file LSP always runs (every mode). Cross-file LSP stays disabled in + // incremental regardless (cbm_parallel_resolve is called with NULL + // cross_registries below). + const worker_count = c.cbm_default_worker_count(true); + interop.logInfo("incremental.mode", .{ + "mode", "parallel", + "workers", itoa(worker_count), + "changed", itoa(ci), + }); + + var shared_ids: i64 = c.cbm_gbuf_next_id(ctx.gbuf); + + const cache: [*c][*c]c.CBMFileResult = + @ptrCast(@alignCast(c.calloc(@bitCast(@as(isize, ci)), @sizeOf([*c]c.CBMFileResult)))); + if (cache != null) { + now(&t); + const rc = c.cbm_parallel_extract(ctx, changed_files, ci, cache, &shared_ids, worker_count); + if (rc == c.CBM_PIPELINE_ABORT_OVER_BUDGET) { + // Over the memory budget: stop before publication so the + // served generation stays as it was. + freeResultCache(cache, ci); + return rc; + } + c.cbm_pipeline_sync_next_id(ctx.gbuf, &shared_ids); + interop.logInfo("pass.timing", .{ "pass", "incr_extract", "elapsed_ms", elapsedText(t) }); + + now(&t); + _ = c.cbm_build_registry_from_cache(ctx, changed_files, ci, cache); + c.cbm_pipeline_share_next_id(ctx.gbuf, &shared_ids); + interop.logInfo("pass.timing", .{ "pass", "incr_registry", "elapsed_ms", elapsedText(t) }); + + // Incremental skips cross-file LSP precondition build — it + // would need all_defs from the full project, not just the + // changed slice. Per-file LSP (run inside cbm_extract_file) + // still fires; cross-file resolution is deferred to the + // next full re-index. Pass NULL/0/NULL to make the fused + // step in resolve_worker a no-op. + now(&t); + _ = c.cbm_parallel_resolve( + ctx, + changed_files, + ci, + cache, + &shared_ids, + worker_count, + null, + 0, + null, + null, // module_def_index + null, // cross_registries — incremental skips Tier 2 prebuild + ); + c.cbm_pipeline_sync_next_id(ctx.gbuf, &shared_ids); + interop.logInfo("pass.timing", .{ "pass", "incr_resolve", "elapsed_ms", elapsedText(t) }); + + c.cbm_pipeline_extract_infra_routes(ctx.gbuf, changed_files, cache, null, ci); + c.cbm_pipeline_process_infra_bindings(ctx.gbuf, changed_files, cache, null, ci); + freeResultCache(cache, ci); + } + return 0; +} + +/// Infra Route nodes are keyed by URL across config files: a URL becomes a +/// Route only when no config file lists it under a denied key, and the node +/// belongs to one of the files that list it. A change to one config file can +/// therefore change what another contributes, so when any config file changed +/// or was deleted, every config file is re-extracted. Returns files added. +fn incrExpandInfraChanges( + files: [*c]const FileInfo, + file_count: c_int, + is_changed: [*c]bool, + deleted: [*c]const MutStr, + deleted_count: c_int, +) c_int { + var infra_touched = false; + for (0..ix(file_count)) |i| { + if (infra_touched) break; + infra_touched = is_changed[i] and c.cbm_pipeline_is_infra_file(files[i].rel_path); + } + for (0..ix(deleted_count)) |i| { + if (infra_touched) break; + infra_touched = c.cbm_pipeline_is_infra_file(deleted[i]); + } + if (!infra_touched) return 0; + var added: c_int = 0; + for (0..ix(file_count)) |i| { + if (!is_changed[i] and c.cbm_pipeline_is_infra_file(files[i].rel_path)) { + is_changed[i] = true; + added += 1; + } + } + return added; +} + +const IncrClientScanCtx = struct { + changed_paths: ?*const HashTable, + client_files: ?*HashTable, +}; + +fn incrFindExistingClientNodes(n_c: [*c]const Node, userdata: ?*anyopaque) callconv(.c) void { + const ctx: *IncrClientScanCtx = @ptrCast(@alignCast(userdata)); + const n: *const Node = n_c orelse return; + if (n.file_path == null or n.label == null or n.properties_json == null) return; + if (!streq(n.label, "Module") and !streq(n.label, "Variable")) return; + if (c.cbm_ht_get(ctx.changed_paths, n.file_path) == null) return; + if (c.strstr(n.properties_json, "\"http_client\":") != null) { + if (c.cbm_ht_get(ctx.client_files, n.file_path) == null) { + const k: MutStr = c.strdup(n.file_path); + if (k != null) _ = c.cbm_ht_set(ctx.client_files, k, k); + } + } +} + +fn incrIsClientFile(fi: *const FileInfo) bool { + if (fi.path == null) return false; + if (fi.language != c.CBM_LANG_JAVASCRIPT and fi.language != c.CBM_LANG_TYPESCRIPT and + fi.language != c.CBM_LANG_TSX) + { + return false; + } + const f = c.cbm_fopen(fi.path, "rb") orelse return false; + _ = c.fseek(f, 0, c.SEEK_END); + const size = c.ftell(f); + _ = c.fseek(f, 0, c.SEEK_SET); + if (size <= 0 or size > c.cbm_max_file_bytes()) { + _ = c.fclose(f); + return false; + } + const buf: MutStr = @ptrCast(c.malloc(@as(usize, @intCast(size)) + 1)); + if (buf == null) { + _ = c.fclose(f); + return false; + } + const r = c.fread(buf, 1, @intCast(size), f); + _ = c.fclose(f); + buf[r] = 0; + + if (c.strstr(buf, "axios.create") == null) { + c.free(buf); + return false; + } + + const fr: ?*c.CBMFileResult = + c.cbm_extract_file(buf, @intCast(r), fi.language, "incr_probe", fi.rel_path, 0, null, null); + c.free(buf); + var has_client = false; + if (fr) |result| { + for (0..ix(result.defs.count)) |i| { + if (result.defs.items[i].http_client != null) { + has_client = true; + break; + } + } + c.cbm_free_result(result); + } + return has_client; +} + +fn freeHtKeysCb(key: CStr, value: ?*anyopaque, userdata: ?*anyopaque) callconv(.c) void { + _ = value; + _ = userdata; + c.free(@constCast(key)); +} + +fn freeStringHashTable(ht: ?*HashTable) void { + if (ht == null) return; + c.cbm_ht_foreach(ht, freeHtKeysCb, null); + c.cbm_ht_free(ht); +} + +/// An axios client instance's baseURL is composed into the HTTP_CALLS edges of +/// files that import it. When any file that defines an http_client changes or was +/// deleted (before or after the edit), treat the files that import it as changed +/// files so they go through full re-extraction. Returns files added. (#75, #1916) +fn incrExpandClientImporterChanges( + existing: ?*const Gbuf, + files: [*c]const FileInfo, + file_count: c_int, + is_changed: [*c]bool, + deleted: [*c]const MutStr, + deleted_count: c_int, +) c_int { + if (existing == null or files == null or file_count <= 0 or is_changed == null) return 0; + const changed_set = c.cbm_ht_create(@as(u32, @bitCast(file_count + deleted_count))); + if (changed_set == null) return 0; + for (0..ix(file_count)) |i| { + if (is_changed[i]) _ = c.cbm_ht_set(changed_set, files[i].rel_path, @ptrFromInt(1)); + } + for (0..ix(deleted_count)) |i| { + _ = c.cbm_ht_set(changed_set, deleted[i], @ptrFromInt(1)); + } + + const client_files = c.cbm_ht_create(16); + if (client_files == null) { + c.cbm_ht_free(changed_set); + return 0; + } + + // 1. Check existing graph for files that defined an http_client before the edit + var scan_ctx: IncrClientScanCtx = .{ + .changed_paths = changed_set, + .client_files = client_files, + }; + c.cbm_gbuf_foreach_node(existing, incrFindExistingClientNodes, &scan_ctx); + + // 2. Check changed files on disk to detect newly added client instances + for (0..ix(file_count)) |i| { + if (!is_changed[i]) continue; + if (c.cbm_ht_get(client_files, files[i].rel_path) != null) continue; + if (incrIsClientFile(util.at(files, i))) { + const k: MutStr = c.strdup(files[i].rel_path); + if (k != null) _ = c.cbm_ht_set(client_files, k, k); + } + } + c.cbm_ht_free(changed_set); + + if (c.cbm_ht_count(client_files) == 0) { + freeStringHashTable(client_files); + return 0; + } + + // 3. Find files in existing graph buffer that import any client file + var edges: [*c][*c]const Edge = null; + var edge_count: c_int = 0; + var added: c_int = 0; + if (c.cbm_gbuf_find_edges_by_type(existing, "IMPORTS", &edges, &edge_count) == 0 and edges != null) { + for (0..ix(edge_count)) |e| { + const edge: *const Edge = edges[e]; + const tgt: *const Node = c.cbm_gbuf_find_by_id(existing, edge.target_id) orelse continue; + if (tgt.file_path == null or c.cbm_ht_get(client_files, tgt.file_path) == null) continue; + const src: *const Node = c.cbm_gbuf_find_by_id(existing, edge.source_id) orelse continue; + if (src.file_path == null) continue; + for (0..ix(file_count)) |f| { + if (streq(files[f].rel_path, src.file_path)) { + if (!is_changed[f]) { + is_changed[f] = true; + added += 1; + } + break; + } + } + } + } + freeStringHashTable(client_files); + return added; +} + +/// HANDLES edges into infra Routes come only from route matching. +fn incrHandlesEdgeIsNotInfraMatch(gbuf: ?*const Gbuf, edge: [*c]const Edge, userdata: ?*anyopaque) callconv(.c) bool { + _ = userdata; + const infra_prefix = "__route__infra__"; + const target: ?*const Node = c.cbm_gbuf_find_by_id(gbuf, edge.*.target_id); + const t = target orelse return true; + return t.qualified_name == null or + c.strncmp(t.qualified_name, infra_prefix, infra_prefix.len) != 0; +} + +fn logPassTiming(pass: interop.Str, t: Timespec) void { + interop.logInfo("pass.timing", .{ "pass", pass, "elapsed_ms", elapsedText(t) }); +} + +/// Run post-extraction passes (tests, decorator tags, configlink). +fn runPostpasses(ctx: *Ctx, changed_files: [*c]FileInfo, ci: c_int, project: CStr, before: ?*const DerivedState) void { + var t: Timespec = undefined; + + now(&t); + _ = c.cbm_pipeline_pass_tests(ctx, changed_files, ci); + logPassTiming("incr_tests", t); + + // File nodes for changed paths are deleted and recreated above, so their + // git-history properties disappear unless the full-repository derived + // pass is reapplied. Rebuild the derived edges as well to match a fresh + // full index instead of accumulating stale FILE_CHANGES_WITH rows. + _ = c.cbm_gbuf_delete_edges_by_type(ctx.gbuf, "FILE_CHANGES_WITH"); + now(&t); + const history_rc = c.cbm_pipeline_pass_githistory(ctx); + interop.logInfo("pass.timing", .{ + "pass", "incr_githistory", + "elapsed_ms", elapsedText(t), + "rc", itoa(history_rc), + }); + + now(&t); + _ = c.cbm_pipeline_pass_decorator_tags(ctx.gbuf, project); + logPassTiming("incr_decorator_tags", t); + + // Route matching is global: an infra Route in an unchanged config file can + // match a handler in a changed file, and DATA_FLOWS follow those matches. + // Remove the previous matches and data flows, then rerun the pass a full + // index runs, before the derived digest reads the graph. + now(&t); + _ = c.cbm_gbuf_delete_edges_by_type_unless(ctx.gbuf, "HANDLES", incrHandlesEdgeIsNotInfraMatch, null); + _ = c.cbm_gbuf_delete_edges_by_type(ctx.gbuf, "DATA_FLOWS"); + _ = c.cbm_pipeline_create_route_nodes(ctx.gbuf); + logPassTiming("incr_route_match", t); + + // Restore the final derived node properties before comparing pass inputs. + // The semantic and similarity passes ignore these complexity fields, but + // reproducing them makes the conservative whole-graph input digest exact. + now(&t); + _ = c.cbm_pipeline_pass_complexity(ctx); + logPassTiming("incr_complexity", t); + + // Importance reads CALLS/USAGE (re-extraction plus relinked inbound + // edges) and TESTS (pass_tests, first in this function). It must run + // before the digest below, which hashes properties_json. Nodes from + // unchanged files already carry the key; the write-back overwrites it. + now(&t); + _ = c.cbm_pipeline_pass_importance(ctx); + logPassTiming("incr_importance", t); + + now(&t); + const after = derivedState(ctx.gbuf); + const input_unchanged = if (before) |b| derivedDigestEqual(&b.input, &after.input) else false; + interop.logInfo("incremental.derived_digest", .{ + "unchanged", @as(CStr, if (input_unchanged) "true" else "false"), + "elapsed_ms", elapsedText(t), + }); + + const config_unchanged = input_unchanged and derivedDigestEqual(&before.?.config, &after.config); + if (!config_unchanged) { + now(&t); + _ = c.cbm_pipeline_pass_configlink(ctx); + logPassTiming("incr_configlink", t); + } else { + interop.logInfo("incremental.derived_gate", .{ "component", "configlink", "decision", "reuse" }); + } + + // SIMILAR_TO + SEMANTICALLY_RELATED edges only in moderate/full modes + if (ctx.mode <= c.CBM_MODE_MODERATE) { + // These are global derived relations. The old graph was loaded before + // this incremental update, so rerunning either pass without first + // removing its prior output leaves relationships that a clean full + // index would no longer select after the corpus changed. + const similarity_unchanged = input_unchanged and + derivedDigestEqual(&before.?.similarity, &after.similarity); + if (!similarity_unchanged) { + _ = c.cbm_gbuf_delete_edges_by_type(ctx.gbuf, "SIMILAR_TO"); + now(&t); + _ = c.cbm_pipeline_pass_similarity(ctx); + logPassTiming("incr_similarity", t); + } else { + interop.logInfo("incremental.derived_gate", .{ "component", "similarity", "decision", "reuse" }); + } + + const semantic_unchanged = input_unchanged and + derivedDigestEqual(&before.?.semantic, &after.semantic) and + c.cbm_gbuf_reuse_zova_semantic_vectors(ctx.gbuf) == 0; + if (!semantic_unchanged) { + _ = c.cbm_gbuf_delete_edges_by_type(ctx.gbuf, "SEMANTICALLY_RELATED"); + c.cbm_gbuf_clear_semantic_vectors(ctx.gbuf); + now(&t); + _ = c.cbm_pipeline_pass_semantic_edges(ctx); + logPassTiming("incr_semantic_edges", t); + } else { + interop.logInfo("incremental.derived_gate", .{ "component", "semantic", "decision", "reuse" }); + } + } +} + +/// Publish the merged graph and hashes to cbm.zova as a delta generation. +/// Mode-skipped hash rows are preserved so subsequent reindexes can correctly +/// distinguish "never indexed" from "indexed but not visited this pass". +fn dumpAndPersist( + pipeline: ?*c.cbm_pipeline_t, + gbuf: ?*Gbuf, + files: [*c]FileInfo, + file_count: c_int, + mode_skipped: [*c]const FileHash, + mode_skipped_count: c_int, + before: ?*c.cbm_zova_workspace_snapshot_t, +) c_int { + // #49: same post-extraction gates as the full path. Nothing is published + // until the delta commit, so an abort leaves the served generation. + c.cbm_pipeline_log_phase_mem("incr_pre_dump"); + var gate_rc = c.cbm_pipeline_budget_gate("before_incr_dump"); + if (gate_rc != 0) return gate_rc; + c.cbm_pipeline_run_before_dump_hook(gbuf); + if (c.cbm_gbuf_prepare_zova_dump_delta(gbuf) != 0) { + logError("incremental.err", .{ "phase", "zova_prepare" }); + return c.CBM_NOT_FOUND; + } + c.cbm_pipeline_log_phase_mem("incr_zova_prepare"); + gate_rc = c.cbm_pipeline_budget_gate("before_incr_publish"); + if (gate_rc != 0) return gate_rc; + if (c.cbm_pipeline_publish_zova_user_database_delta(pipeline, gbuf, files, file_count, mode_skipped, mode_skipped_count, before) != 0) { + logError("incremental.err", .{ "phase", "zova_single_file" }); + return c.CBM_NOT_FOUND; + } + c.cbm_pipeline_log_phase_mem("incr_zova_publish"); + return 0; +} + +// ── Incremental pipeline entry point ──────────────────────────────── + +fn copyZovaSnapshotHashes( + project: CStr, + snapshot: *const c.cbm_zova_workspace_snapshot_t, + out_hashes: *[*c]FileHash, + out_count: *c_int, +) c_int { + out_hashes.* = null; + out_count.* = 0; + if (snapshot.file_hash_count <= 0) return 0; + const hashes = interop.callocN(FileHash, ix(snapshot.file_hash_count)); + if (hashes == null) return c.CBM_NOT_FOUND; + for (0..ix(snapshot.file_hash_count)) |i| { + const source = util.at(snapshot.file_hashes, i); + hashes[i].project = c.strdup(project); + hashes[i].rel_path = c.strdup(source.file_path); + hashes[i].sha256 = c.strdup(source.content_hash); + hashes[i].mtime_ns = source.mtime_ns; + hashes[i].size = source.size_bytes; + if (hashes[i].project == null or hashes[i].rel_path == null or hashes[i].sha256 == null) { + c.cbm_store_free_file_hashes(hashes, snapshot.file_hash_count); + return c.CBM_NOT_FOUND; + } + } + out_hashes.* = hashes; + out_count.* = snapshot.file_hash_count; + return 0; +} + +fn freeDeleted(deleted: [*c]MutStr, deleted_count: c_int) void { + for (0..ix(deleted_count)) |i| c.free(deleted[i]); + c.free(@ptrCast(deleted)); +} + +fn runIncremental( + p: ?*c.cbm_pipeline_t, + files: [*c]FileInfo, + file_count: c_int, + snapshot: *c.cbm_zova_workspace_snapshot_t, + existing: ?*Gbuf, +) c_int { + var t0: Timespec = undefined; + now(&t0); + + const project: CStr = c.cbm_pipeline_project_name(p); + + var stored: [*c]FileHash = null; + var stored_count: c_int = 0; + if (copyZovaSnapshotHashes(project, snapshot, &stored, &stored_count) != 0) { + logError("incremental.err", .{ "msg", "load_zova_hashes_failed" }); + c.cbm_gbuf_free(existing); + return c.CBM_NOT_FOUND; + } + + // Classify files + var n_changed: c_int = 0; + var n_unchanged: c_int = 0; + const is_changed = classifyFiles(files, file_count, stored, stored_count, &n_changed, &n_unchanged); + + // Classify stored files absent from current discovery: truly-deleted + // (purge) vs mode-skipped (preserve nodes AND hash rows). + var deleted: [*c]MutStr = null; + var mode_skipped: [*c]FileHash = null; + var mode_skipped_count: c_int = 0; + const deleted_count = findDeletedFiles( + c.cbm_pipeline_repo_path(p), + files, + file_count, + stored, + stored_count, + &deleted, + &mode_skipped, + &mode_skipped_count, + ); + + if (is_changed != null) { + const infra_added = incrExpandInfraChanges(files, file_count, is_changed, deleted, deleted_count); + n_changed += infra_added; + n_unchanged -= infra_added; + + const client_added = incrExpandClientImporterChanges(existing, files, file_count, is_changed, deleted, deleted_count); + n_changed += client_added; + n_unchanged -= client_added; + } + + interop.logInfo("incremental.classify", .{ + "changed", itoa(n_changed), + "unchanged", itoa(n_unchanged), + "deleted", itoa(deleted_count), + "mode_skipped", itoa(mode_skipped_count), + }); + + // Fast path: nothing changed → skip. The committed generation is left + // untouched, which means existing hash rows (including for any + // mode-skipped files that were already preserved by an earlier run) + // remain intact. + if (n_changed == 0 and deleted_count == 0) { + interop.logInfo("incremental.noop", .{ "reason", "no_changes" }); + c.free(is_changed); + c.free(@ptrCast(deleted)); + freeModeSkipped(mode_skipped, mode_skipped_count); + c.cbm_store_free_file_hashes(stored, stored_count); + c.cbm_gbuf_free(existing); + return 0; + } + + // Module and Folder spellings follow the stem collisions of the whole + // discovered set, as in a full run. + var stem_marks: StemMarks = undefined; + c.cbm_stem_marks_init(&stem_marks, project, files, file_count); + if (incrStemMarksChanged(project, files, file_count, &stem_marks, stored, stored_count, deleted, deleted_count)) { + interop.logInfo("incremental.declined", .{ "reason", "stem_collision_changed" }); + c.cbm_stem_marks_free(&stem_marks); + c.free(is_changed); + freeDeleted(deleted, deleted_count); + freeModeSkipped(mode_skipped, mode_skipped_count); + c.cbm_store_free_file_hashes(stored, stored_count); + c.cbm_gbuf_free(existing); + return c.CBM_PIPELINE_INCREMENTAL_DECLINED; + } + + c.cbm_store_free_file_hashes(stored, stored_count); + + // Build list of changed files + const changed_files: [*c]FileInfo = + if (n_changed > 0) interop.mallocN(FileInfo, ix(n_changed)) else null; + var ci: c_int = 0; + for (0..ix(file_count)) |i| { + if (is_changed[i]) { + changed_files[ix(ci)] = files[i]; + ci += 1; + } + } + c.free(is_changed); + + interop.logInfo("incremental.reparse", .{ "files", itoa(ci) }); + + // Step 1: the existing graph was exported from the committed generation + interop.logInfo("incremental.load_zova", .{ + "nodes", itoa(c.cbm_gbuf_node_count(existing)), + "edges", itoa(c.cbm_gbuf_edge_count(existing)), + }); + + var t: Timespec = undefined; + now(&t); + const derived_before = derivedState(existing); + interop.logInfo("incremental.derived_digest", .{ + "phase", "baseline", + "elapsed_ms", elapsedText(t), + }); + + // Snapshot inbound cross-file edges into changed files BEFORE purging, so + // the cascade delete doesn't permanently drop edges whose source lives in + // an unchanged (never-re-parsed) file. Re-linked after re-resolution. + var edge_cap: EdgeCapture = .{ .gbuf = existing, .changed_paths = null }; + { + const changed_paths = c.cbm_ht_create(if (ci > 0) htCap(ci) else c.CBM_SZ_64); + for (0..ix(ci)) |i| { + _ = c.cbm_ht_set(changed_paths, changed_files[i].rel_path, util.at(changed_files, i)); + } + for (0..ix(deleted_count)) |i| { + _ = c.cbm_ht_set(changed_paths, deleted[i], deleted[i]); + } + edge_cap.changed_paths = changed_paths; + now(&t); + c.cbm_gbuf_foreach_edge(existing, incrCaptureInboundEdge, &edge_cap); + edge_cap.changed_paths = null; + c.cbm_ht_free(changed_paths); // keys borrowed from changed_files; not freed here + } + interop.logInfo("incremental.edge_snapshot", .{ + "captured", itoa(edge_cap.count), + "elapsed_ms", elapsedText(t), + }); + + // Step 2: Purge stale nodes + now(&t); + for (0..ix(ci)) |i| { + _ = c.cbm_gbuf_delete_by_file(existing, changed_files[i].rel_path); + } + for (0..ix(deleted_count)) |i| { + _ = c.cbm_gbuf_delete_by_file(existing, deleted[i]); + } + + const affected_count = ci + deleted_count; + var affected_paths: [*c]CStr = null; + if (affected_count > 0) { + affected_paths = interop.callocN(CStr, ix(affected_count)); + if (affected_paths != null) { + for (0..ix(ci)) |i| affected_paths[i] = changed_files[i].rel_path; + for (0..ix(deleted_count)) |i| affected_paths[ix(ci) + i] = deleted[i]; + } + } + if (affected_paths != null) { + _ = c.cbm_gbuf_delete_empty_folders_under(existing, affected_paths, affected_count); + c.free(@ptrCast(affected_paths)); + } else if (affected_count > 0) { + interop.logWarn("incremental.folder_prune_oom", .{ "affected_paths", itoa(affected_count) }); + } + freeDeleted(deleted, deleted_count); + interop.logInfo("incremental.purge", .{ "elapsed_ms", elapsedText(t) }); + + // Step 3-5: Registry + extract + resolve + const registry = c.cbm_registry_new(); + now(&t); + c.cbm_gbuf_foreach_node(existing, registryVisitor, registry); + interop.logInfo("incremental.registry_seed", .{ + "symbols", itoa(c.cbm_registry_size(registry)), + "elapsed_ms", elapsedText(t), + }); + + // Discovery exclusions (gitignore + skip dirs) captured by the run that + // routed here. Borrowed from the pipeline so the auxiliary repo walks + // (pkgmap via merge_pkg_entries, path aliases) skip excluded subtrees on + // incremental runs too — same borrow as the full path (#792/#804). + var excluded_dirs: [*c]MutStr = null; + var excluded_count: c_int = 0; + c.cbm_pipeline_get_excluded(p, &excluded_dirs, &excluded_count); + + const path_aliases = + c.cbm_load_path_aliases_excluded(c.cbm_pipeline_repo_path(p), excluded_dirs, excluded_count); + + var ctx = std.mem.zeroes(Ctx); + ctx.project_name = project; + ctx.repo_path = c.cbm_pipeline_repo_path(p); + ctx.gbuf = existing; + ctx.registry = registry; + ctx.cancelled = c.cbm_pipeline_cancelled_ptr(p); + ctx.pipeline = p; // so passes can record per-file skips (Track B) + ctx.mode = c.cbm_pipeline_get_mode(p); + ctx.path_aliases = path_aliases; + ctx.excluded_dirs = excluded_dirs; + ctx.excluded_count = excluded_count; + ctx.stem_marks = &stem_marks; + + for (0..ix(ci)) |i| { + incrAddChangedFileStructure(existing, project, &stem_marks, util.at(changed_files, i)); + } + + var extract_rc = runExtractResolve(&ctx, changed_files, ci); + ctx.stem_marks = null; + c.cbm_stem_marks_free(&stem_marks); + // #49: extract + resolve work sets are done; gate before the derived + // passes and publication so an explicitly over-budget run stops here. + // RSS includes the preloaded served graph. That is deliberate: a user who + // capped memory below the graph should be told indexing cannot proceed, + // while the detected RAM-fraction default never gates this path. + if (extract_rc == 0) extract_rc = c.cbm_pipeline_budget_gate("before_incr_passes"); + if (extract_rc != 0) { + incrFreeEdgeCapture(&edge_cap); + c.free(changed_files); + c.cbm_registry_free(registry); + c.cbm_path_alias_collection_free(path_aliases); + c.cbm_pkgmap_free(ctx.pkgmap); + ctx.pkgmap = null; + c.cbm_pipeline_set_pkgmap(null); + freeModeSkipped(mode_skipped, mode_skipped_count); + c.cbm_gbuf_free(existing); + return extract_rc; + } + // Changed files got fresh File props above, so a stale coverage stamp + // cannot survive; unchanged files keep theirs from the generation. + c.cbm_pipeline_stamp_coverage(p, existing); + _ = c.cbm_pipeline_pass_k8s(&ctx, changed_files, ci); + + incrRebuildSnapshotCallers(&ctx, &edge_cap, files, file_count); + + // Restore inbound cross-file edges before any derived pass reads the + // mixed graph. Semantic vectors and complexity both consume CALLS + // topology, so restoring only after those passes made their output differ + // from a clean full index even though the final SQLite edges matched. + now(&t); + const relinked = incrRestoreInboundEdges(existing, &edge_cap); + interop.logInfo("incremental.edge_relink", .{ + "relinked", itoa(relinked), + "captured", itoa(edge_cap.count), + "elapsed_ms", elapsedText(t), + }); + incrFreeEdgeCapture(&edge_cap); + + runPostpasses(&ctx, changed_files, ci, project, &derived_before); + + c.free(changed_files); + c.cbm_registry_free(registry); + c.cbm_path_alias_collection_free(path_aliases); + c.cbm_pkgmap_free(ctx.pkgmap); // built by the parallel extract, if it ran + ctx.pkgmap = null; + c.cbm_pipeline_set_pkgmap(null); + + // Step 7: Publish (preserves mode-skipped hash rows so the next reindex + // can correctly classify those files instead of seeing them as + // never-existed). + // Record committed counts before dumpAndPersist (whose dump frees the + // gbuf node index, zeroing the count) so the #334 plausibility gate also + // covers incremental reindexes, not just full ones. + c.cbm_pipeline_set_committed_counts(p, c.cbm_gbuf_node_count(existing), c.cbm_gbuf_edge_count(existing)); + const persist_rc = dumpAndPersist(p, existing, files, file_count, mode_skipped, mode_skipped_count, snapshot); + freeModeSkipped(mode_skipped, mode_skipped_count); + c.cbm_gbuf_free(existing); + if (persist_rc != 0) return persist_rc; + + interop.logInfo("incremental.done", .{ "elapsed_ms", elapsedText(t0) }); + return 0; +} + +pub export fn cbm_pipeline_run_incremental_zova( + p: ?*c.cbm_pipeline_t, + zova_path: CStr, + files: [*c]FileInfo, + file_count: c_int, + snapshot: ?*c.cbm_zova_workspace_snapshot_t, + preloaded_graph: ?*Gbuf, +) c_int { + _ = zova_path; + const snap = snapshot orelse return c.CBM_NOT_FOUND; + if (preloaded_graph == null) return c.CBM_NOT_FOUND; + return runIncremental(p, files, file_count, snap, preloaded_graph); +} diff --git a/src/pipeline/registry.c b/src/pipeline/registry.c deleted file mode 100644 index 4442c7f4..00000000 --- a/src/pipeline/registry.c +++ /dev/null @@ -1,1420 +0,0 @@ - -/* - * registry.c — Function/Method/Class registry for call resolution. - * - * Indexes all callable symbols by qualified name and simple name. - * Resolution uses a prioritized strategy chain: - * 1. Import map lookup - * 2. Same-module match - * 3. Unique name (single candidate project-wide) - * 4. Suffix match with import distance scoring - */ -#include "foundation/constants.h" - -enum { REG_INIT_CAP = 16, REG_MIN_CANDIDATES = 3, REG_RESOLVED = 1, REG_SUFFIX_ALLOC = 2 }; -/* Names with more registered definitions than this are unresolvable by name - * alone: candidate_count_penalty already floors their confidence to ~3/count - * (<= 0.006 at 256), so the emitted edge is noise — while the candidate walk - * (reachability + scoring per candidate, re-done per file) is the dominant - * resolution cost on identifier-dense repos. On the Linux kernel, 274 names - * exceed 256 candidates ("list_head" 7188, "flags" 5520, "dev" 4374, ...) and - * accounted for ~900 s of the 987 s usage-resolution CPU. Bail out early. */ -enum { REG_MAX_CANDIDATES = 256 }; -#define REG_FULL_CONF 1.0 -#define REG_HALF_PENALTY 0.5 - -#define DEFAULT_CONFIDENCE 0.5 -#include "pipeline/pipeline.h" -#include "pipeline/pipeline_internal.h" -#include "discover/discover.h" /* cbm_language_for_filename for #725 guard */ -#include "foundation/compat.h" /* CBM_TLS */ -#include "foundation/hash_table.h" -#include "foundation/dyn_array.h" -#include "foundation/platform.h" -#include "foundation/str_util.h" - -#include -#include -#include -#include - -/* Confidence score → human-readable band label. */ -#define CONF_BAND_HIGH 0.7 -#define CONF_BAND_MEDIUM 0.45 -#define CONF_BAND_SPECULATIVE 0.25 - -const char *cbm_confidence_band(double score) { - if (score >= CONF_BAND_HIGH) { - return "high"; - } - if (score >= CONF_BAND_MEDIUM) { - return "medium"; - } - if (score >= CONF_BAND_SPECULATIVE) { - return "speculative"; - } - return ""; -} - -/* ── Resolution confidence scores ────────────────────────────────── */ -/* Strategy 1: import_map — direct import → high confidence */ -#define CONF_IMPORT_MAP 0.95 -#define CONF_IMPORT_MAP_SUFFIX 0.85 -/* Strategy 2: same_module — same file/package → high confidence */ -#define CONF_SAME_MODULE 0.90 -/* Strategy 3: unique_name — only one candidate project-wide */ -#define CONF_UNIQUE_NAME 0.75 -/* Strategy 4: suffix_match — multiple candidates, filtered */ -#define CONF_SUFFIX_MATCH 0.55 -/* Fuzzy fallback: lower confidence */ -#define CONF_FUZZY_SINGLE 0.40 -#define CONF_FUZZY_MULTI 0.30 -/* Candidate count penalty cap */ -#define CANDIDATE_PENALTY_CAP 3.0 - -/* ── Internal types ──────────────────────────────────────────────── */ - -/* Array of QN strings for byName index */ -typedef CBM_DYN_ARRAY(char *) qn_array_t; - -struct cbm_registry { - /* Interned label strings (<=~30 distinct labels; owned here, freed in - * _free). The exact map's VALUES point into this pool instead of one - * strdup per registered definition (~8.5M strdups on the kernel). */ - char *label_pool[64]; - int label_pool_n; - /* exact: qualifiedName → label string (heap-owned copies) */ - CBMHashTable *exact; - - /* byName: simpleName → qn_array_t* (heap-owned) */ - CBMHashTable *by_name; -}; - -/* ── Helpers ─────────────────────────────────────────────────────── */ - -/* Extract the last path segment from a QN. Returns pointer into s. - * Recognizes both '.' (most langs) and Rust/C++ '::' separators, so a - * scoped callee like "lib::square" yields "square" rather than the whole - * scoped path (which never matches the by-name index). */ -static const char *simple_name(const char *qn) { - const char *dot = strrchr(qn, '.'); - const char *seg = dot ? dot + SKIP_ONE : qn; - /* Find the last "::" and, if it sits after the last '.', use the - * segment following it. */ - const char *colons = NULL; - for (const char *p = qn; (p = strstr(p, "::")) != NULL; p += 2) { - colons = p; - } - if (colons && colons + 2 > seg) { - seg = colons + 2; - } - return seg; -} - -/* Extract everything before the last dot. Returns heap-allocated string. */ - -/* Count common dot-separated prefix segments. */ -static int common_prefix_len(const char *a, const char *b) { - if (!a || !b) { - return 0; - } - int count = 0; - while (*a && *b) { - /* Find next segment in each */ - const char *adot = strchr(a, '.'); - const char *bdot = strchr(b, '.'); - size_t alen = adot ? (size_t)(adot - a) : strlen(a); - size_t blen = bdot ? (size_t)(bdot - b) : strlen(b); - if (alen != blen || memcmp(a, b, alen) != 0) { - break; - } - count++; - a += alen + (adot ? SKIP_ONE : 0); - b += blen + (bdot ? SKIP_ONE : 0); - if (!adot || !bdot) { - break; - } - } - return count; -} - -enum { REG_TEST_PENALTY = 1000 }; - -/* Check if a qualified name looks like a test/mock path. */ -static bool is_test_qn(const char *qn) { - if (!qn) { - return false; - } - return (strstr(qn, "Test") != NULL || strstr(qn, "test") != NULL || - strstr(qn, "Mock") != NULL || strstr(qn, "mock") != NULL || - strstr(qn, "Stub") != NULL || strstr(qn, "stub") != NULL || - strstr(qn, "Fake") != NULL || strstr(qn, "fake") != NULL || - strstr(qn, "Fixture") != NULL || strstr(qn, "spec") != NULL); -} - -/* Score a candidate for tiebreaking. Higher = better. - * Layer 1: Non-test code preferred over test code (+1000) - * Layer 2: Namespace proximity via common prefix length (+plen) */ -static int candidate_score(const char *candidate_qn, const char *module_qn) { - int score = 0; - if (!is_test_qn(candidate_qn)) { - score += REG_TEST_PENALTY; - } - score += common_prefix_len(candidate_qn, module_qn); - return score; -} - -/* Pick candidate with highest composite score (test-deprioritization + namespace proximity). */ -static const char *best_by_import_distance(const char **candidates, int count, - const char *module_qn) { - const char *best = NULL; - int best_score = CBM_NOT_FOUND; - for (int i = 0; i < count; i++) { - int score = candidate_score(candidates[i], module_qn); - if (score > best_score) { - best_score = score; - best = candidates[i]; - } - } - return best; -} - -/* ── Per-file is_import_reachable memoization cache ─────────────── - * - * The hot path on kubernetes spends ~50% of resolve_calls CPU in - * is_import_reachable's O(candidates × imports × strstr) scan. The - * SAME candidate_qn is re-evaluated dozens of times per file because - * the same callee_name appears in multiple call sites and each lookup - * re-checks all candidates for that name. - * - * Cache life-cycle is per-FILE (because import_vals changes between - * files). resolve_file_calls calls _begin at file entry and _end at - * file exit. Thread-local so each worker has its own cache without - * contention. */ -static CBM_TLS CBMHashTable *_reach_cache = NULL; - -/* Sentinels stored as values in the cache. NULL means "not cached". - * We need two distinct non-NULL pointers to encode true/false. */ -#define REACH_CACHE_TRUE ((void *)(uintptr_t)1) -#define REACH_CACHE_FALSE ((void *)(uintptr_t)2) - -static void reach_cache_free_key(const char *key, void *val, void *ud) { - (void)val; - (void)ud; - free((char *)key); -} - -void cbm_registry_reach_cache_begin(int estimated_capacity) { - if (_reach_cache) { - /* Defensive: caller forgot to call _end. Clear and reuse. */ - cbm_ht_foreach(_reach_cache, reach_cache_free_key, NULL); - cbm_ht_clear(_reach_cache); - return; - } - if (estimated_capacity < 16) - estimated_capacity = 16; - _reach_cache = cbm_ht_create((uint32_t)estimated_capacity); -} - -void cbm_registry_reach_cache_end(void) { - if (!_reach_cache) - return; - cbm_ht_foreach(_reach_cache, reach_cache_free_key, NULL); - cbm_ht_free(_reach_cache); - _reach_cache = NULL; -} - -/* ── Per-file import-map prefix → module-QN hash ────────────────── - * - * resolve_import_map does a linear strcmp scan over the per-file - * import list for every call (and every usage, throw, rw, def). On - * kubernetes typical files have 20-30 imports and ~200 calls. That's - * 4000-6000 strcmps per file × 20K files = ~80M strcmps. - * - * Build a hash from prefix → module_qn ONCE per file at file entry. - * Each lookup becomes O(1). Keys and values are BORROWED from the - * caller's import_map — they outlive the cache scope. */ -static CBM_TLS CBMHashTable *_import_map_cache = NULL; - -void cbm_registry_import_map_cache_begin(const char **keys, const char **vals, int count) { - if (_import_map_cache) { - cbm_ht_free(_import_map_cache); - _import_map_cache = NULL; - } - if (!keys || !vals || count <= 0) - return; - _import_map_cache = cbm_ht_create((uint32_t)count * 2u + 8u); - if (!_import_map_cache) - return; - for (int i = 0; i < count; i++) { - if (keys[i] && vals[i]) { - cbm_ht_set(_import_map_cache, keys[i], (void *)(uintptr_t)vals[i]); - } - } -} - -void cbm_registry_import_map_cache_end(void) { - if (!_import_map_cache) - return; - /* Keys/values borrowed from caller — no free callback needed. */ - cbm_ht_free(_import_map_cache); - _import_map_cache = NULL; -} - -/* ── Per-file full-result cache for cbm_registry_resolve ────────── - * - * THE big one: 98.7% of resolve_calls CPU on kubernetes lives inside - * cbm_registry_resolve's strategy chain (suffix_match iterating - * hundreds of candidates per common name like "Get"/"Add"/"New" - * × 1M LSP-miss calls = ~800s of work). - * - * Since module_qn is CONSTANT per file, the resolution for a given - * callee_name within a file is also constant — same name resolves - * to the same QN every time. Cache the full cbm_resolution_t result - * keyed by callee_name; first lookup does the full chain, repeats - * are O(1). On K8s a typical file has ~200 calls but only ~50 - * unique callee_names → ~75% hit rate → ~75% of the resolve cost - * eliminated. */ -typedef struct { - cbm_resolution_t res; -} resolve_cache_entry_t; - -static CBM_TLS CBMHashTable *_resolve_cache = NULL; -/* Entries are malloc'd; keys are strdup'd. Both freed in _end. */ - -static void resolve_cache_free_entry(const char *key, void *val, void *ud) { - (void)ud; - free((char *)key); - free(val); -} - -void cbm_registry_resolve_cache_begin(int estimated_capacity) { - if (_resolve_cache) { - cbm_ht_foreach(_resolve_cache, resolve_cache_free_entry, NULL); - cbm_ht_free(_resolve_cache); - _resolve_cache = NULL; - } - if (estimated_capacity < 32) - estimated_capacity = 32; - _resolve_cache = cbm_ht_create((uint32_t)estimated_capacity); -} - -void cbm_registry_resolve_cache_end(void) { - if (!_resolve_cache) - return; - cbm_ht_foreach(_resolve_cache, resolve_cache_free_entry, NULL); - cbm_ht_free(_resolve_cache); - _resolve_cache = NULL; -} - -/* Check if candidate's module prefix appears in import map values. - * Uses stack buffer to avoid malloc/free per call in hot resolution loop. - * Per-file memoization via TLS cache: repeated lookups of the same - * candidate_qn (same name appears in many call sites) become O(1) - * after the first computation. */ -static bool is_import_reachable(const char *candidate_qn, const char **import_vals, - int import_count) { - if (_reach_cache) { - void *cached = cbm_ht_get(_reach_cache, candidate_qn); - if (cached == REACH_CACHE_TRUE) - return true; - if (cached == REACH_CACHE_FALSE) - return false; - } - - char cand_mod[CBM_SZ_512]; - const char *last = strrchr(candidate_qn, '.'); - if (last) { - size_t len = (size_t)(last - candidate_qn); - if (len >= sizeof(cand_mod)) { - len = sizeof(cand_mod) - SKIP_ONE; - } - memcpy(cand_mod, candidate_qn, len); - cand_mod[len] = '\0'; - } else { - snprintf(cand_mod, sizeof(cand_mod), "%s", candidate_qn); - } - bool reachable = false; - for (int i = 0; i < import_count; i++) { - if (strstr(cand_mod, import_vals[i]) || strstr(import_vals[i], cand_mod)) { - reachable = true; - break; - } - } - - if (_reach_cache) { - char *kdup = strdup(candidate_qn); - if (kdup) { - cbm_ht_set(_reach_cache, kdup, reachable ? REACH_CACHE_TRUE : REACH_CACHE_FALSE); - } - } - return reachable; -} - -/* Scale confidence inversely with candidate count. */ -static double candidate_count_penalty(double base, int count) { - if (count <= REG_MIN_CANDIDATES) { - return base; - } - return base * fmin(REG_FULL_CONF, CANDIDATE_PENALTY_CAP / (double)count); -} - -static cbm_resolution_t empty_result(void) { - cbm_resolution_t r = {0}; - return r; -} - -/* ── Perl builtin guard (#459 follow-up: call-graph noise) ────────── - * Curated subset of perlfunc core builtins. When a Perl CALL resolves - * only by the generic short-name matcher (no LSP, no import, after the - * same-module/name-lookup chain), a builtin name like `push`/`shift`/ - * `keys` must NOT be wired to a project sub that merely shares the name - * — that is virtually always a false positive. A genuine intra-project - * call is resolved by earlier (LSP/textual) stages before this guard. - * MUST stay sorted ASCII-ascending for bsearch. */ -static const char *const PERL_BUILTINS[] = { - "abs", "atan2", "binmode", "bless", "caller", "chdir", "chmod", "chomp", - "chop", "chown", "chr", "chroot", "close", "closedir", "cos", "defined", - "delete", "die", "do", "each", "eof", "eval", "exec", "exists", - "exit", "fork", "gmtime", "goto", "grep", "hex", "index", "int", - "join", "keys", "last", "lc", "lcfirst", "length", "local", "localtime", - "log", "lstat", "map", "mkdir", "my", "next", "oct", "open", - "opendir", "ord", "our", "pop", "pos", "print", "printf", "push", - "quotemeta", "rand", "read", "readdir", "readline", "redo", "ref", "rename", - "require", "return", "reverse", "rindex", "rmdir", "say", "scalar", "seek", - "shift", "sin", "sleep", "sort", "splice", "split", "sprintf", "sqrt", - "srand", "stat", "substr", "system", "time", "uc", "ucfirst", "undef", - "unlink", "unshift", "values", "wantarray", "warn", "write", -}; - -static int perl_builtin_cmp(const void *key, const void *elem) { - return strcmp((const char *)key, *(const char *const *)elem); -} - -/* True if `name` is one of the curated Perl core builtins. Used to suppress - * generic-resolver CALLS edges from Perl builtin invocations to project subs - * that happen to share the builtin's name. Perl-scoped: callers gate on the - * file language so no other language's resolution is affected. */ -bool cbm_perl_is_builtin(const char *name) { - if (!name || !name[0]) { - return false; - } - return bsearch(name, PERL_BUILTINS, sizeof(PERL_BUILTINS) / sizeof(PERL_BUILTINS[0]), - sizeof(PERL_BUILTINS[0]), perl_builtin_cmp) != NULL; -} - -/* Decide whether a *resolved* Perl call edge is generic-resolver noise that - * should be suppressed (#476). Returns true only for Perl, only for a builtin - * invocation or a method call, and only when the registry landed the match via - * a WEAK short-name strategy. High-confidence import/same-module strategies - * (same_module, import_map, import_map_suffix) are KEPT so a genuine same-file - * or imported call to a builtin-named sub still resolves — only the weak - * short-name guesses (suffix_match, unique_name) are dropped. `strategy` is the - * cbm_resolution_t.strategy of a non-empty match; - * NULL/empty (no match) returns false. Pure + side-effect-free so the - * suppression contract is unit-testable without a full pipeline. */ -bool cbm_perl_suppress_generic_match(bool is_perl, bool is_method, const char *callee_name, - const char *strategy) { - if (!is_perl) { - return false; - } - if (!(is_method || cbm_perl_is_builtin(callee_name))) { - return false; - } - if (!strategy || !strategy[0]) { - return false; - } - if (strcmp(strategy, "same_module") == 0 || strcmp(strategy, "import_map") == 0 || - strcmp(strategy, "import_map_suffix") == 0) { - return false; /* high-confidence import/same-module match — keep the genuine edge */ - } - return true; /* weak short-name match (suffix_match / unique_name / …) → drop */ -} - -/* Dynamic-language analogue of the Perl guard above (#592/#606/#1276 - * direction; precedent #477). A member call `x.foo()` reaches the weak textual - * cascade ONLY when the language's LSP could not resolve the receiver type — - * type-resolved calls win via lsp_* strategies before the registry runs. - * Binding such a call to a project symbol by a weak short-name strategy - * fabricates a CALLS edge (`re.test()` -> SalesforceRestClient.test, - * `accelerator.print()` -> MockAccelerator.print). Drop ONLY the weak - * strategies; keep import/same-module/qualified-tail matches and every lsp_* - * strategy. Uses an EXPLICIT drop-list (not keep-list + default-drop) because - * the parallel resolver runs lsp_* strategies through the same guard variable — - * a default-drop would silently kill lsp_ts_method. Pure + side-effect-free so - * the contract is unit-testable without a full pipeline. - * - * `enabled` is the CALLER's per-language gate, deliberately kept OUT of this - * helper: the guard applies only to the language set each call site enumerates - * (today Python plus the JS/TS family). Widening it is a - * per-language decision made at the call site in pass_parallel.c. */ -/* The weak short-name strategies that actually reach the call-resolution - * guards: the registry's suffix_match / unique_name and the parallel - * field_type_hint. "fuzzy" is listed as defensive insurance only — - * cbm_registry_fuzzy_resolve is not wired into the resolver - * today, so it never reaches these helpers, but naming it keeps a future wiring - * from silently reintroducing the noise. Everything else — same_module / - * import_map / import_map_suffix / qualified_suffix / callee_suffix / - * service_pattern / lsp_* — is a receiver- or import-aware match and is KEPT. - * - * Shared by BOTH weak-call guards below so the drop-list exists exactly once: a - * list that drifted between the member guard and the local-binding guard would - * make the two disagree about what "weak" means. */ -static bool weak_short_name_strategy(const char *strategy) { - if (!strategy || !strategy[0]) { - return false; - } - return strcmp(strategy, "suffix_match") == 0 || strcmp(strategy, "unique_name") == 0 || - strcmp(strategy, "field_type_hint") == 0 || strcmp(strategy, "fuzzy") == 0; -} - -bool cbm_registry_strategy_is_weak(const char *strategy) { - return weak_short_name_strategy(strategy); -} - -bool cbm_suppress_weak_member_match(bool enabled, bool is_method, const char *strategy) { - if (!enabled || !is_method) { - return false; - } - return weak_short_name_strategy(strategy); -} - -/* Zig member calls are typed by the Zig LSP: mapped @import aliases and relative - * files resolve as lsp_zig_*. A qualified_suffix match on a member call that - * reached the registry therefore only matched spelling, e.g. `old.hello()` with - * an unmapped `old` binding a same-named libs/old/src/old.zig. */ -bool cbm_suppress_zig_qualified_member(bool is_zig, bool is_method, const char *strategy) { - return is_zig && is_method && strategy && strcmp(strategy, "qualified_suffix") == 0; -} - -/* Bare-call counterpart of the member guard above. A Python call `foo()` whose - * callee identifier is bound as a parameter of an enclosing scope cannot be the - * module-level `foo`: the parameter shadows it for the whole body. Binding such - * a call to a project Function/Method by a weak short-name strategy fabricates - * the edge by construction (`def _run_with_heavy_slot(run): run()` -> - * SatoriLive.run). - * - * This is deliberately NOT keyed on the callee's spelling. A list of - * "generic-looking" names (get / run / execute) asserts that certain spellings - * are usually noise, which is a claim about corpus fashion rather than about - * what the resolver knew — and it ages invisibly, because nothing fails when the - * distribution shifts, the graph just quietly loses different edges. A parameter - * binding is a fact about THIS file's scope, decidable outright. - * - * `enabled` is the caller's per-language gate, kept out of the helper for the - * same reason as the member guard. Pure; unit-tested in test_registry.c. */ -bool cbm_suppress_weak_local_binding_call(bool enabled, bool callee_is_locally_bound, - const char *strategy) { - if (!enabled || !callee_is_locally_bound) { - return false; - } - return weak_short_name_strategy(strategy); -} - -static bool js_ts_family(CBMLanguage lang) { - return lang == CBM_LANG_JAVASCRIPT || lang == CBM_LANG_TYPESCRIPT || lang == CBM_LANG_TSX; -} - -/* C and C++ are one family for cross-language checks: .h maps to CBM_LANG_CPP - * in the extension table, so a .c file referencing a symbol declared in its - * own header would otherwise read as a language boundary. */ -static bool c_cpp_family(CBMLanguage lang) { - return lang == CBM_LANG_C || lang == CBM_LANG_CPP; -} - -/* cbm_path_base, but on Windows a backslash also separates directories. */ -static const char *native_path_basename(const char *path) { - const char *base = cbm_path_base(path); -#ifdef _WIN32 - const char *bslash = strrchr(base, '\\'); - if (bslash) { - base = bslash + SKIP_ONE; - } -#endif - return base; -} - -bool cbm_suppress_cross_language_suffix_match(CBMLanguage caller_lang, const char *target_file_path, - const char *strategy) { - /* Two same-named symbols in different languages: suffix_match picks one - * winner by import-distance and attaches every bare-name call to it - * (#725, Bash/Python main, JS/Python commit). unique_name is the - * candidates==1 case (#1572) and is not this guard. */ - if (!strategy || strcmp(strategy, "suffix_match") != 0) { - return false; - } - if (caller_lang == CBM_LANG_COUNT || !target_file_path || !target_file_path[0]) { - return false; - } - CBMLanguage target_lang = cbm_language_for_filename(native_path_basename(target_file_path)); - if (target_lang == CBM_LANG_COUNT) { - return false; - } - if (caller_lang == target_lang) { - return false; - } - if (js_ts_family(caller_lang) && js_ts_family(target_lang)) { - return false; - } - return true; -} - -bool cbm_suppress_cross_language_ref(CBMLanguage caller_lang, const char *target_file_path) { - /* #1928: USAGE / WRITES / READS analog of the CALLS guard above. A - * variable or field reference resolved by the short-name registry must - * not cross a language boundary: unlike CALLS, a reference edge carries - * no import-closure evidence at all — a Go test's local `event` and an - * eBPF C probe's automatic `event` share nothing but the spelling, so - * EVERY registry strategy is a bare-name guess here and none is exempt. - * LSP-backed semantic references resolve before the registry fallback - * and never reach this predicate, which is where a genuine cross-language - * binding (a future cgo resolver) would live. The JS/TS family keeps its - * exemption (.js/.ts/.d.ts pairs legitimately share symbols), and C/C++ - * count as one family (.h maps to CBM_LANG_CPP). */ - if (caller_lang == CBM_LANG_COUNT || !target_file_path || !target_file_path[0]) { - return false; - } - CBMLanguage target_lang = cbm_language_for_filename(native_path_basename(target_file_path)); - if (target_lang == CBM_LANG_COUNT) { - return false; - } - if (caller_lang == target_lang) { - return false; - } - if (js_ts_family(caller_lang) && js_ts_family(target_lang)) { - return false; - } - if (c_cpp_family(caller_lang) && c_cpp_family(target_lang)) { - return false; - } - return true; -} - -bool cbm_go_suppress_bare_field_ref(bool is_go, bool is_member_access, const char *target_label) { - /* #1942/#1962: a bare Go identifier can never denote a struct field — - * field access is always a selector expression (x.f). The extractor - * strips the receiver before the resolver runs (resolve_lhs_write_name - * records the trailing field name; is_reference_node records the inner - * field_identifier), so the reference TEXT is always dot-less and cannot - * carry the distinction — the recorded is_member_access shape can. Only a - * reference that was never the member half of a selector is refused a - * Field bind. Go-gated: a C#/Java/C++/Python method body legitimately - * references its own members bare (cp_reads_writes_cs_static_field pins - * that shape as required), so a global veto would break those languages. */ - if (!is_go || is_member_access || !target_label) { - return false; - } - return strcmp(target_label, "Field") == 0; -} - -/* ── Lifecycle ──────────────────────────────────────────────────── */ - -cbm_registry_t *cbm_registry_new(void) { - cbm_registry_t *r = calloc(CBM_ALLOC_ONE, sizeof(cbm_registry_t)); - if (!r) { - return NULL; - } - r->exact = cbm_ht_create(CBM_SZ_1K); - r->by_name = cbm_ht_create(CBM_SZ_512); - return r; -} - -static void free_label(const char *key, void *value, void *ud) { - (void)ud; - (void)value; /* interned in the registry's label_pool */ - free((void *)key); -} - -static void free_qn_array(const char *key, void *value, void *ud) { - (void)ud; - qn_array_t *arr = value; - if (arr) { - /* items borrow the exact map's keys — freed there, not here */ - cbm_da_free(arr); - free(arr); - } - free((void *)key); -} - -void cbm_registry_free(cbm_registry_t *r) { - if (!r) { - return; - } - /* by_name first: its items borrow exact's keys. */ - cbm_ht_foreach(r->by_name, free_qn_array, NULL); - cbm_ht_free(r->by_name); - cbm_ht_foreach(r->exact, free_label, NULL); - cbm_ht_free(r->exact); - for (int i = 0; i < r->label_pool_n; i++) { - free(r->label_pool[i]); - } - free(r); -} - -/* ── Registration ────────────────────────────────────────────────── */ - -void cbm_registry_add(cbm_registry_t *r, const char *name, const char *qualified_name, - const char *label) { - (void)name; - if (!r || !qualified_name || !label) { - return; - } - - /* Check for duplicate */ - if (cbm_ht_get(r->exact, qualified_name)) { - return; - } - - /* Intern the label (bounded set; linear scan is fine at this size). */ - const char *interned = NULL; - for (int i = 0; i < r->label_pool_n; i++) { - if (strcmp(r->label_pool[i], label) == 0) { - interned = r->label_pool[i]; - break; - } - } - if (!interned && r->label_pool_n < (int)(sizeof(r->label_pool) / sizeof(r->label_pool[0]))) { - r->label_pool[r->label_pool_n] = strdup(label); - interned = r->label_pool[r->label_pool_n]; - r->label_pool_n++; - } - if (!interned) { - return; /* pool exhausted (cannot happen with sane label sets) */ - } - - /* Store in exact map: QN → interned label. The key is the registry's ONE - * owned copy of the QN; by_name below borrows it (same lifetime) instead - * of a second strdup — this pair of copies was ~280 MB on the kernel. */ - cbm_ht_set(r->exact, strdup(qualified_name), (void *)interned); - const char *owned_qn = cbm_ht_get_key(r->exact, qualified_name); - - /* Index by simple name. - * No array dedup needed: exact-map check above guarantees uniqueness. */ - const char *simple = simple_name(qualified_name); - qn_array_t *arr = cbm_ht_get(r->by_name, simple); - if (!arr) { - arr = calloc(CBM_ALLOC_ONE, sizeof(qn_array_t)); - cbm_ht_set(r->by_name, strdup(simple), arr); - } - cbm_da_push(arr, (char *)owned_qn); -} - -/* ── Lookup ──────────────────────────────────────────────────────── */ - -bool cbm_registry_exists(const cbm_registry_t *r, const char *qn) { - if (!r || !qn) { - return false; - } - return cbm_ht_get(r->exact, qn) != NULL; -} - -const char *cbm_registry_label_of(const cbm_registry_t *r, const char *qn) { - if (!r || !qn) { - return NULL; - } - return cbm_ht_get(r->exact, qn); -} - -int cbm_registry_find_by_name(const cbm_registry_t *r, const char *name, const char ***out, - int *count) { - if (!r || !out || !count) { - return CBM_NOT_FOUND; - } - qn_array_t *arr = cbm_ht_get(r->by_name, name); - if (arr && arr->count > 0) { - *out = (const char **)arr->items; - *count = arr->count; - } else { - *out = NULL; - *count = 0; - } - return 0; -} - -int cbm_registry_size(const cbm_registry_t *r) { - return r ? (int)cbm_ht_count(r->exact) : 0; -} - -/* ── Resolution ──────────────────────────────────────────────────── */ - -/* Callback context for import_map_suffix scan */ -/* Strategy 1: Import map lookup (exact → suffix fallback) */ -static cbm_resolution_t resolve_import_map(const cbm_registry_t *r, const char *prefix, - const char *suffix, const char **keys, const char **vals, - int map_count) { - if (!keys || !vals || map_count <= 0) { - return empty_result(); - } - - /* Find prefix in import map keys. Prefer the per-file TLS hash - * cache (O(1)) over the linear scan when available. */ - const char *resolved = NULL; - if (_import_map_cache) { - resolved = (const char *)cbm_ht_get(_import_map_cache, prefix); - } else { - for (int i = 0; i < map_count; i++) { - if (strcmp(keys[i], prefix) == 0) { - resolved = vals[i]; - break; - } - } - } - if (!resolved) { - return empty_result(); - } - - /* Build candidate: resolved.suffix or resolved.prefix. - * When the callee has a dot ("pkg.Func"), prefix="pkg" is the import key - * and suffix="Func" is the function name, so the target QN is - * resolved.Func. When the callee is bare ("requireAdmin"), prefix IS the - * function name and suffix is NULL, so the target QN must be - * resolved.requireAdmin — not just resolved, which would point at the - * module node and miss the function entirely. */ - /* Direct hit only for a bare callee whose import already names the - * definition: `from m import f as g; g()` must reach m.f, not invent m.g - * (upstream #875/#979, 7fda9f71). With a suffix (`imported.method()`) a - * direct hit would swallow the suffix and bind the call to the imported - * symbol itself (upstream #1000). */ - if (!suffix || !suffix[0]) { - const char *direct = cbm_ht_get_key(r->exact, resolved); - if (direct) { - return (cbm_resolution_t){direct, "import_map", CONF_IMPORT_MAP, REG_RESOLVED}; - } - } - char candidate[CBM_SZ_512]; - if (suffix && suffix[0]) { - snprintf(candidate, sizeof(candidate), "%s.%s", resolved, suffix); - } else { - snprintf(candidate, sizeof(candidate), "%s.%s", resolved, prefix); - } - /* Use cbm_ht_get_key to get the persistent heap-owned key string */ - const char *stored_key = cbm_ht_get_key(r->exact, candidate); - if (stored_key) { - return (cbm_resolution_t){stored_key, "import_map", CONF_IMPORT_MAP, REG_RESOLVED}; - } - - /* import_map_suffix fallback: find a QN starting with resolved+"." and - * ending with "."+suffix. Any such QN's last segment equals the last - * segment of suffix, so probe the by_name index and tail-check the (few) - * candidates instead of cbm_ht_foreach over the WHOLE exact table — that - * scan ran per unresolved call and dominated elasticsearch's resolve - * phase (94% of samples: 700k-entry foreach + strlen per entry). */ - if (suffix && suffix[0]) { - char resolved_dot[CBM_SZ_512]; - char dot_suffix[CBM_SZ_256]; - snprintf(resolved_dot, sizeof(resolved_dot), "%s.", resolved); - snprintf(dot_suffix, sizeof(dot_suffix), ".%s", suffix); - qn_array_t *arr = cbm_ht_get(r->by_name, simple_name(suffix)); - if (arr) { - size_t rd_len = strlen(resolved_dot); - size_t ds_len = strlen(dot_suffix); - for (int i = 0; i < arr->count; i++) { - const char *qn = arr->items[i]; - size_t klen = strlen(qn); - if (klen >= rd_len + ds_len && strncmp(qn, resolved_dot, rd_len) == 0 && - strcmp(qn + klen - ds_len, dot_suffix) == 0) { - return (cbm_resolution_t){qn, "import_map_suffix", CONF_IMPORT_MAP_SUFFIX, - REG_RESOLVED}; - } - } - } - } - return empty_result(); -} - -/* Strategy 2: Same-module match */ -static cbm_resolution_t resolve_same_module(const cbm_registry_t *r, const char *callee_name, - const char *suffix, const char *module_qn) { - char candidate[CBM_SZ_512]; - snprintf(candidate, sizeof(candidate), "%s.%s", module_qn, callee_name); - const char *stored_key = cbm_ht_get_key(r->exact, candidate); - if (stored_key) { - return (cbm_resolution_t){stored_key, "same_module", CONF_SAME_MODULE, REG_RESOLVED}; - } - if (suffix && suffix[0]) { - snprintf(candidate, sizeof(candidate), "%s.%s", module_qn, suffix); - stored_key = cbm_ht_get_key(r->exact, candidate); - if (stored_key) { - return (cbm_resolution_t){stored_key, "same_module", CONF_SAME_MODULE, REG_RESOLVED}; - } - } - return empty_result(); -} - -/* Strategy 4: multiple candidates with import filtering. */ -static cbm_resolution_t resolve_multi_with_imports(const qn_array_t *arr, const char *module_qn, - const char **import_vals, int import_count) { - const char *filtered[CBM_SZ_256]; - int fcount = 0; - for (int i = 0; i < arr->count && fcount < CBM_SZ_256; i++) { - if (is_import_reachable(arr->items[i], import_vals, import_count)) { - filtered[fcount++] = arr->items[i]; - } - } - if (fcount == SKIP_ONE) { - double conf = candidate_count_penalty(CONF_SUFFIX_MATCH, arr->count); - return (cbm_resolution_t){filtered[0], "suffix_match", conf, arr->count}; - } - if (fcount > SKIP_ONE) { - const char *best = best_by_import_distance(filtered, fcount, module_qn); - if (best) { - double conf = candidate_count_penalty(CONF_SUFFIX_MATCH, fcount); - return (cbm_resolution_t){best, "suffix_match", conf, fcount}; - } - } - /* No import-reachable — use all candidates with penalty */ - const char *best = best_by_import_distance((const char **)arr->items, arr->count, module_qn); - if (best) { - double conf = candidate_count_penalty(CONF_SUFFIX_MATCH * REG_HALF_PENALTY, arr->count); - return (cbm_resolution_t){best, "suffix_match", conf, arr->count}; - } - return empty_result(); -} - -/* Confidence for a full qualified-tail match (Strategy 3.5). A package- or - * namespace-qualified callee that uniquely matches one candidate's full tail is - * as trustworthy as a same-module hit. */ -#define CONF_QUALIFIED_SUFFIX 0.90 - -/* When a callee is package/namespace-qualified (Foo::Bar::sub or Foo.Bar.sub), - * disambiguate among same-simple-name candidates by matching the FULL qualified - * tail against each candidate QN at a segment boundary. Returns the sole - * candidate whose QN equals or ends with ".", or NULL when zero - * or several candidates match (the caller then falls back to bare-name scoring). - * - * Fixes qualified cross-file calls collapsing onto one namespace when the bare - * symbol name is defined in several — e.g. Perl's Foo::Bar::run and Foo::Baz::run - * both reduce to "run", so the bare-name scorer would route every caller to a - * single winner. Language agnostic: callees with no separator return NULL and - * leave behavior unchanged. */ -static const char *qualified_suffix_match(const qn_array_t *arr, const char *callee_name) { - /* Normalize "::" → "." so the tail composes with dotted candidate QNs. */ - char dotted[CBM_SZ_512]; - size_t w = 0; - for (const char *s = callee_name; *s && w + SKIP_ONE < sizeof(dotted);) { - if (s[0] == ':' && s[1] == ':') { - dotted[w++] = '.'; - s += 2; - } else { - dotted[w++] = *s++; - } - } - dotted[w] = '\0'; - /* Must be qualified (contain a '.') — a bare name matches every candidate - * and carries no disambiguating signal. */ - if (!strchr(dotted, '.')) { - return NULL; - } - const char *match = NULL; - for (int i = 0; i < arr->count; i++) { - const char *qn = arr->items[i]; - size_t qlen = strlen(qn); - if (qlen < w) { - continue; - } - const char *tail = qn + (qlen - w); - if (strcmp(tail, dotted) != 0) { - continue; - } - /* Segment boundary: tail is the whole QN or is preceded by '.'. */ - if (tail != qn && tail[-1] != '.') { - continue; - } - if (match) { - return NULL; /* ambiguous — more than one qualified tail matches */ - } - match = qn; - } - return match; -} - -/* A dotted callee whose FIRST segment starts upper-case names a type — URLSession, - * Calendar, JSONEncoder. That receiver chain is evidence the bare-name scorers - * throw away, and throwing it away binds Foundation's URLSession.shared.data to - * a project's own PickedFile.data: high confidence, and nothing in the graph - * shows it is wrong. Require instead that the candidate's own parent segment - * appears somewhere in the chain. Calendar.utcGregorian.startOfDayUTC resolving - * to AuthDTOs.Calendar.startOfDayUTC passes, because Calendar is in the chain. - * - * Only an upper-case first segment is guarded. A lower-case root names a value - * (vm.load, http.Get, os.path.join) whose declared type the chain does not - * show, so the chain proves nothing there and the call passes through - * unchanged. A callee with no separator passes through as well. - * - * Language agnostic by design: the registry holds no language, and every - * language that writes receiver chains gains the same protection. */ -static bool receiver_chain_admits(const char *callee_name, const char *candidate_qn) { - /* Normalize "::" -> "." so the chain composes with dotted candidate QNs, - * the same way qualified_suffix_match does. */ - char dotted[CBM_SZ_512]; - size_t w = 0; - for (const char *s = callee_name; *s && w + SKIP_ONE < sizeof(dotted);) { - if (s[0] == ':' && s[1] == ':') { - dotted[w++] = '.'; - s += 2; - } else { - dotted[w++] = *s++; - } - } - dotted[w] = '\0'; - - const char *last_dot = strrchr(dotted, '.'); - if (!last_dot) { - return true; /* bare name — no receiver chain to judge */ - } - if (dotted[0] < 'A' || dotted[0] > 'Z') { - return true; /* lower-case root names a value, not a type */ - } - /* A name written in capitals with underscores is a constant holding a - * value, not a type: ISO_4217_URL.lower is a string's own method. JSON and - * URL carry no underscore and stay guarded. */ - int has_underscore = 0; - int all_caps = 1; - for (const char *c = dotted; c < last_dot && *c != '.'; c++) { - if (*c == '_') { - has_underscore = 1; - } else if (*c >= 'a' && *c <= 'z') { - all_caps = 0; - break; - } - } - if (all_caps && has_underscore) { - return true; - } - - /* The candidate's parent segment: the one before its final name. */ - const char *cand_last = strrchr(candidate_qn, '.'); - if (!cand_last || cand_last == candidate_qn) { - return true; /* top-level candidate — no parent to look for */ - } - const char *parent = cand_last; - while (parent > candidate_qn && parent[-1] != '.') { - parent--; - } - size_t parent_len = (size_t)(cand_last - parent); - - /* Walk the chain — every segment before the final callee name. A trailing - * "()" is dropped so JSONEncoder().encode reads as JSONEncoder. */ - for (const char *seg = dotted; seg < last_dot;) { - const char *end = strchr(seg, '.'); - size_t len = (size_t)(end - seg); - if (len >= 2 && seg[len - 2] == '(' && seg[len - 1] == ')') { - len -= 2; /* an empty "()" — JSONEncoder().encode names JSONEncoder */ - } - if (len == parent_len && strncmp(seg, parent, parent_len) == 0) { - return true; - } - seg = end + SKIP_ONE; - } - return false; -} - -/* Strategy 3+4: Name lookup + suffix match */ -static cbm_resolution_t resolve_name_lookup(const cbm_registry_t *r, const char *callee_name, - const char *module_qn, const char **import_vals, - int import_count) { - const char *lookup = simple_name(callee_name); - qn_array_t *arr = cbm_ht_get(r->by_name, lookup); - if (!arr || arr->count == 0) { - return empty_result(); - } - if (arr->count > REG_MAX_CANDIDATES) { - return empty_result(); /* unresolvably ambiguous — see REG_MAX_CANDIDATES */ - } - - /* Strategy 3.5: a qualified callee disambiguates among multiple same-name - * candidates by full qualified tail, before bare-name scoring collapses - * them onto a single winner. */ - if (arr->count > 1) { - const char *q = qualified_suffix_match(arr, callee_name); - if (q) { - return (cbm_resolution_t){q, "qualified_suffix", CONF_QUALIFIED_SUFFIX, REG_RESOLVED}; - } - } - - /* Strategy 3: unique name */ - if (arr->count == SKIP_ONE) { - if (!receiver_chain_admits(callee_name, arr->items[0])) { - return empty_result(); - } - double conf = CONF_UNIQUE_NAME; - if (import_vals && import_count > 0 && - !is_import_reachable(arr->items[0], import_vals, import_count)) { - conf *= DEFAULT_CONFIDENCE; - } - return (cbm_resolution_t){arr->items[0], "unique_name", conf, REG_RESOLVED}; - } - - /* Strategy 4: multiple candidates */ - if (import_vals && import_count > 0) { - return resolve_multi_with_imports(arr, module_qn, import_vals, import_count); - } - const char *best = best_by_import_distance((const char **)arr->items, arr->count, module_qn); - if (best) { - if (!receiver_chain_admits(callee_name, best)) { - return empty_result(); - } - double conf = candidate_count_penalty(CONF_SUFFIX_MATCH, arr->count); - return (cbm_resolution_t){best, "suffix_match", conf, arr->count}; - } - return empty_result(); -} - -cbm_resolution_t cbm_registry_resolve(const cbm_registry_t *r, const char *callee_name, - const char *module_qn, const char **import_map_keys, - const char **import_map_vals, int import_map_count) { - if (!r || !callee_name) { - return empty_result(); - } - - /* Per-file cache: same callee_name in N call sites → 1 chain walk - * + N-1 O(1) hash hits. module_qn is constant per file so the - * cache key only needs callee_name. */ - if (_resolve_cache) { - resolve_cache_entry_t *cached = - (resolve_cache_entry_t *)cbm_ht_get(_resolve_cache, callee_name); - if (cached) { - return cached->res; - } - } - - /* Split callee at the first path separator: "pkg.Func" → prefix="pkg", - * suffix="Func". Rust/C++ use "::" rather than ".", so honor whichever - * separator appears first ("lib::square" → prefix="lib", suffix="square"). - * Both separators are unambiguous, so handling "::" never affects "."-only - * callees. */ - char prefix[CBM_SZ_256] = {0}; - const char *suffix = NULL; - const char *dot = strchr(callee_name, '.'); - const char *colons = strstr(callee_name, "::"); - const char *sep = dot; - size_t sep_len = SKIP_ONE; /* length of '.' */ - if (colons && (!sep || colons < sep)) { - sep = colons; - sep_len = 2; /* length of "::" */ - } - if (sep) { - size_t plen = sep - callee_name; - if (plen >= sizeof(prefix)) { - plen = sizeof(prefix) - SKIP_ONE; - } - memcpy(prefix, callee_name, plen); - prefix[plen] = '\0'; - suffix = sep + sep_len; - } else { - snprintf(prefix, sizeof(prefix), "%s", callee_name); - } - - /* Strategy 1: import map */ - cbm_resolution_t res = - resolve_import_map(r, prefix, suffix, import_map_keys, import_map_vals, import_map_count); - if (!(res.qualified_name && res.qualified_name[0])) { - /* Strategy 2: same module */ - res = resolve_same_module(r, callee_name, suffix, module_qn); - } - if (!(res.qualified_name && res.qualified_name[0])) { - /* Strategy 3+4: name lookup */ - res = resolve_name_lookup(r, callee_name, module_qn, import_map_vals, import_map_count); - } - - /* Cache the result (including empty — caching the negative answer - * is just as valuable; same name asks the same question). */ - if (_resolve_cache) { - resolve_cache_entry_t *e = (resolve_cache_entry_t *)malloc(sizeof(*e)); - if (e) { - e->res = res; - char *kdup = strdup(callee_name); - if (kdup) { - cbm_ht_set(_resolve_cache, kdup, e); - } else { - free(e); - } - } - } - return res; -} - -/* ── Fuzzy Resolve ──────────────────────────────────────────────── */ - -/* Filter candidates by import reachability. Returns count of reachable. */ -static int filter_import_reachable(const char **candidates, int count, const char **import_vals, - int import_count, const char **out, int max_out) { - int n = 0; - for (int i = 0; i < count && n < max_out; i++) { - if (is_import_reachable(candidates[i], import_vals, import_count)) { - out[n++] = candidates[i]; - } - } - return n; -} - -cbm_fuzzy_result_t cbm_registry_fuzzy_resolve(const cbm_registry_t *r, const char *callee_name, - const char *module_qn, const char **import_map_keys, - const char **import_map_vals, int import_map_count) { - (void)import_map_keys; - cbm_fuzzy_result_t no_match = {{0}, false}; - if (!r || !callee_name) { - return no_match; - } - - /* Extract simple name (last segment after dots) */ - const char *lookup = simple_name(callee_name); - qn_array_t *arr = cbm_ht_get(r->by_name, lookup); - if (!arr || arr->count == 0) { - return no_match; - } - if (arr->count > REG_MAX_CANDIDATES) { - return no_match; /* unresolvably ambiguous — see REG_MAX_CANDIDATES */ - } - - bool have_imports = (import_map_vals && import_map_count > 0); - - /* Single candidate */ - if (arr->count == SKIP_ONE) { - double conf = CONF_FUZZY_SINGLE; - if (have_imports && - !is_import_reachable(arr->items[0], import_map_vals, import_map_count)) { - conf *= DEFAULT_CONFIDENCE; - } - return (cbm_fuzzy_result_t){{arr->items[0], "fuzzy", conf, REG_RESOLVED}, true}; - } - - /* Multiple candidates: filter by import reachability */ - const char *filtered[CBM_SZ_256]; - int fcount = arr->count; - const char **fptr = (const char **)arr->items; - - if (have_imports) { - fcount = filter_import_reachable((const char **)arr->items, arr->count, import_map_vals, - import_map_count, filtered, CBM_SZ_256); - fptr = filtered; - } - - if (fcount == 0) { - /* No import-reachable — use originals with penalty */ - const char *best = - best_by_import_distance((const char **)arr->items, arr->count, module_qn); - if (!best) { - return no_match; - } - return (cbm_fuzzy_result_t){ - {best, "fuzzy", - candidate_count_penalty(CONF_FUZZY_MULTI * REG_HALF_PENALTY, arr->count), arr->count}, - true}; - } - if (fcount == SKIP_ONE) { - return (cbm_fuzzy_result_t){ - {fptr[0], "fuzzy", candidate_count_penalty(CONF_FUZZY_SINGLE, arr->count), arr->count}, - true}; - } - const char *best = best_by_import_distance(fptr, fcount, module_qn); - if (!best) { - return no_match; - } - return (cbm_fuzzy_result_t){ - {best, "fuzzy", candidate_count_penalty(CONF_FUZZY_MULTI, fcount), fcount}, true}; -} - -/* ── FindEndingWith ─────────────────────────────────────────────── */ - -struct few_ctx { - const char *target; /* ".suffix" */ - size_t target_len; - const char **results; - int count; - int cap; -}; - -static void few_scan(const char *key, void *value, void *ud) { - (void)value; - struct few_ctx *ctx = ud; - size_t klen = strlen(key); - if (klen >= ctx->target_len && strcmp(key + klen - ctx->target_len, ctx->target) == 0) { - if (ctx->count >= ctx->cap) { - ctx->cap = ctx->cap ? ctx->cap * PAIR_LEN : REG_INIT_CAP; - ctx->results = safe_realloc(ctx->results, (size_t)ctx->cap * sizeof(char *)); - } - ctx->results[ctx->count++] = key; - } -} - -int cbm_registry_find_ending_with(const cbm_registry_t *r, const char *suffix, const char ***out) { - if (!r || !suffix || !out) { - if (out) { - *out = NULL; - } - return 0; - } - - /* Build ".suffix" target */ - size_t slen = strlen(suffix); - char *target = malloc(slen + REG_SUFFIX_ALLOC); - target[0] = '.'; - memcpy(target + SKIP_ONE, suffix, slen + SKIP_ONE); - - struct few_ctx ctx = {target, slen + SKIP_ONE, NULL, 0, 0}; - cbm_ht_foreach(r->exact, few_scan, &ctx); - - free(target); - *out = ctx.results; - return ctx.count; -} - -/* Public wrapper for is_import_reachable (testing). */ -bool cbm_registry_is_import_reachable(const char *candidate_qn, const char **import_vals, - int import_count) { - return is_import_reachable(candidate_qn, import_vals, import_count); -} - -/* ── Class-qualified route handlers (PHP, #1146) ─────────────────── */ - -/* Method QNs are path-based: ..... - * A PHP class is named by its namespace instead, so a "Ns\\Class::method" - * handler is placed on a QN by what places the class file itself. */ -#define CONF_PSR4_MEMBER 0.95 -#define CONF_NAMESPACE_PATH_MEMBER 0.90 - -static bool seg_eq_nocase(const char *a, size_t alen, const char *b, size_t blen) { - if (alen != blen) { - return false; - } - for (size_t i = 0; i < alen; i++) { - char ca = (char)((a[i] >= 'A' && a[i] <= 'Z') ? a[i] - 'A' + 'a' : a[i]); - char cb = (char)((b[i] >= 'A' && b[i] <= 'Z') ? b[i] - 'A' + 'a' : b[i]); - if (ca != cb) { - return false; - } - } - return true; -} - -/* Does qn end with ".."? Returns the offset of that tail or -1. */ -static int qn_class_member_tail(const char *qn, const char *cls, size_t cls_len, - const char *member) { - size_t qlen = strlen(qn); - size_t mlen = strlen(member); - if (qlen < cls_len + mlen + PAIR_LEN) { - return CBM_NOT_FOUND; - } - const char *m = qn + qlen - mlen; - const char *c = m - SKIP_ONE - cls_len; - if (strcmp(m, member) != 0 || m[-SKIP_ONE] != '.' || strncmp(c, cls, cls_len) != 0 || - c[-SKIP_ONE] != '.') { - return CBM_NOT_FOUND; - } - return (int)(c - SKIP_ONE - qn); -} - -/* PSR-4: the class file Composer's autoloader loads for the FQN, the class it - * defines, and that class's member — through the PSR-4 import resolver - * (cbm_pipeline_psr4_member_qn), so a root whose folder does not mirror its - * namespace (Acme\\Blog\\ -> packages/blog/src) places the class as well. - * Returns its tri-state outcome; *out_qn is the member QN when RESOLVED. */ -static cbm_psr4_member_t psr4_member_qn(const cbm_gbuf_t *gbuf, const char *fqn, size_t fqn_len, - const char *member, const char **out_qn) { - char cls_fqn[CBM_SZ_512]; - *out_qn = NULL; - if (!gbuf || fqn_len >= sizeof(cls_fqn)) { - return CBM_PSR4_MEMBER_NOT_COVERED; - } - memcpy(cls_fqn, fqn, fqn_len); - cls_fqn[fqn_len] = '\0'; - return cbm_pipeline_psr4_member_qn(gbuf, cls_fqn, member, out_qn); -} - -/* Do the directory segments of qn (between the project root segment and the - * file stem, i.e. before offset tail = "..") end with every - * segment of the namespace fqn[0, ns_len)? Case-insensitive: App\\Http is - * conventionally app/Http. An empty namespace is trivially aligned. */ -static bool namespace_matches_dirs(const char *qn, int tail, const char *fqn, size_t ns_len) { - const char *root_dot = strchr(qn, '.'); - const char *stem_end = qn + tail; - if (!root_dot || stem_end <= root_dot) { - return false; - } - const char *dir_end = stem_end - SKIP_ONE; /* back over the file stem */ - while (*dir_end != '.') { - dir_end--; - } - size_t ns_end = ns_len; - while (ns_end > 0) { - size_t ns_start = ns_end; - while (ns_start > 0 && fqn[ns_start - SKIP_ONE] != '\\') { - ns_start--; - } - if (dir_end <= root_dot) { - return false; /* more namespace segments than directories */ - } - const char *dir_start = dir_end; - while (dir_start[-SKIP_ONE] != '.') { - dir_start--; - } - if (!seg_eq_nocase(fqn + ns_start, ns_end - ns_start, dir_start, - (size_t)(dir_end - dir_start))) { - return false; - } - dir_end = dir_start - SKIP_ONE; - ns_end = ns_start > 0 ? ns_start - SKIP_ONE : 0; - } - return true; -} - -/* Namespace/path alignment: the one "." candidate that sits - * where its namespace says. NULL when none or several do. */ -static const char *aligned_member_qn(const cbm_registry_t *r, const char *fqn, size_t ns_len, - const char *cls, size_t cls_len, const char *member) { - const qn_array_t *arr = cbm_ht_get(r->by_name, member); - const char *match = NULL; - for (int i = 0; arr && i < arr->count; i++) { - const char *qn = arr->items[i]; - int tail = qn_class_member_tail(qn, cls, cls_len, member); - if (tail == CBM_NOT_FOUND || !namespace_matches_dirs(qn, tail, fqn, ns_len)) { - continue; - } - if (match) { - return NULL; /* ambiguous */ - } - match = qn; - } - return match; -} - -cbm_resolution_t cbm_registry_resolve_handler(const cbm_registry_t *r, const char *handler_ref, - const char *module_qn, const char **import_map_keys, - const char **import_map_vals, int import_map_count, - const cbm_gbuf_t *gbuf) { - const char *sep = handler_ref ? strstr(handler_ref, "::") : NULL; - if (!r || !sep) { - return cbm_registry_resolve(r, handler_ref, module_qn, import_map_keys, import_map_vals, - import_map_count); - } - const char *member = sep + PAIR_LEN; - size_t fqn_len = (size_t)(sep - handler_ref); - size_t ns_len = fqn_len; - while (ns_len > 0 && handler_ref[ns_len - SKIP_ONE] != '\\') { - ns_len--; - } - const char *cls = handler_ref + ns_len; - size_t cls_len = fqn_len - ns_len; - ns_len = ns_len > 0 ? ns_len - SKIP_ONE : 0; /* drop the separator before the class */ - if (cls_len == 0 || !member[0]) { - return empty_result(); - } - const char *qn = NULL; - switch (psr4_member_qn(gbuf, handler_ref, fqn_len, member, &qn)) { - case CBM_PSR4_MEMBER_RESOLVED: - return (cbm_resolution_t){qn, "php_psr4", CONF_PSR4_MEMBER, REG_RESOLVED}; - case CBM_PSR4_MEMBER_UNRESOLVED: - /* A covered class lives where PSR-4 says or nowhere; a namespace- - * mirroring folder elsewhere would be a guessed edge (#1186). */ - return empty_result(); - case CBM_PSR4_MEMBER_NOT_COVERED: - break; - } - qn = aligned_member_qn(r, handler_ref, ns_len, cls, cls_len, member); - if (qn) { - return (cbm_resolution_t){qn, "php_namespace_path", CONF_NAMESPACE_PATH_MEMBER, - REG_RESOLVED}; - } - return empty_result(); -} diff --git a/src/pipeline/registry.zig b/src/pipeline/registry.zig new file mode 100644 index 00000000..179f6bdc --- /dev/null +++ b/src/pipeline/registry.zig @@ -0,0 +1,1360 @@ +//! registry.zig — Function/Method/Class registry for call resolution. +//! +//! Implements the registry functions of src/pipeline/pipeline.h. Indexes all +//! callable symbols by qualified name and simple name. Resolution uses a +//! prioritized strategy chain: +//! 1. Import map lookup +//! 2. Same-module match +//! 3. Unique name (single candidate project-wide) +//! 4. Suffix match with import distance scoring + +const std = @import("std"); +const builtin = @import("builtin"); +const c = @import("c"); +const interop = @import("../foundation/interop.zig"); +const util = @import("util.zig"); + +const CStr = util.CStr; +const MutStr = util.MutStr; +const ix = util.ix; +const Resolution = c.cbm_resolution_t; +const HashTable = c.CBMHashTable; + +const REG_INIT_CAP = 16; +const REG_MIN_CANDIDATES = 3; +const REG_RESOLVED = 1; +const REG_SUFFIX_ALLOC = 2; +/// Names with more registered definitions than this are unresolvable by name +/// alone: candidateCountPenalty already floors their confidence to ~3/count +/// (<= 0.006 at 256), so the emitted edge is noise — while the candidate walk +/// (reachability + scoring per candidate, re-done per file) is the dominant +/// resolution cost on identifier-dense repos. On the Linux kernel, 274 names +/// exceed 256 candidates ("list_head" 7188, "flags" 5520, "dev" 4374, ...) and +/// accounted for ~900 s of the 987 s usage-resolution CPU. Bail out early. +const REG_MAX_CANDIDATES = 256; +const REG_FULL_CONF: f64 = 1.0; +const REG_HALF_PENALTY: f64 = 0.5; + +const DEFAULT_CONFIDENCE: f64 = 0.5; + +/// Confidence score → human-readable band label. +const CONF_BAND_HIGH: f64 = 0.7; +const CONF_BAND_MEDIUM: f64 = 0.45; +const CONF_BAND_SPECULATIVE: f64 = 0.25; + +pub export fn cbm_confidence_band(score: f64) CStr { + if (score >= CONF_BAND_HIGH) return "high"; + if (score >= CONF_BAND_MEDIUM) return "medium"; + if (score >= CONF_BAND_SPECULATIVE) return "speculative"; + return ""; +} + +// ── Resolution confidence scores ──────────────────────────────────── +/// Strategy 1: import_map — direct import → high confidence +const CONF_IMPORT_MAP: f64 = 0.95; +const CONF_IMPORT_MAP_SUFFIX: f64 = 0.85; +/// Strategy 2: same_module — same file/package → high confidence +const CONF_SAME_MODULE: f64 = 0.90; +/// Strategy 3: unique_name — only one candidate project-wide +const CONF_UNIQUE_NAME: f64 = 0.75; +/// Strategy 4: suffix_match — multiple candidates, filtered +const CONF_SUFFIX_MATCH: f64 = 0.55; +/// Fuzzy fallback: lower confidence +const CONF_FUZZY_SINGLE: f64 = 0.40; +const CONF_FUZZY_MULTI: f64 = 0.30; +/// Candidate count penalty cap +const CANDIDATE_PENALTY_CAP: f64 = 3.0; + +// ── Internal types ────────────────────────────────────────────────── + +/// Array of QN strings for byName index +const QnArray = util.DynArray(MutStr); + +const LABEL_POOL_CAP = 64; + +const Registry = extern struct { + /// Interned label strings (<=~30 distinct labels; owned here, freed in + /// _free). The exact map's VALUES point into this pool instead of one + /// strdup per registered definition (~8.5M strdups on the kernel). + label_pool: [LABEL_POOL_CAP]MutStr, + label_pool_n: c_int, + /// exact: qualifiedName → label string (heap-owned copies) + exact: ?*HashTable, + + /// byName: simpleName → QnArray* (heap-owned) + by_name: ?*HashTable, +}; + +fn byName(r: *const Registry, name: CStr) ?*QnArray { + return @ptrCast(@alignCast(c.cbm_ht_get(r.by_name, name))); +} + +// ── Helpers ───────────────────────────────────────────────────────── + +/// Extract the last path segment from a QN. Returns pointer into s. +/// Recognizes both '.' (most langs) and Rust/C++ '::' separators, so a +/// scoped callee like "lib::square" yields "square" rather than the whole +/// scoped path (which never matches the by-name index). +fn simpleName(qn: CStr) CStr { + const dot: CStr = c.strrchr(qn, '.'); + var seg: CStr = if (dot != null) dot + 1 else qn; + // Find the last "::" and, if it sits after the last '.', use the + // segment following it. + var colons: CStr = null; + var p: CStr = c.strstr(qn, "::"); + while (p != null) : (p = c.strstr(p + 2, "::")) { + colons = p; + } + if (colons != null and @intFromPtr(colons + 2) > @intFromPtr(seg)) { + seg = colons + 2; + } + return seg; +} + +/// Count common dot-separated prefix segments. +fn commonPrefixLen(a_in: CStr, b_in: CStr) c_int { + if (a_in == null or b_in == null) return 0; + var a = a_in; + var b = b_in; + var count: c_int = 0; + while (a[0] != 0 and b[0] != 0) { + // Find next segment in each + const adot: CStr = c.strchr(a, '.'); + const bdot: CStr = c.strchr(b, '.'); + const alen: usize = if (adot != null) util.ptrDiff(adot, a) else c.strlen(a); + const blen: usize = if (bdot != null) util.ptrDiff(bdot, b) else c.strlen(b); + if (alen != blen or c.memcmp(a, b, alen) != 0) break; + count += 1; + a += alen + @as(usize, if (adot != null) 1 else 0); + b += blen + @as(usize, if (bdot != null) 1 else 0); + if (adot == null or bdot == null) break; + } + return count; +} + +const REG_TEST_PENALTY = 1000; + +/// Check if a qualified name looks like a test/mock path. +fn isTestQn(qn: CStr) bool { + if (qn == null) return false; + return c.strstr(qn, "Test") != null or c.strstr(qn, "test") != null or + c.strstr(qn, "Mock") != null or c.strstr(qn, "mock") != null or + c.strstr(qn, "Stub") != null or c.strstr(qn, "stub") != null or + c.strstr(qn, "Fake") != null or c.strstr(qn, "fake") != null or + c.strstr(qn, "Fixture") != null or c.strstr(qn, "spec") != null; +} + +/// Score a candidate for tiebreaking. Higher = better. +/// Layer 1: Non-test code preferred over test code (+1000) +/// Layer 2: Namespace proximity via common prefix length (+plen) +fn candidateScore(candidate_qn: CStr, module_qn: CStr) c_int { + var score: c_int = 0; + if (!isTestQn(candidate_qn)) score += REG_TEST_PENALTY; + score += commonPrefixLen(candidate_qn, module_qn); + return score; +} + +/// Pick candidate with highest composite score (test-deprioritization + namespace proximity). +fn bestByImportDistance(candidates: [*c]const CStr, count: c_int, module_qn: CStr) CStr { + var best: CStr = null; + var best_score: c_int = c.CBM_NOT_FOUND; + var i: usize = 0; + while (i < count) : (i += 1) { + const score = candidateScore(candidates[i], module_qn); + if (score > best_score) { + best_score = score; + best = candidates[i]; + } + } + return best; +} + +// ── Per-file is_import_reachable memoization cache ────────────────── +// +// The hot path on kubernetes spends ~50% of resolve_calls CPU in +// isImportReachable's O(candidates × imports × strstr) scan. The +// SAME candidate_qn is re-evaluated dozens of times per file because +// the same callee_name appears in multiple call sites and each lookup +// re-checks all candidates for that name. +// +// Cache life-cycle is per-FILE (because import_vals changes between +// files). resolve_file_calls calls _begin at file entry and _end at +// file exit. Thread-local so each worker has its own cache without +// contention. +threadlocal var reach_cache: ?*HashTable = null; + +/// Sentinels stored as values in the cache. NULL means "not cached". +/// We need two distinct non-NULL pointers to encode true/false. +const REACH_CACHE_TRUE: ?*anyopaque = @ptrFromInt(1); +const REACH_CACHE_FALSE: ?*anyopaque = @ptrFromInt(2); + +fn reachCacheFreeKey(key: CStr, val: ?*anyopaque, ud: ?*anyopaque) callconv(.c) void { + _ = val; + _ = ud; + c.free(@constCast(key)); +} + +pub export fn cbm_registry_reach_cache_begin(estimated_capacity_in: c_int) void { + var estimated_capacity = estimated_capacity_in; + if (reach_cache) |cache| { + // Defensive: caller forgot to call _end. Clear and reuse. + c.cbm_ht_foreach(cache, reachCacheFreeKey, null); + c.cbm_ht_clear(cache); + return; + } + if (estimated_capacity < 16) estimated_capacity = 16; + reach_cache = c.cbm_ht_create(@intCast(estimated_capacity)); +} + +pub export fn cbm_registry_reach_cache_end() void { + const cache = reach_cache orelse return; + c.cbm_ht_foreach(cache, reachCacheFreeKey, null); + c.cbm_ht_free(cache); + reach_cache = null; +} + +// ── Per-file import-map prefix → module-QN hash ───────────────────── +// +// resolveImportMap does a linear strcmp scan over the per-file +// import list for every call (and every usage, throw, rw, def). On +// kubernetes typical files have 20-30 imports and ~200 calls. That's +// 4000-6000 strcmps per file × 20K files = ~80M strcmps. +// +// Build a hash from prefix → module_qn ONCE per file at file entry. +// Each lookup becomes O(1). Keys and values are BORROWED from the +// caller's import_map — they outlive the cache scope. +threadlocal var import_map_cache: ?*HashTable = null; + +pub export fn cbm_registry_import_map_cache_begin(keys: [*c]CStr, vals: [*c]CStr, count: c_int) void { + if (import_map_cache) |cache| { + c.cbm_ht_free(cache); + import_map_cache = null; + } + if (keys == null or vals == null or count <= 0) return; + import_map_cache = c.cbm_ht_create(@as(u32, @intCast(count)) *% 2 +% 8); + const cache = import_map_cache orelse return; + for (0..ix(count)) |i| { + if (keys[i] != null and vals[i] != null) { + _ = c.cbm_ht_set(cache, keys[i], @ptrCast(@constCast(vals[i]))); + } + } +} + +pub export fn cbm_registry_import_map_cache_end() void { + const cache = import_map_cache orelse return; + // Keys/values borrowed from caller — no free callback needed. + c.cbm_ht_free(cache); + import_map_cache = null; +} + +// ── Per-file full-result cache for cbm_registry_resolve ───────────── +// +// THE big one: 98.7% of resolve_calls CPU on kubernetes lives inside +// cbm_registry_resolve's strategy chain (suffix_match iterating +// hundreds of candidates per common name like "Get"/"Add"/"New" +// × 1M LSP-miss calls = ~800s of work). +// +// Since module_qn is CONSTANT per file, the resolution for a given +// callee_name within a file is also constant — same name resolves +// to the same QN every time. Cache the full cbm_resolution_t result +// keyed by callee_name; first lookup does the full chain, repeats +// are O(1). On K8s a typical file has ~200 calls but only ~50 +// unique callee_names → ~75% hit rate → ~75% of the resolve cost +// eliminated. +const ResolveCacheEntry = extern struct { + res: Resolution, +}; + +/// Entries are malloc'd; keys are strdup'd. Both freed in _end. +threadlocal var resolve_cache: ?*HashTable = null; + +fn resolveCacheFreeEntry(key: CStr, val: ?*anyopaque, ud: ?*anyopaque) callconv(.c) void { + _ = ud; + c.free(@constCast(key)); + c.free(val); +} + +pub export fn cbm_registry_resolve_cache_begin(estimated_capacity_in: c_int) void { + var estimated_capacity = estimated_capacity_in; + if (resolve_cache) |cache| { + c.cbm_ht_foreach(cache, resolveCacheFreeEntry, null); + c.cbm_ht_free(cache); + resolve_cache = null; + } + if (estimated_capacity < 32) estimated_capacity = 32; + resolve_cache = c.cbm_ht_create(@intCast(estimated_capacity)); +} + +pub export fn cbm_registry_resolve_cache_end() void { + const cache = resolve_cache orelse return; + c.cbm_ht_foreach(cache, resolveCacheFreeEntry, null); + c.cbm_ht_free(cache); + resolve_cache = null; +} + +/// Check if candidate's module prefix appears in import map values. +/// Uses stack buffer to avoid malloc/free per call in hot resolution loop. +/// Per-file memoization via TLS cache: repeated lookups of the same +/// candidate_qn (same name appears in many call sites) become O(1) +/// after the first computation. +fn isImportReachable(candidate_qn: CStr, import_vals: [*c]const CStr, import_count: c_int) bool { + if (reach_cache) |cache| { + const cached = c.cbm_ht_get(cache, candidate_qn); + if (cached == REACH_CACHE_TRUE) return true; + if (cached == REACH_CACHE_FALSE) return false; + } + + var cand_mod: [c.CBM_SZ_512]u8 = undefined; + const last: CStr = c.strrchr(candidate_qn, '.'); + if (last != null) { + var len = util.ptrDiff(last, candidate_qn); + if (len >= cand_mod.len) len = cand_mod.len - 1; + @memcpy(cand_mod[0..len], candidate_qn[0..len]); + cand_mod[len] = 0; + } else { + _ = c.snprintf(&cand_mod, cand_mod.len, "%s", candidate_qn); + } + var reachable = false; + var i: usize = 0; + while (i < import_count) : (i += 1) { + if (c.strstr(&cand_mod, import_vals[i]) != null or c.strstr(import_vals[i], &cand_mod) != null) { + reachable = true; + break; + } + } + + if (reach_cache) |cache| { + const kdup: MutStr = c.strdup(candidate_qn); + if (kdup != null) { + _ = c.cbm_ht_set(cache, kdup, if (reachable) REACH_CACHE_TRUE else REACH_CACHE_FALSE); + } + } + return reachable; +} + +/// Scale confidence inversely with candidate count. +fn candidateCountPenalty(base: f64, count: c_int) f64 { + if (count <= REG_MIN_CANDIDATES) return base; + return base * @min(REG_FULL_CONF, CANDIDATE_PENALTY_CAP / @as(f64, @floatFromInt(count))); +} + +fn emptyResult() Resolution { + return std.mem.zeroes(Resolution); +} + +fn resolved(qn: CStr, strategy: CStr, confidence: f64, candidate_count: c_int) Resolution { + return .{ + .qualified_name = qn, + .strategy = strategy, + .confidence = confidence, + .candidate_count = candidate_count, + }; +} + +// ── Perl builtin guard (#459 follow-up: call-graph noise) ─────────── +// Curated subset of perlfunc core builtins. When a Perl CALL resolves +// only by the generic short-name matcher (no LSP, no import, after the +// same-module/name-lookup chain), a builtin name like `push`/`shift`/ +// `keys` must NOT be wired to a project sub that merely shares the name +// — that is virtually always a false positive. A genuine intra-project +// call is resolved by earlier (LSP/textual) stages before this guard. +// MUST stay sorted ASCII-ascending for bsearch. +const PERL_BUILTINS = [_][*:0]const u8{ + "abs", "atan2", "binmode", "bless", "caller", "chdir", "chmod", "chomp", + "chop", "chown", "chr", "chroot", "close", "closedir", "cos", "defined", + "delete", "die", "do", "each", "eof", "eval", "exec", "exists", + "exit", "fork", "gmtime", "goto", "grep", "hex", "index", "int", + "join", "keys", "last", "lc", "lcfirst", "length", "local", "localtime", + "log", "lstat", "map", "mkdir", "my", "next", "oct", "open", + "opendir", "ord", "our", "pop", "pos", "print", "printf", "push", + "quotemeta", "rand", "read", "readdir", "readline", "redo", "ref", "rename", + "require", "return", "reverse", "rindex", "rmdir", "say", "scalar", "seek", + "shift", "sin", "sleep", "sort", "splice", "split", "sprintf", "sqrt", + "srand", "stat", "substr", "system", "time", "uc", "ucfirst", "undef", + "unlink", "unshift", "values", "wantarray", "warn", "write", +}; + +fn perlBuiltinCmp(key: ?*const anyopaque, elem: ?*const anyopaque) callconv(.c) c_int { + const entry: *const [*:0]const u8 = @ptrCast(@alignCast(elem)); + return c.strcmp(@ptrCast(key), entry.*); +} + +/// True if `name` is one of the curated Perl core builtins. Used to suppress +/// generic-resolver CALLS edges from Perl builtin invocations to project subs +/// that happen to share the builtin's name. Perl-scoped: callers gate on the +/// file language so no other language's resolution is affected. +pub export fn cbm_perl_is_builtin(name: CStr) bool { + if (name == null or name[0] == 0) return false; + return c.bsearch( + name, + &PERL_BUILTINS, + PERL_BUILTINS.len, + @sizeOf([*:0]const u8), + perlBuiltinCmp, + ) != null; +} + +/// Decide whether a *resolved* Perl call edge is generic-resolver noise that +/// should be suppressed (#476). Returns true only for Perl, only for a builtin +/// invocation or a method call, and only when the registry landed the match via +/// a WEAK short-name strategy. High-confidence import/same-module strategies +/// (same_module, import_map, import_map_suffix) are KEPT so a genuine same-file +/// or imported call to a builtin-named sub still resolves — only the weak +/// short-name guesses (suffix_match, unique_name) are dropped. `strategy` is the +/// cbm_resolution_t.strategy of a non-empty match; +/// NULL/empty (no match) returns false. Pure + side-effect-free so the +/// suppression contract is unit-testable without a full pipeline. +pub export fn cbm_perl_suppress_generic_match( + is_perl: bool, + is_method: bool, + callee_name: CStr, + strategy: CStr, +) bool { + if (!is_perl) return false; + if (!(is_method or cbm_perl_is_builtin(callee_name))) return false; + if (strategy == null or strategy[0] == 0) return false; + if (util.streq(strategy, "same_module") or util.streq(strategy, "import_map") or + util.streq(strategy, "import_map_suffix")) + { + return false; // high-confidence import/same-module match — keep the genuine edge + } + return true; // weak short-name match (suffix_match / unique_name / …) → drop +} + +/// Dynamic-language analogue of the Perl guard above (#592/#606/#1276 +/// direction; precedent #477). A member call `x.foo()` reaches the weak textual +/// cascade ONLY when the language's LSP could not resolve the receiver type — +/// type-resolved calls win via lsp_* strategies before the registry runs. +/// Binding such a call to a project symbol by a weak short-name strategy +/// fabricates a CALLS edge (`re.test()` -> SalesforceRestClient.test, +/// `accelerator.print()` -> MockAccelerator.print). Drop ONLY the weak +/// strategies; keep import/same-module/qualified-tail matches and every lsp_* +/// strategy. Uses an EXPLICIT drop-list (not keep-list + default-drop) because +/// the parallel resolver runs lsp_* strategies through the same guard variable — +/// a default-drop would silently kill lsp_ts_method. Pure + side-effect-free so +/// the contract is unit-testable without a full pipeline. +/// +/// `enabled` is the CALLER's per-language gate, deliberately kept OUT of this +/// helper: the guard applies only to the language set each call site enumerates +/// (today Python plus the JS/TS family). Widening it is a +/// per-language decision made at the call site in pass_parallel. +/// +/// The weak short-name strategies that actually reach the call-resolution +/// guards: the registry's suffix_match / unique_name and the parallel +/// field_type_hint. "fuzzy" is listed as defensive insurance only — +/// cbm_registry_fuzzy_resolve is not wired into the resolver +/// today, so it never reaches these helpers, but naming it keeps a future wiring +/// from silently reintroducing the noise. Everything else — same_module / +/// import_map / import_map_suffix / qualified_suffix / callee_suffix / +/// service_pattern / lsp_* — is a receiver- or import-aware match and is KEPT. +/// +/// Shared by BOTH weak-call guards below so the drop-list exists exactly once: a +/// list that drifted between the member guard and the local-binding guard would +/// make the two disagree about what "weak" means. +fn weakShortNameStrategy(strategy: CStr) bool { + if (strategy == null or strategy[0] == 0) return false; + return util.streq(strategy, "suffix_match") or util.streq(strategy, "unique_name") or + util.streq(strategy, "field_type_hint") or util.streq(strategy, "fuzzy"); +} + +pub export fn cbm_registry_strategy_is_weak(strategy: CStr) bool { + return weakShortNameStrategy(strategy); +} + +pub export fn cbm_suppress_weak_member_match(enabled: bool, is_method: bool, strategy: CStr) bool { + if (!enabled or !is_method) return false; + return weakShortNameStrategy(strategy); +} + +/// Zig member calls are typed by the Zig LSP: mapped @import aliases and relative +/// files resolve as lsp_zig_*. A qualified_suffix match on a member call that +/// reached the registry therefore only matched spelling, e.g. `old.hello()` with +/// an unmapped `old` binding a same-named libs/old/src/old.zig. +pub export fn cbm_suppress_zig_qualified_member(is_zig: bool, is_method: bool, strategy: CStr) bool { + return is_zig and is_method and strategy != null and util.streq(strategy, "qualified_suffix"); +} + +/// Bare-call counterpart of the member guard above. A Python call `foo()` whose +/// callee identifier is bound as a parameter of an enclosing scope cannot be the +/// module-level `foo`: the parameter shadows it for the whole body. Binding such +/// a call to a project Function/Method by a weak short-name strategy fabricates +/// the edge by construction (`def _run_with_heavy_slot(run): run()` -> +/// SatoriLive.run). +/// +/// This is deliberately NOT keyed on the callee's spelling. A list of +/// "generic-looking" names (get / run / execute) asserts that certain spellings +/// are usually noise, which is a claim about corpus fashion rather than about +/// what the resolver knew — and it ages invisibly, because nothing fails when the +/// distribution shifts, the graph just quietly loses different edges. A parameter +/// binding is a fact about THIS file's scope, decidable outright. +/// +/// `enabled` is the caller's per-language gate, kept out of the helper for the +/// same reason as the member guard. Pure; unit-tested in test_registry.c. +pub export fn cbm_suppress_weak_local_binding_call( + enabled: bool, + callee_is_locally_bound: bool, + strategy: CStr, +) bool { + if (!enabled or !callee_is_locally_bound) return false; + return weakShortNameStrategy(strategy); +} + +fn jsTsFamily(lang: c.CBMLanguage) bool { + return lang == c.CBM_LANG_JAVASCRIPT or lang == c.CBM_LANG_TYPESCRIPT or lang == c.CBM_LANG_TSX; +} + +/// C and C++ are one family for cross-language checks: .h maps to CBM_LANG_CPP +/// in the extension table, so a .c file referencing a symbol declared in its +/// own header would otherwise read as a language boundary. +fn cCppFamily(lang: c.CBMLanguage) bool { + return lang == c.CBM_LANG_C or lang == c.CBM_LANG_CPP; +} + +/// cbm_path_base, but on Windows a backslash also separates directories. +fn nativePathBasename(path: CStr) CStr { + var base: CStr = c.cbm_path_base(path); + if (builtin.os.tag == .windows) { + const bslash: CStr = c.strrchr(base, '\\'); + if (bslash != null) base = bslash + 1; + } + return base; +} + +pub export fn cbm_suppress_cross_language_suffix_match( + caller_lang: c.CBMLanguage, + target_file_path: CStr, + strategy: CStr, +) bool { + // Two same-named symbols in different languages: suffix_match picks one + // winner by import-distance and attaches every bare-name call to it + // (#725, Bash/Python main, JS/Python commit). unique_name is the + // candidates==1 case (#1572) and is not this guard. + if (strategy == null or !util.streq(strategy, "suffix_match")) return false; + if (caller_lang == c.CBM_LANG_COUNT or target_file_path == null or target_file_path[0] == 0) { + return false; + } + const target_lang = c.cbm_language_for_filename(nativePathBasename(target_file_path)); + if (target_lang == c.CBM_LANG_COUNT) return false; + if (caller_lang == target_lang) return false; + if (jsTsFamily(caller_lang) and jsTsFamily(target_lang)) return false; + return true; +} + +pub export fn cbm_suppress_cross_language_ref(caller_lang: c.CBMLanguage, target_file_path: CStr) bool { + // #1928: USAGE / WRITES / READS analog of the CALLS guard above. A + // variable or field reference resolved by the short-name registry must + // not cross a language boundary: unlike CALLS, a reference edge carries + // no import-closure evidence at all — a Go test's local `event` and an + // eBPF C probe's automatic `event` share nothing but the spelling, so + // EVERY registry strategy is a bare-name guess here and none is exempt. + // LSP-backed semantic references resolve before the registry fallback + // and never reach this predicate, which is where a genuine cross-language + // binding (a future cgo resolver) would live. The JS/TS family keeps its + // exemption (.js/.ts/.d.ts pairs legitimately share symbols), and C/C++ + // count as one family (.h maps to CBM_LANG_CPP). + if (caller_lang == c.CBM_LANG_COUNT or target_file_path == null or target_file_path[0] == 0) { + return false; + } + const target_lang = c.cbm_language_for_filename(nativePathBasename(target_file_path)); + if (target_lang == c.CBM_LANG_COUNT) return false; + if (caller_lang == target_lang) return false; + if (jsTsFamily(caller_lang) and jsTsFamily(target_lang)) return false; + if (cCppFamily(caller_lang) and cCppFamily(target_lang)) return false; + return true; +} + +pub export fn cbm_go_suppress_bare_field_ref(is_go: bool, is_member_access: bool, target_label: CStr) bool { + // #1942/#1962: a bare Go identifier can never denote a struct field — + // field access is always a selector expression (x.f). The extractor + // strips the receiver before the resolver runs (resolve_lhs_write_name + // records the trailing field name; is_reference_node records the inner + // field_identifier), so the reference TEXT is always dot-less and cannot + // carry the distinction — the recorded is_member_access shape can. Only a + // reference that was never the member half of a selector is refused a + // Field bind. Go-gated: a C#/Java/C++/Python method body legitimately + // references its own members bare (cp_reads_writes_cs_static_field pins + // that shape as required), so a global veto would break those languages. + if (!is_go or is_member_access or target_label == null) return false; + return util.streq(target_label, "Field"); +} + +// ── Lifecycle ─────────────────────────────────────────────────────── + +pub export fn cbm_registry_new() ?*Registry { + const r: *Registry = @as(?*Registry, interop.callocN(Registry, 1)) orelse return null; + r.exact = c.cbm_ht_create(c.CBM_SZ_1K); + r.by_name = c.cbm_ht_create(c.CBM_SZ_512); + return r; +} + +fn freeLabel(key: CStr, value: ?*anyopaque, ud: ?*anyopaque) callconv(.c) void { + _ = ud; + _ = value; // interned in the registry's label_pool + c.free(@constCast(key)); +} + +fn freeQnArray(key: CStr, value: ?*anyopaque, ud: ?*anyopaque) callconv(.c) void { + _ = ud; + const arr_c: ?*QnArray = @ptrCast(@alignCast(value)); + if (arr_c) |arr| { + // items borrow the exact map's keys — freed there, not here + util.daFree(arr); + c.free(arr); + } + c.free(@constCast(key)); +} + +pub export fn cbm_registry_free(r_c: ?*Registry) void { + const r = r_c orelse return; + // by_name first: its items borrow exact's keys. + c.cbm_ht_foreach(r.by_name, freeQnArray, null); + c.cbm_ht_free(r.by_name); + c.cbm_ht_foreach(r.exact, freeLabel, null); + c.cbm_ht_free(r.exact); + for (r.label_pool[0..ix(r.label_pool_n)]) |label| { + c.free(label); + } + c.free(r); +} + +// ── Registration ──────────────────────────────────────────────────── + +pub export fn cbm_registry_add(r_c: ?*Registry, name: CStr, qualified_name: CStr, label: CStr) void { + _ = name; + const r = r_c orelse return; + if (qualified_name == null or label == null) return; + + // Check for duplicate + if (c.cbm_ht_get(r.exact, qualified_name) != null) return; + + // Intern the label (bounded set; linear scan is fine at this size). + var interned: CStr = null; + for (r.label_pool[0..ix(r.label_pool_n)]) |pooled| { + if (util.streq(pooled, label)) { + interned = pooled; + break; + } + } + if (interned == null and r.label_pool_n < LABEL_POOL_CAP) { + r.label_pool[ix(r.label_pool_n)] = c.strdup(label); + interned = r.label_pool[ix(r.label_pool_n)]; + r.label_pool_n += 1; + } + if (interned == null) return; // pool exhausted (cannot happen with sane label sets) + + // Store in exact map: QN → interned label. The key is the registry's ONE + // owned copy of the QN; by_name below borrows it (same lifetime) instead + // of a second strdup — this pair of copies was ~280 MB on the kernel. + _ = c.cbm_ht_set(r.exact, c.strdup(qualified_name), @ptrCast(@constCast(interned))); + const owned_qn: CStr = c.cbm_ht_get_key(r.exact, qualified_name); + + // Index by simple name. + // No array dedup needed: exact-map check above guarantees uniqueness. + const simple = simpleName(qualified_name); + var arr: ?*QnArray = byName(r, simple); + if (arr == null) { + arr = interop.callocN(QnArray, 1); + _ = c.cbm_ht_set(r.by_name, c.strdup(simple), arr); + } + util.daPush(arr.?, @constCast(owned_qn)); +} + +// ── Lookup ────────────────────────────────────────────────────────── + +pub export fn cbm_registry_exists(r_c: ?*const Registry, qn: CStr) bool { + const r = r_c orelse return false; + if (qn == null) return false; + return c.cbm_ht_get(r.exact, qn) != null; +} + +pub export fn cbm_registry_label_of(r_c: ?*const Registry, qn: CStr) CStr { + const r = r_c orelse return null; + if (qn == null) return null; + return @ptrCast(c.cbm_ht_get(r.exact, qn)); +} + +pub export fn cbm_registry_find_by_name( + r_c: ?*const Registry, + name: CStr, + out_c: ?*[*c]CStr, + count_c: ?*c_int, +) c_int { + const r = r_c orelse return c.CBM_NOT_FOUND; + const out = out_c orelse return c.CBM_NOT_FOUND; + const count = count_c orelse return c.CBM_NOT_FOUND; + const arr = byName(r, name); + if (arr != null and arr.?.count > 0) { + out.* = @ptrCast(arr.?.items); + count.* = arr.?.count; + } else { + out.* = null; + count.* = 0; + } + return 0; +} + +pub export fn cbm_registry_size(r: ?*const Registry) c_int { + return if (r) |reg| @intCast(c.cbm_ht_count(reg.exact)) else 0; +} + +// ── Resolution ────────────────────────────────────────────────────── + +/// Strategy 1: Import map lookup (exact → suffix fallback) +fn resolveImportMap( + r: *const Registry, + prefix: CStr, + suffix: CStr, + keys: [*c]const CStr, + vals: [*c]const CStr, + map_count: c_int, +) Resolution { + if (keys == null or vals == null or map_count <= 0) return emptyResult(); + + // Find prefix in import map keys. Prefer the per-file TLS hash + // cache (O(1)) over the linear scan when available. + var target: CStr = null; + if (import_map_cache) |cache| { + target = @ptrCast(c.cbm_ht_get(cache, prefix)); + } else { + for (0..ix(map_count)) |i| { + if (util.streq(keys[i], prefix)) { + target = vals[i]; + break; + } + } + } + if (target == null) return emptyResult(); + + // Build candidate: resolved.suffix or resolved.prefix. + // When the callee has a dot ("pkg.Func"), prefix="pkg" is the import key + // and suffix="Func" is the function name, so the target QN is + // resolved.Func. When the callee is bare ("requireAdmin"), prefix IS the + // function name and suffix is NULL, so the target QN must be + // resolved.requireAdmin — not just resolved, which would point at the + // module node and miss the function entirely. + // + // Direct hit only for a bare callee whose import already names the + // definition: `from m import f as g; g()` must reach m.f, not invent m.g + // (upstream #875/#979, 7fda9f71). With a suffix (`imported.method()`) a + // direct hit would swallow the suffix and bind the call to the imported + // symbol itself (upstream #1000). + const has_suffix = suffix != null and suffix[0] != 0; + if (!has_suffix) { + const direct: CStr = c.cbm_ht_get_key(r.exact, target); + if (direct != null) { + return resolved(direct, "import_map", CONF_IMPORT_MAP, REG_RESOLVED); + } + } + var candidate: [c.CBM_SZ_512]u8 = undefined; + if (has_suffix) { + _ = c.snprintf(&candidate, candidate.len, "%s.%s", target, suffix); + } else { + _ = c.snprintf(&candidate, candidate.len, "%s.%s", target, prefix); + } + // Use cbm_ht_get_key to get the persistent heap-owned key string + const stored_key: CStr = c.cbm_ht_get_key(r.exact, &candidate); + if (stored_key != null) { + return resolved(stored_key, "import_map", CONF_IMPORT_MAP, REG_RESOLVED); + } + + // import_map_suffix fallback: find a QN starting with resolved+"." and + // ending with "."+suffix. Any such QN's last segment equals the last + // segment of suffix, so probe the by_name index and tail-check the (few) + // candidates instead of cbm_ht_foreach over the WHOLE exact table — that + // scan ran per unresolved call and dominated elasticsearch's resolve + // phase (94% of samples: 700k-entry foreach + strlen per entry). + if (has_suffix) { + var resolved_dot: [c.CBM_SZ_512]u8 = undefined; + var dot_suffix: [c.CBM_SZ_256]u8 = undefined; + _ = c.snprintf(&resolved_dot, resolved_dot.len, "%s.", target); + _ = c.snprintf(&dot_suffix, dot_suffix.len, ".%s", suffix); + if (byName(r, simpleName(suffix))) |arr| { + const rd_len = c.strlen(&resolved_dot); + const ds_len = c.strlen(&dot_suffix); + for (0..ix(arr.count)) |i| { + const qn: CStr = arr.items[i]; + const klen = c.strlen(qn); + if (klen >= rd_len + ds_len and c.strncmp(qn, &resolved_dot, rd_len) == 0 and + util.streq(qn + klen - ds_len, &dot_suffix)) + { + return resolved(qn, "import_map_suffix", CONF_IMPORT_MAP_SUFFIX, REG_RESOLVED); + } + } + } + } + return emptyResult(); +} + +/// Strategy 2: Same-module match +fn resolveSameModule(r: *const Registry, callee_name: CStr, suffix: CStr, module_qn: CStr) Resolution { + var candidate: [c.CBM_SZ_512]u8 = undefined; + _ = c.snprintf(&candidate, candidate.len, "%s.%s", module_qn, callee_name); + var stored_key: CStr = c.cbm_ht_get_key(r.exact, &candidate); + if (stored_key != null) { + return resolved(stored_key, "same_module", CONF_SAME_MODULE, REG_RESOLVED); + } + if (suffix != null and suffix[0] != 0) { + _ = c.snprintf(&candidate, candidate.len, "%s.%s", module_qn, suffix); + stored_key = c.cbm_ht_get_key(r.exact, &candidate); + if (stored_key != null) { + return resolved(stored_key, "same_module", CONF_SAME_MODULE, REG_RESOLVED); + } + } + return emptyResult(); +} + +/// Strategy 4: multiple candidates with import filtering. +fn resolveMultiWithImports( + arr: *const QnArray, + module_qn: CStr, + import_vals: [*c]const CStr, + import_count: c_int, +) Resolution { + var filtered: [c.CBM_SZ_256]CStr = undefined; + var fcount: c_int = 0; + var i: usize = 0; + while (i < arr.count and fcount < c.CBM_SZ_256) : (i += 1) { + if (isImportReachable(arr.items[i], import_vals, import_count)) { + filtered[ix(fcount)] = arr.items[i]; + fcount += 1; + } + } + if (fcount == 1) { + const conf = candidateCountPenalty(CONF_SUFFIX_MATCH, arr.count); + return resolved(filtered[0], "suffix_match", conf, arr.count); + } + if (fcount > 1) { + const best = bestByImportDistance(&filtered, fcount, module_qn); + if (best != null) { + const conf = candidateCountPenalty(CONF_SUFFIX_MATCH, fcount); + return resolved(best, "suffix_match", conf, fcount); + } + } + // No import-reachable — use all candidates with penalty + const best = bestByImportDistance(@ptrCast(arr.items), arr.count, module_qn); + if (best != null) { + const conf = candidateCountPenalty(CONF_SUFFIX_MATCH * REG_HALF_PENALTY, arr.count); + return resolved(best, "suffix_match", conf, arr.count); + } + return emptyResult(); +} + +/// Confidence for a full qualified-tail match (Strategy 3.5). A package- or +/// namespace-qualified callee that uniquely matches one candidate's full tail is +/// as trustworthy as a same-module hit. +const CONF_QUALIFIED_SUFFIX: f64 = 0.90; + +/// Copy callee_name into dotted with every "::" turned into ".", so the tail +/// composes with dotted candidate QNs. Returns the length written. +fn normalizeScopeSeparators(callee_name: CStr, dotted: *[c.CBM_SZ_512]u8) usize { + var w: usize = 0; + var s: [*]const u8 = callee_name; + while (s[0] != 0 and w + 1 < dotted.len) { + if (s[0] == ':' and s[1] == ':') { + dotted[w] = '.'; + s += 2; + } else { + dotted[w] = s[0]; + s += 1; + } + w += 1; + } + dotted[w] = 0; + return w; +} + +/// When a callee is package/namespace-qualified (Foo::Bar::sub or Foo.Bar.sub), +/// disambiguate among same-simple-name candidates by matching the FULL qualified +/// tail against each candidate QN at a segment boundary. Returns the sole +/// candidate whose QN equals or ends with ".", or NULL when zero +/// or several candidates match (the caller then falls back to bare-name scoring). +/// +/// Fixes qualified cross-file calls collapsing onto one namespace when the bare +/// symbol name is defined in several — e.g. Perl's Foo::Bar::run and Foo::Baz::run +/// both reduce to "run", so the bare-name scorer would route every caller to a +/// single winner. Language agnostic: callees with no separator return NULL and +/// leave behavior unchanged. +fn qualifiedSuffixMatch(arr: *const QnArray, callee_name: CStr) CStr { + // Normalize "::" → "." so the tail composes with dotted candidate QNs. + var dotted: [c.CBM_SZ_512]u8 = undefined; + const w = normalizeScopeSeparators(callee_name, &dotted); + // Must be qualified (contain a '.') — a bare name matches every candidate + // and carries no disambiguating signal. + if (c.strchr(&dotted, '.') == null) return null; + var match: CStr = null; + for (0..ix(arr.count)) |i| { + const qn: CStr = arr.items[i]; + const qlen = c.strlen(qn); + if (qlen < w) continue; + const tail = qn + (qlen - w); + if (!util.streq(tail, &dotted)) continue; + // Segment boundary: tail is the whole QN or is preceded by '.'. + if (tail != qn and (tail - 1)[0] != '.') continue; + if (match != null) return null; // ambiguous — more than one qualified tail matches + match = qn; + } + return match; +} + +/// A dotted callee whose FIRST segment starts upper-case names a type — URLSession, +/// Calendar, JSONEncoder. That receiver chain is evidence the bare-name scorers +/// throw away, and throwing it away binds Foundation's URLSession.shared.data to +/// a project's own PickedFile.data: high confidence, and nothing in the graph +/// shows it is wrong. Require instead that the candidate's own parent segment +/// appears somewhere in the chain. Calendar.utcGregorian.startOfDayUTC resolving +/// to AuthDTOs.Calendar.startOfDayUTC passes, because Calendar is in the chain. +/// +/// Only an upper-case first segment is guarded. A lower-case root names a value +/// (vm.load, http.Get, os.path.join) whose declared type the chain does not +/// show, so the chain proves nothing there and the call passes through +/// unchanged. A callee with no separator passes through as well. +/// +/// Language agnostic by design: the registry holds no language, and every +/// language that writes receiver chains gains the same protection. +fn receiverChainAdmits(callee_name: CStr, candidate_qn: CStr) bool { + // Normalize "::" -> "." so the chain composes with dotted candidate QNs, + // the same way qualifiedSuffixMatch does. + var dotted: [c.CBM_SZ_512]u8 = undefined; + _ = normalizeScopeSeparators(callee_name, &dotted); + + const last_dot: CStr = c.strrchr(&dotted, '.'); + if (last_dot == null) return true; // bare name — no receiver chain to judge + if (dotted[0] < 'A' or dotted[0] > 'Z') return true; // lower-case root names a value, not a type + // A name written in capitals with underscores is a constant holding a + // value, not a type: ISO_4217_URL.lower is a string's own method. JSON and + // URL carry no underscore and stay guarded. + var has_underscore = false; + var all_caps = true; + var ch: [*]const u8 = &dotted; + while (@intFromPtr(ch) < @intFromPtr(last_dot) and ch[0] != '.') : (ch += 1) { + if (ch[0] == '_') { + has_underscore = true; + } else if (ch[0] >= 'a' and ch[0] <= 'z') { + all_caps = false; + break; + } + } + if (all_caps and has_underscore) return true; + + // The candidate's parent segment: the one before its final name. + const cand_last: CStr = c.strrchr(candidate_qn, '.'); + if (cand_last == null or cand_last == candidate_qn) { + return true; // top-level candidate — no parent to look for + } + var parent: [*]const u8 = cand_last; + while (@intFromPtr(parent) > @intFromPtr(candidate_qn) and (parent - 1)[0] != '.') { + parent -= 1; + } + const parent_len = util.ptrDiff(cand_last, parent); + + // Walk the chain — every segment before the final callee name. A trailing + // "()" is dropped so JSONEncoder().encode reads as JSONEncoder. + var seg: [*]const u8 = &dotted; + while (@intFromPtr(seg) < @intFromPtr(last_dot)) { + const end: [*]const u8 = c.strchr(seg, '.'); + var len = util.ptrDiff(end, seg); + if (len >= 2 and seg[len - 2] == '(' and seg[len - 1] == ')') { + len -= 2; // an empty "()" — JSONEncoder().encode names JSONEncoder + } + if (len == parent_len and c.strncmp(seg, parent, parent_len) == 0) return true; + seg = end + 1; + } + return false; +} + +/// Strategy 3+4: Name lookup + suffix match +fn resolveNameLookup( + r: *const Registry, + callee_name: CStr, + module_qn: CStr, + import_vals: [*c]const CStr, + import_count: c_int, +) Resolution { + const lookup = simpleName(callee_name); + const arr = byName(r, lookup) orelse return emptyResult(); + if (arr.count == 0) return emptyResult(); + if (arr.count > REG_MAX_CANDIDATES) { + return emptyResult(); // unresolvably ambiguous — see REG_MAX_CANDIDATES + } + + // Strategy 3.5: a qualified callee disambiguates among multiple same-name + // candidates by full qualified tail, before bare-name scoring collapses + // them onto a single winner. + if (arr.count > 1) { + const q = qualifiedSuffixMatch(arr, callee_name); + if (q != null) { + return resolved(q, "qualified_suffix", CONF_QUALIFIED_SUFFIX, REG_RESOLVED); + } + } + + // Strategy 3: unique name + if (arr.count == 1) { + if (!receiverChainAdmits(callee_name, arr.items[0])) return emptyResult(); + var conf: f64 = CONF_UNIQUE_NAME; + if (import_vals != null and import_count > 0 and + !isImportReachable(arr.items[0], import_vals, import_count)) + { + conf *= DEFAULT_CONFIDENCE; + } + return resolved(arr.items[0], "unique_name", conf, REG_RESOLVED); + } + + // Strategy 4: multiple candidates + if (import_vals != null and import_count > 0) { + return resolveMultiWithImports(arr, module_qn, import_vals, import_count); + } + const best = bestByImportDistance(@ptrCast(arr.items), arr.count, module_qn); + if (best != null) { + if (!receiverChainAdmits(callee_name, best)) return emptyResult(); + const conf = candidateCountPenalty(CONF_SUFFIX_MATCH, arr.count); + return resolved(best, "suffix_match", conf, arr.count); + } + return emptyResult(); +} + +pub export fn cbm_registry_resolve( + r_c: ?*const Registry, + callee_name: CStr, + module_qn: CStr, + import_map_keys: [*c]CStr, + import_map_vals: [*c]CStr, + import_map_count: c_int, +) Resolution { + const r = r_c orelse return emptyResult(); + if (callee_name == null) return emptyResult(); + + // Per-file cache: same callee_name in N call sites → 1 chain walk + // + N-1 O(1) hash hits. module_qn is constant per file so the + // cache key only needs callee_name. + if (resolve_cache) |cache| { + const cached: ?*ResolveCacheEntry = @ptrCast(@alignCast(c.cbm_ht_get(cache, callee_name))); + if (cached) |entry| return entry.res; + } + + // Split callee at the first path separator: "pkg.Func" → prefix="pkg", + // suffix="Func". Rust/C++ use "::" rather than ".", so honor whichever + // separator appears first ("lib::square" → prefix="lib", suffix="square"). + // Both separators are unambiguous, so handling "::" never affects "."-only + // callees. + var prefix: [c.CBM_SZ_256]u8 = @splat(0); + var suffix: CStr = null; + const dot: CStr = c.strchr(callee_name, '.'); + const colons: CStr = c.strstr(callee_name, "::"); + var sep: CStr = dot; + var sep_len: usize = 1; // length of '.' + if (colons != null and (sep == null or @intFromPtr(colons) < @intFromPtr(sep))) { + sep = colons; + sep_len = 2; // length of "::" + } + if (sep != null) { + var plen = util.ptrDiff(sep, callee_name); + if (plen >= prefix.len) plen = prefix.len - 1; + @memcpy(prefix[0..plen], callee_name[0..plen]); + prefix[plen] = 0; + suffix = sep + sep_len; + } else { + _ = c.snprintf(&prefix, prefix.len, "%s", callee_name); + } + + // Strategy 1: import map + var res = resolveImportMap(r, &prefix, suffix, import_map_keys, import_map_vals, import_map_count); + if (!(res.qualified_name != null and res.qualified_name[0] != 0)) { + // Strategy 2: same module + res = resolveSameModule(r, callee_name, suffix, module_qn); + } + if (!(res.qualified_name != null and res.qualified_name[0] != 0)) { + // Strategy 3+4: name lookup + res = resolveNameLookup(r, callee_name, module_qn, import_map_vals, import_map_count); + } + + // Cache the result (including empty — caching the negative answer + // is just as valuable; same name asks the same question). + if (resolve_cache) |cache| { + const entry: ?*ResolveCacheEntry = interop.mallocN(ResolveCacheEntry, 1); + if (entry) |e| { + e.res = res; + const kdup: MutStr = c.strdup(callee_name); + if (kdup != null) { + _ = c.cbm_ht_set(cache, kdup, e); + } else { + c.free(e); + } + } + } + return res; +} + +// ── Fuzzy Resolve ─────────────────────────────────────────────────── + +/// Filter candidates by import reachability. Returns count of reachable. +fn filterImportReachable( + candidates: [*c]const CStr, + count: c_int, + import_vals: [*c]const CStr, + import_count: c_int, + out: []CStr, +) c_int { + var n: usize = 0; + var i: usize = 0; + while (i < count and n < out.len) : (i += 1) { + if (isImportReachable(candidates[i], import_vals, import_count)) { + out[n] = candidates[i]; + n += 1; + } + } + return @intCast(n); +} + +fn fuzzy(qn: CStr, confidence: f64, candidate_count: c_int) c.cbm_fuzzy_result_t { + return .{ .result = resolved(qn, "fuzzy", confidence, candidate_count), .ok = true }; +} + +pub export fn cbm_registry_fuzzy_resolve( + r_c: ?*const Registry, + callee_name: CStr, + module_qn: CStr, + import_map_keys: [*c]CStr, + import_map_vals: [*c]CStr, + import_map_count: c_int, +) c.cbm_fuzzy_result_t { + _ = import_map_keys; + const no_match: c.cbm_fuzzy_result_t = .{ .result = emptyResult(), .ok = false }; + const r = r_c orelse return no_match; + if (callee_name == null) return no_match; + + // Extract simple name (last segment after dots) + const lookup = simpleName(callee_name); + const arr = byName(r, lookup) orelse return no_match; + if (arr.count == 0) return no_match; + if (arr.count > REG_MAX_CANDIDATES) { + return no_match; // unresolvably ambiguous — see REG_MAX_CANDIDATES + } + + const have_imports = import_map_vals != null and import_map_count > 0; + + // Single candidate + if (arr.count == 1) { + var conf: f64 = CONF_FUZZY_SINGLE; + if (have_imports and !isImportReachable(arr.items[0], import_map_vals, import_map_count)) { + conf *= DEFAULT_CONFIDENCE; + } + return fuzzy(arr.items[0], conf, REG_RESOLVED); + } + + // Multiple candidates: filter by import reachability + var filtered: [c.CBM_SZ_256]CStr = undefined; + var fcount = arr.count; + var fptr: [*c]const CStr = @ptrCast(arr.items); + + if (have_imports) { + fcount = filterImportReachable(@ptrCast(arr.items), arr.count, import_map_vals, import_map_count, &filtered); + fptr = &filtered; + } + + if (fcount == 0) { + // No import-reachable — use originals with penalty + const best = bestByImportDistance(@ptrCast(arr.items), arr.count, module_qn); + if (best == null) return no_match; + return fuzzy(best, candidateCountPenalty(CONF_FUZZY_MULTI * REG_HALF_PENALTY, arr.count), arr.count); + } + if (fcount == 1) { + return fuzzy(fptr[0], candidateCountPenalty(CONF_FUZZY_SINGLE, arr.count), arr.count); + } + const best = bestByImportDistance(fptr, fcount, module_qn); + if (best == null) return no_match; + return fuzzy(best, candidateCountPenalty(CONF_FUZZY_MULTI, fcount), fcount); +} + +// ── FindEndingWith ────────────────────────────────────────────────── + +const FewCtx = struct { + target: CStr, // ".suffix" + target_len: usize, + results: [*c]CStr, + count: c_int, + cap: c_int, +}; + +fn fewScan(key: CStr, value: ?*anyopaque, ud: ?*anyopaque) callconv(.c) void { + _ = value; + const ctx: *FewCtx = @ptrCast(@alignCast(ud)); + const klen = c.strlen(key); + if (klen >= ctx.target_len and util.streq(key + klen - ctx.target_len, ctx.target)) { + if (ctx.count >= ctx.cap) { + ctx.cap = if (ctx.cap != 0) ctx.cap * 2 else REG_INIT_CAP; + ctx.results = interop.safeReallocN(CStr, ctx.results, ix(ctx.cap)); + } + ctx.results[ix(ctx.count)] = key; + ctx.count += 1; + } +} + +pub export fn cbm_registry_find_ending_with(r_c: ?*const Registry, suffix: CStr, out_c: ?*[*c]CStr) c_int { + const out = out_c orelse return 0; + const r = r_c orelse { + out.* = null; + return 0; + }; + if (suffix == null) { + out.* = null; + return 0; + } + + // Build ".suffix" target + const slen = c.strlen(suffix); + const target: MutStr = @ptrCast(c.malloc(slen + REG_SUFFIX_ALLOC)); + target[0] = '.'; + @memcpy(target[1 .. slen + 2], suffix[0 .. slen + 1]); + + var ctx: FewCtx = .{ .target = target, .target_len = slen + 1, .results = null, .count = 0, .cap = 0 }; + c.cbm_ht_foreach(r.exact, fewScan, &ctx); + + c.free(target); + out.* = ctx.results; + return ctx.count; +} + +/// Public wrapper for isImportReachable (testing). +pub export fn cbm_registry_is_import_reachable(candidate_qn: CStr, import_vals: [*c]CStr, import_count: c_int) bool { + return isImportReachable(candidate_qn, import_vals, import_count); +} + +// ── Class-qualified route handlers (PHP, #1146) ───────────────────── + +/// Method QNs are path-based: ..... +/// A PHP class is named by its namespace instead, so a "Ns\\Class::method" +/// handler is placed on a QN by what places the class file itself. +const CONF_PSR4_MEMBER: f64 = 0.95; +const CONF_NAMESPACE_PATH_MEMBER: f64 = 0.90; + +fn segEqNocase(a: []const u8, b: []const u8) bool { + if (a.len != b.len) return false; + for (a, b) |x, y| { + const ca = if (x >= 'A' and x <= 'Z') x - 'A' + 'a' else x; + const cb = if (y >= 'A' and y <= 'Z') y - 'A' + 'a' else y; + if (ca != cb) return false; + } + return true; +} + +/// Does qn end with ".."? Returns the offset of that tail or -1. +fn qnClassMemberTail(qn: CStr, cls: [*]const u8, cls_len: usize, member: CStr) c_int { + const qlen = c.strlen(qn); + const mlen = c.strlen(member); + if (qlen < cls_len + mlen + 2) return c.CBM_NOT_FOUND; + const m: [*]const u8 = qn + qlen - mlen; + const cl: [*]const u8 = m - 1 - cls_len; + if (!util.streq(m, member) or (m - 1)[0] != '.' or c.strncmp(cl, cls, cls_len) != 0 or + (cl - 1)[0] != '.') + { + return c.CBM_NOT_FOUND; + } + return @intCast(util.ptrDiff(cl - 1, qn)); +} + +/// PSR-4: the class file Composer's autoloader loads for the FQN, the class it +/// defines, and that class's member — through the PSR-4 import resolver +/// (cbm_pipeline_psr4_member_qn), so a root whose folder does not mirror its +/// namespace (Acme\\Blog\\ -> packages/blog/src) places the class as well. +/// Returns its tri-state outcome; out_qn is the member QN when RESOLVED. +fn psr4MemberQn( + gbuf: ?*const c.cbm_gbuf_t, + fqn: [*]const u8, + fqn_len: usize, + member: CStr, + out_qn: *CStr, +) c.cbm_psr4_member_t { + var cls_fqn: [c.CBM_SZ_512]u8 = undefined; + out_qn.* = null; + if (gbuf == null or fqn_len >= cls_fqn.len) return c.CBM_PSR4_MEMBER_NOT_COVERED; + @memcpy(cls_fqn[0..fqn_len], fqn[0..fqn_len]); + cls_fqn[fqn_len] = 0; + return c.cbm_pipeline_psr4_member_qn(gbuf, &cls_fqn, member, out_qn); +} + +/// Do the directory segments of qn (between the project root segment and the +/// file stem, i.e. before offset tail = "..") end with every +/// segment of the namespace fqn[0, ns_len)? Case-insensitive: App\\Http is +/// conventionally app/Http. An empty namespace is trivially aligned. +fn namespaceMatchesDirs(qn: CStr, tail: c_int, fqn: [*]const u8, ns_len: usize) bool { + const root_dot: CStr = c.strchr(qn, '.'); + const stem_end: [*]const u8 = qn + ix(tail); + if (root_dot == null or @intFromPtr(stem_end) <= @intFromPtr(root_dot)) return false; + var dir_end: [*]const u8 = stem_end - 1; // back over the file stem + while (dir_end[0] != '.') dir_end -= 1; + var ns_end = ns_len; + while (ns_end > 0) { + var ns_start = ns_end; + while (ns_start > 0 and fqn[ns_start - 1] != '\\') ns_start -= 1; + if (@intFromPtr(dir_end) <= @intFromPtr(root_dot)) { + return false; // more namespace segments than directories + } + var dir_start = dir_end; + while ((dir_start - 1)[0] != '.') dir_start -= 1; + if (!segEqNocase(fqn[ns_start..ns_end], dir_start[0..util.ptrDiff(dir_end, dir_start)])) { + return false; + } + dir_end = dir_start - 1; + ns_end = if (ns_start > 0) ns_start - 1 else 0; + } + return true; +} + +/// Namespace/path alignment: the one "." candidate that sits +/// where its namespace says. NULL when none or several do. +fn alignedMemberQn( + r: *const Registry, + fqn: [*]const u8, + ns_len: usize, + cls: [*]const u8, + cls_len: usize, + member: CStr, +) CStr { + const arr = byName(r, member) orelse return null; + var match: CStr = null; + for (0..ix(arr.count)) |i| { + const qn: CStr = arr.items[i]; + const tail = qnClassMemberTail(qn, cls, cls_len, member); + if (tail == c.CBM_NOT_FOUND or !namespaceMatchesDirs(qn, tail, fqn, ns_len)) continue; + if (match != null) return null; // ambiguous + match = qn; + } + return match; +} + +pub export fn cbm_registry_resolve_handler( + r_c: ?*const Registry, + handler_ref: CStr, + module_qn: CStr, + import_map_keys: [*c]CStr, + import_map_vals: [*c]CStr, + import_map_count: c_int, + gbuf: ?*const c.cbm_gbuf_t, +) Resolution { + const sep: CStr = if (handler_ref != null) c.strstr(handler_ref, "::") else null; + const r = r_c orelse return cbm_registry_resolve(r_c, handler_ref, module_qn, import_map_keys, import_map_vals, import_map_count); + if (sep == null) { + return cbm_registry_resolve(r, handler_ref, module_qn, import_map_keys, import_map_vals, import_map_count); + } + const member: CStr = sep + 2; + const fqn_len = util.ptrDiff(sep, handler_ref); + var ns_len = fqn_len; + while (ns_len > 0 and handler_ref[ns_len - 1] != '\\') ns_len -= 1; + const cls: [*]const u8 = handler_ref + ns_len; + const cls_len = fqn_len - ns_len; + ns_len = if (ns_len > 0) ns_len - 1 else 0; // drop the separator before the class + if (cls_len == 0 or member[0] == 0) return emptyResult(); + var qn: CStr = null; + switch (psr4MemberQn(gbuf, handler_ref, fqn_len, member, &qn)) { + c.CBM_PSR4_MEMBER_RESOLVED => return resolved(qn, "php_psr4", CONF_PSR4_MEMBER, REG_RESOLVED), + // A covered class lives where PSR-4 says or nowhere; a namespace- + // mirroring folder elsewhere would be a guessed edge (#1186). + c.CBM_PSR4_MEMBER_UNRESOLVED => return emptyResult(), + else => {}, + } + qn = alignedMemberQn(r, handler_ref, ns_len, cls, cls_len, member); + if (qn != null) { + return resolved(qn, "php_namespace_path", CONF_NAMESPACE_PATH_MEMBER, REG_RESOLVED); + } + return emptyResult(); +} diff --git a/src/pipeline/result_spill.c b/src/pipeline/result_spill.c deleted file mode 100644 index a95ddfae..00000000 --- a/src/pipeline/result_spill.c +++ /dev/null @@ -1,656 +0,0 @@ -/* - * result_spill.c — see result_spill.h. - * - * Encoding: each record is written as its raw struct bytes (every scalar - * field, whatever its type) followed by its string and array fields in a - * fixed order. Decoding copies the raw bytes back and then replaces every - * pointer field with a copy in the new result's arena. A pointer field added - * to one of these structs must be added here too; the size assertions below - * fail the build until it is. - */ -#include "pipeline/result_spill.h" - -#include "foundation/compat_fs.h" -#include "foundation/compat_thread.h" -#include "foundation/log.h" - -#include -#include -#include -#include - -#ifdef _WIN32 -#include -#define getpid _getpid -#define spill_fseek _fseeki64 -#else -#include -#define spill_fseek fseeko -#endif - -#if UINTPTR_MAX == UINT64_MAX -_Static_assert(sizeof(CBMDefinition) == 248, "update result_spill.c for CBMDefinition"); -_Static_assert(sizeof(CBMCall) == 376, "update result_spill.c for CBMCall"); -_Static_assert(sizeof(CBMCallArg) == 40, "update result_spill.c for CBMCallArg"); -_Static_assert(sizeof(CBMImport) == 32, "update result_spill.c for CBMImport"); -_Static_assert(sizeof(CBMUsage) == 32, "update result_spill.c for CBMUsage"); -_Static_assert(sizeof(CBMThrow) == 16, "update result_spill.c for CBMThrow"); -_Static_assert(sizeof(CBMReadWrite) == 24, "update result_spill.c for CBMReadWrite"); -_Static_assert(sizeof(CBMTypeRef) == 16, "update result_spill.c for CBMTypeRef"); -_Static_assert(sizeof(CBMEnvAccess) == 16, "update result_spill.c for CBMEnvAccess"); -_Static_assert(sizeof(CBMTypeAssign) == 24, "update result_spill.c for CBMTypeAssign"); -_Static_assert(sizeof(CBMStringRef) == 32, "update result_spill.c for CBMStringRef"); -_Static_assert(sizeof(CBMInfraBinding) == 24, "update result_spill.c for CBMInfraBinding"); -_Static_assert(sizeof(CBMChannel) == 32, "update result_spill.c for CBMChannel"); -_Static_assert(sizeof(CBMImplTrait) == 16, "update result_spill.c for CBMImplTrait"); -_Static_assert(sizeof(CBMResolvedCall) == 56, "update result_spill.c for CBMResolvedCall"); -_Static_assert(sizeof(CBMFileResult) == 4480, "update result_spill.c for CBMFileResult"); -#endif - -enum { SPILL_NULL = 0xFFFFFFFFU, SPILL_MAGIC = 0x31535243U /* "CRS1" */ }; - -/* ── Encoder ─────────────────────────────────────────────────────── */ - -typedef struct { - char *data; - size_t len; - size_t cap; - bool failed; -} spill_buf_t; - -static void put(spill_buf_t *b, const void *p, size_t n) { - if (b->failed || n == 0) { - return; - } - if (n > SIZE_MAX - b->len) { - b->failed = true; - return; - } - if (b->len + n > b->cap) { - size_t cap = b->cap ? b->cap : 4096; - while (cap < b->len + n) { - cap *= 2; - } - char *data = realloc(b->data, cap); - if (!data) { - b->failed = true; - return; - } - b->data = data; - b->cap = cap; - } - memcpy(b->data + b->len, p, n); - b->len += n; -} - -static void put_u32(spill_buf_t *b, uint32_t v) { - put(b, &v, sizeof(v)); -} - -static void put_bytes(spill_buf_t *b, const char *s, size_t n) { - if (!s) { - put_u32(b, SPILL_NULL); - return; - } - if (n >= SPILL_NULL) { - b->failed = true; - return; - } - put_u32(b, (uint32_t)n); - put(b, s, n); -} - -static void put_str(spill_buf_t *b, const char *s) { - put_bytes(b, s, s ? strlen(s) : 0); -} - -/* NULL-terminated string array. */ -static void put_strv(spill_buf_t *b, const char *const *v) { - if (!v) { - put_u32(b, SPILL_NULL); - return; - } - uint32_t n = 0; - while (v[n]) { - n++; - } - put_u32(b, n); - for (uint32_t i = 0; i < n; i++) { - put_str(b, v[i]); - } -} - -/* ── Decoder ─────────────────────────────────────────────────────── */ - -typedef struct { - const char *p; - size_t left; - CBMArena *arena; - bool failed; -} spill_rd_t; - -static void take(spill_rd_t *r, void *out, size_t n) { - if (r->failed || n > r->left) { - r->failed = true; - memset(out, 0, n); - return; - } - memcpy(out, r->p, n); - r->p += n; - r->left -= n; -} - -static uint32_t take_u32(spill_rd_t *r) { - uint32_t v = 0; - take(r, &v, sizeof(v)); - return v; -} - -/* A length-prefixed byte string copied into the arena and NUL-terminated. */ -static const char *take_bytes(spill_rd_t *r, int *out_len) { - uint32_t n = take_u32(r); - if (out_len) { - *out_len = 0; - } - if (r->failed || n == SPILL_NULL) { - return NULL; - } - if (n > r->left) { - r->failed = true; - return NULL; - } - char *s = cbm_arena_alloc(r->arena, (size_t)n + 1); - if (!s) { - r->failed = true; - return NULL; - } - memcpy(s, r->p, n); - s[n] = '\0'; - r->p += n; - r->left -= n; - if (out_len) { - *out_len = (int)n; - } - return s; -} - -static const char *take_str(spill_rd_t *r) { - return take_bytes(r, NULL); -} - -static const char **take_strv(spill_rd_t *r) { - uint32_t n = take_u32(r); - if (r->failed || n == SPILL_NULL) { - return NULL; - } - /* Each entry costs at least its 4-byte length. */ - if (n > r->left / sizeof(uint32_t)) { - r->failed = true; - return NULL; - } - const char **v = cbm_arena_alloc(r->arena, ((size_t)n + 1) * sizeof(*v)); - if (!v) { - r->failed = true; - return NULL; - } - for (uint32_t i = 0; i < n; i++) { - v[i] = take_str(r); - } - v[n] = NULL; - return v; -} - -/* Array header: the item count, then that many raw records. Returns arena - * storage for the records (NULL for zero) with the count in *out_count. */ -static void *take_items(spill_rd_t *r, size_t item_size, int *out_count) { - uint32_t n = take_u32(r); - *out_count = 0; - if (r->failed || n == 0) { - return NULL; - } - if (n > INT32_MAX || n > r->left / item_size) { - r->failed = true; - return NULL; - } - void *items = cbm_arena_alloc(r->arena, (size_t)n * item_size); - if (!items) { - r->failed = true; - return NULL; - } - take(r, items, (size_t)n * item_size); - *out_count = (int)n; - return items; -} - -/* ── Records ─────────────────────────────────────────────────────── */ - -/* Each record type lists its pointer fields once (X-macros). An array is - * written as its count, then every record's raw bytes with the pointer - * fields nulled (so encoding is a pure function of the content), then each - * record's strings in field order. */ -#define ENC_STR(f) put_str(b, x->f); -#define DEC_STR(f) x->f = take_str(r); -#define ENC_STRV(f) put_strv(b, x->f); -#define DEC_STRV(f) x->f = take_strv(r); -#define NULL_FIELD(f) x->f = NULL; - -#define DEF_STRS(X) \ - X(name) X(qualified_name) X(label) X(file_path) X(signature) X(return_type) \ - X(receiver) X(docstring) X(parent_class) X(route_path) X(route_method) \ - X(structural_profile) X(body_tokens) X(variants) X(alias_value) \ - X(http_client) X(http_base_url) -#define DEF_STRVS(X) X(decorators) X(base_classes) X(param_names) X(param_types) X(return_types) - -static void null_def(CBMDefinition *x) { - DEF_STRS(NULL_FIELD) - DEF_STRVS(NULL_FIELD) - x->fingerprint = NULL; -} - -static void enc_def(spill_buf_t *b, const CBMDefinition *x) { - DEF_STRS(ENC_STR) - DEF_STRVS(ENC_STRV) - if (x->fingerprint && x->fingerprint_k > 0) { - put_u32(b, (uint32_t)x->fingerprint_k); - put(b, x->fingerprint, (size_t)x->fingerprint_k * sizeof(*x->fingerprint)); - } else { - put_u32(b, SPILL_NULL); - } -} - -static void dec_def(spill_rd_t *r, CBMDefinition *x) { - DEF_STRS(DEC_STR) - DEF_STRVS(DEC_STRV) - uint32_t k = take_u32(r); - x->fingerprint = NULL; - if (r->failed || k == SPILL_NULL) { - return; - } - if (k > r->left / sizeof(*x->fingerprint)) { - r->failed = true; - return; - } - x->fingerprint = cbm_arena_alloc(r->arena, (size_t)k * sizeof(*x->fingerprint)); - if (!x->fingerprint) { - r->failed = true; - return; - } - take(r, x->fingerprint, (size_t)k * sizeof(*x->fingerprint)); -} - -#define CALL_STRS(X) X(callee_name) X(enclosing_func_qn) X(first_string_arg) X(second_arg_name) -#define ARG_STRS(X) X(expr) X(value) X(url_value) X(keyword) - -static void null_call(CBMCall *x) { - CALL_STRS(NULL_FIELD) - for (int i = 0; i < CBM_MAX_CALL_ARGS; i++) { - x->args[i].expr = NULL; - x->args[i].value = NULL; - x->args[i].url_value = NULL; - x->args[i].keyword = NULL; - } -} - -static void enc_call(spill_buf_t *b, const CBMCall *x) { - CALL_STRS(ENC_STR) - for (int i = 0; i < CBM_MAX_CALL_ARGS; i++) { - put_str(b, x->args[i].expr); - put_str(b, x->args[i].value); - put_str(b, x->args[i].url_value); - put_str(b, x->args[i].keyword); - } -} - -static void dec_call(spill_rd_t *r, CBMCall *x) { - CALL_STRS(DEC_STR) - for (int i = 0; i < CBM_MAX_CALL_ARGS; i++) { - x->args[i].expr = take_str(r); - x->args[i].value = take_str(r); - x->args[i].url_value = take_str(r); - x->args[i].keyword = take_str(r); - } -} - -/* Records whose pointer fields are only strings. */ -#define STRING_RECORD(T, sfx, FIELDS) \ - static void null_##sfx(T *x) { FIELDS(NULL_FIELD) } \ - static void enc_##sfx(spill_buf_t *b, const T *x) { FIELDS(ENC_STR) } \ - static void dec_##sfx(spill_rd_t *r, T *x) { FIELDS(DEC_STR) } - -#define IMPORT_STRS(X) X(local_name) X(module_path) X(member) -#define USAGE_STRS(X) X(ref_name) X(enclosing_func_qn) -#define THROW_STRS(X) X(exception_name) X(enclosing_func_qn) -#define RW_STRS(X) X(var_name) X(enclosing_func_qn) -#define TYPEREF_STRS(X) X(type_name) X(enclosing_func_qn) -#define ENV_STRS(X) X(env_key) X(enclosing_func_qn) -#define TYPEASSIGN_STRS(X) X(var_name) X(type_name) X(enclosing_func_qn) -#define IMPLTRAIT_STRS(X) X(trait_name) X(struct_name) -#define RESOLVED_STRS(X) X(caller_qn) X(callee_qn) X(strategy) X(reason) -#define STRREF_STRS(X) X(value) X(enclosing_func_qn) X(key_path) -#define INFRA_STRS(X) X(source_name) X(target_url) X(broker) -#define CHANNEL_STRS(X) X(channel_name) X(transport) X(enclosing_func_qn) - -STRING_RECORD(CBMImport, import, IMPORT_STRS) -STRING_RECORD(CBMUsage, usage, USAGE_STRS) -STRING_RECORD(CBMThrow, throw, THROW_STRS) -STRING_RECORD(CBMReadWrite, rw, RW_STRS) -STRING_RECORD(CBMTypeRef, typeref, TYPEREF_STRS) -STRING_RECORD(CBMEnvAccess, env, ENV_STRS) -STRING_RECORD(CBMTypeAssign, typeassign, TYPEASSIGN_STRS) -STRING_RECORD(CBMImplTrait, impltrait, IMPLTRAIT_STRS) -STRING_RECORD(CBMResolvedCall, resolved, RESOLVED_STRS) -STRING_RECORD(CBMStringRef, strref, STRREF_STRS) -STRING_RECORD(CBMInfraBinding, infra, INFRA_STRS) -STRING_RECORD(CBMChannel, channel, CHANNEL_STRS) - -#define ENCODE_ARRAY(b, arr, sfx) \ - do { \ - put_u32((b), (uint32_t)(arr).count); \ - for (int i_ = 0; i_ < (arr).count; i_++) { \ - __typeof__(*(arr).items) raw_; \ - memcpy(&raw_, &(arr).items[i_], sizeof(raw_)); \ - null_##sfx(&raw_); \ - put((b), &raw_, sizeof(raw_)); \ - } \ - for (int i_ = 0; i_ < (arr).count; i_++) { \ - enc_##sfx((b), &(arr).items[i_]); \ - } \ - } while (0) - -#define DECODE_ARRAY(r, arr, sfx) \ - do { \ - (arr).items = take_items((r), sizeof(*(arr).items), &(arr).count); \ - (arr).cap = (arr).count; \ - for (int i_ = 0; i_ < (arr).count && !(r)->failed; i_++) { \ - dec_##sfx((r), &(arr).items[i_]); \ - } \ - } while (0) - -/* Scalar result fields, written as one block so a decode restores them in - * one step. */ -typedef struct { - bool has_error; - bool parse_incomplete; - bool parse_unusable; - bool is_test_file; - int32_t error_region_count; - int32_t imports_count; - int32_t cached_lang; -} spill_scalars_t; - -char *cbm_result_spill_encode(const CBMFileResult *result, size_t *out_len) { - spill_buf_t b = {0}; - put_u32(&b, SPILL_MAGIC); - ENCODE_ARRAY(&b, result->defs, def); - ENCODE_ARRAY(&b, result->calls, call); - ENCODE_ARRAY(&b, result->imports, import); - ENCODE_ARRAY(&b, result->usages, usage); - ENCODE_ARRAY(&b, result->throws, throw); - ENCODE_ARRAY(&b, result->rw, rw); - ENCODE_ARRAY(&b, result->type_refs, typeref); - ENCODE_ARRAY(&b, result->env_accesses, env); - ENCODE_ARRAY(&b, result->type_assigns, typeassign); - ENCODE_ARRAY(&b, result->impl_traits, impltrait); - ENCODE_ARRAY(&b, result->resolved_calls, resolved); - ENCODE_ARRAY(&b, result->string_refs, strref); - ENCODE_ARRAY(&b, result->infra_bindings, infra); - ENCODE_ARRAY(&b, result->channels, channel); - put_str(&b, result->module_qn); - put_str(&b, result->namespace_name); - put_strv(&b, result->exports); - put_strv(&b, result->constants); - put_strv(&b, result->global_vars); - put_strv(&b, result->macros); - put_str(&b, result->error_msg); - put_str(&b, result->error_ranges); - spill_scalars_t scalars = { - .has_error = result->has_error, - .parse_incomplete = result->parse_incomplete, - .parse_unusable = result->parse_unusable, - .is_test_file = result->is_test_file, - .error_region_count = result->error_region_count, - .imports_count = result->imports_count, - .cached_lang = (int32_t)result->cached_lang, - }; - put(&b, &scalars, sizeof(scalars)); - put_bytes(&b, result->source, result->source ? (size_t)result->source_len : 0); - if (b.failed) { - free(b.data); - return NULL; - } - *out_len = b.len; - return b.data; -} - -CBMFileResult *cbm_result_spill_decode(const char *data, size_t len) { - CBMFileResult *result = calloc(1, sizeof(*result)); - if (!result) { - return NULL; - } - cbm_arena_init(&result->arena); - spill_rd_t r = {.p = data, .left = len, .arena = &result->arena}; - if (take_u32(&r) != SPILL_MAGIC) { - r.failed = true; - } - DECODE_ARRAY(&r, result->defs, def); - DECODE_ARRAY(&r, result->calls, call); - DECODE_ARRAY(&r, result->imports, import); - DECODE_ARRAY(&r, result->usages, usage); - DECODE_ARRAY(&r, result->throws, throw); - DECODE_ARRAY(&r, result->rw, rw); - DECODE_ARRAY(&r, result->type_refs, typeref); - DECODE_ARRAY(&r, result->env_accesses, env); - DECODE_ARRAY(&r, result->type_assigns, typeassign); - DECODE_ARRAY(&r, result->impl_traits, impltrait); - DECODE_ARRAY(&r, result->resolved_calls, resolved); - DECODE_ARRAY(&r, result->string_refs, strref); - DECODE_ARRAY(&r, result->infra_bindings, infra); - DECODE_ARRAY(&r, result->channels, channel); - result->module_qn = take_str(&r); - result->namespace_name = take_str(&r); - result->exports = take_strv(&r); - result->constants = take_strv(&r); - result->global_vars = take_strv(&r); - result->macros = take_strv(&r); - result->error_msg = take_str(&r); - result->error_ranges = take_str(&r); - spill_scalars_t scalars; - take(&r, &scalars, sizeof(scalars)); - result->has_error = scalars.has_error; - result->parse_incomplete = scalars.parse_incomplete; - result->parse_unusable = scalars.parse_unusable; - result->is_test_file = scalars.is_test_file; - result->error_region_count = scalars.error_region_count; - result->imports_count = scalars.imports_count; - result->cached_lang = (CBMLanguage)scalars.cached_lang; - result->source = take_bytes(&r, &result->source_len); - if (r.failed || r.left != 0) { - cbm_free_result(result); - return NULL; - } - return result; -} - -/* ── Store ───────────────────────────────────────────────────────── */ - -typedef struct { - int64_t offset; /* -1: empty */ - int64_t len; - int infra_bindings; - int string_refs; -} spill_slot_t; - -struct cbm_result_spill { - char path[1024]; - FILE *file; /* opened on the first park */ - bool open_failed; - int64_t end; - cbm_mutex_t mu; - spill_slot_t *slots; - int slot_count; - _Atomic int64_t parked; - _Atomic int64_t bytes; - _Atomic int64_t loads; - _Atomic int64_t load_failures; -}; - -cbm_result_spill_t *cbm_result_spill_open(const char *dir, int slots) { - if (!dir || slots <= 0) { - return NULL; - } - cbm_result_spill_t *sp = calloc(1, sizeof(*sp)); - spill_slot_t *table = calloc((size_t)slots, sizeof(*table)); - if (!sp || !table) { - free(sp); - free(table); - return NULL; - } - static _Atomic int sequence; - int n = snprintf(sp->path, sizeof(sp->path), "%s/spill-%d-%d.bin", dir, (int)getpid(), - atomic_fetch_add(&sequence, 1)); - if (n < 0 || (size_t)n >= sizeof(sp->path)) { - free(sp); - free(table); - return NULL; - } - for (int i = 0; i < slots; i++) { - table[i].offset = -1; - } - sp->slots = table; - sp->slot_count = slots; - cbm_mutex_init(&sp->mu); - return sp; -} - -/* Under sp->mu. Creates and the file on first use. */ -static bool spill_ensure_file(cbm_result_spill_t *sp) { - if (sp->file || sp->open_failed) { - return sp->file != NULL; - } - char dir[sizeof(sp->path)]; - snprintf(dir, sizeof(dir), "%s", sp->path); - char *slash = strrchr(dir, '/'); - if (slash) { - *slash = '\0'; - cbm_mkdir_p(dir, 0700); - } - sp->file = cbm_fopen(sp->path, "w+b"); - if (!sp->file) { - sp->open_failed = true; - cbm_log_warn("mem.spill.open_failed", "path", sp->path); - } - return sp->file != NULL; -} - -bool cbm_result_spill_park(cbm_result_spill_t *sp, int slot, CBMFileResult *result) { - if (!sp || !result || slot < 0 || slot >= sp->slot_count) { - return false; - } - size_t len = 0; - char *data = cbm_result_spill_encode(result, &len); - if (!data) { - return false; - } - bool ok = false; - cbm_mutex_lock(&sp->mu); - if (sp->slots[slot].offset < 0 && spill_ensure_file(sp) && - spill_fseek(sp->file, sp->end, SEEK_SET) == 0 && fwrite(data, 1, len, sp->file) == len) { - sp->slots[slot] = (spill_slot_t){ - .offset = sp->end, - .len = (int64_t)len, - .infra_bindings = result->infra_bindings.count, - .string_refs = result->string_refs.count, - }; - sp->end += (int64_t)len; - ok = true; - } - cbm_mutex_unlock(&sp->mu); - free(data); - if (ok) { - atomic_fetch_add(&sp->parked, 1); - atomic_fetch_add(&sp->bytes, (int64_t)len); - cbm_free_result(result); - } - return ok; -} - -bool cbm_result_spill_has(const cbm_result_spill_t *sp, int slot) { - /* A slot is written before its result leaves the cache and read only by - * later phases, so no lock is needed here. */ - return sp && slot >= 0 && slot < sp->slot_count && sp->slots[slot].offset >= 0; -} - -int cbm_result_spill_infra_binding_count(const cbm_result_spill_t *sp, int slot) { - return cbm_result_spill_has(sp, slot) ? sp->slots[slot].infra_bindings : 0; -} - -int cbm_result_spill_string_ref_count(const cbm_result_spill_t *sp, int slot) { - return cbm_result_spill_has(sp, slot) ? sp->slots[slot].string_refs : 0; -} - -CBMFileResult *cbm_result_spill_load(cbm_result_spill_t *sp, int slot) { - if (!cbm_result_spill_has(sp, slot)) { - return NULL; - } - spill_slot_t entry = sp->slots[slot]; - char *data = malloc((size_t)entry.len); - if (!data) { - return NULL; - } - cbm_mutex_lock(&sp->mu); - bool read = fflush(sp->file) == 0 && spill_fseek(sp->file, entry.offset, SEEK_SET) == 0 && - fread(data, 1, (size_t)entry.len, sp->file) == (size_t)entry.len; - cbm_mutex_unlock(&sp->mu); - CBMFileResult *result = read ? cbm_result_spill_decode(data, (size_t)entry.len) : NULL; - free(data); - if (result) { - atomic_fetch_add(&sp->loads, 1); - } else { - atomic_fetch_add(&sp->load_failures, 1); - cbm_log_error("mem.spill.load_failed", "path", sp->path); - } - return result; -} - -int64_t cbm_result_spill_load_failures(const cbm_result_spill_t *sp) { - return sp ? atomic_load(&sp->load_failures) : 0; -} - -void cbm_result_spill_stats(const cbm_result_spill_t *sp, int64_t *parked, int64_t *bytes, - int64_t *loads) { - *parked = sp ? atomic_load(&sp->parked) : 0; - *bytes = sp ? atomic_load(&sp->bytes) : 0; - *loads = sp ? atomic_load(&sp->loads) : 0; -} - -void cbm_result_spill_close(cbm_result_spill_t *sp) { - if (!sp) { - return; - } - if (sp->file) { - fclose(sp->file); - cbm_unlink(sp->path); - } - cbm_mutex_destroy(&sp->mu); - free(sp->slots); - free(sp); -} - -/* ── Cache access ────────────────────────────────────────────────── */ - -CBMFileResult *cbm_result_cache_acquire(CBMFileResult **cache, cbm_result_spill_t *sp, int i) { - if (cache[i]) { - return cache[i]; - } - return cbm_result_spill_load(sp, i); -} - -void cbm_result_cache_release(CBMFileResult **cache, int i, CBMFileResult *result) { - if (result && result != cache[i]) { - cbm_free_result(result); - } -} - -bool cbm_result_cache_present(CBMFileResult *const *cache, const cbm_result_spill_t *sp, int i) { - return cache[i] != NULL || cbm_result_spill_has(sp, i); -} diff --git a/src/pipeline/result_spill.zig b/src/pipeline/result_spill.zig new file mode 100644 index 00000000..23ee5eab --- /dev/null +++ b/src/pipeline/result_spill.zig @@ -0,0 +1,624 @@ +//! result_spill.zig — see result_spill.h. +//! +//! Encoding: each record is written as its raw struct bytes (every scalar +//! field, whatever its type) followed by its string and array fields in a +//! fixed order. Decoding copies the raw bytes back and then replaces every +//! pointer field with a copy in the new result's arena. A pointer field added +//! to one of these structs must be added here too; the size assertions below +//! fail the build until it is. + +const std = @import("std"); +const builtin = @import("builtin"); +const c = @import("c"); +const interop = @import("../foundation/interop.zig"); +const util = @import("util.zig"); + +const CStr = util.CStr; +const MutStr = util.MutStr; +const ix = util.ix; +const FileResult = c.CBMFileResult; +const AtomicI64 = std.atomic.Value(i64); +const is_windows = builtin.os.tag == .windows; + +comptime { + if (@sizeOf(usize) == 8) { + const sizes = .{ + .{ c.CBMDefinition, 248 }, .{ c.CBMCall, 376 }, .{ c.CBMCallArg, 40 }, + .{ c.CBMImport, 32 }, .{ c.CBMUsage, 32 }, .{ c.CBMThrow, 16 }, + .{ c.CBMReadWrite, 24 }, .{ c.CBMTypeRef, 16 }, .{ c.CBMEnvAccess, 16 }, + .{ c.CBMTypeAssign, 24 }, .{ c.CBMStringRef, 32 }, .{ c.CBMInfraBinding, 24 }, + .{ c.CBMChannel, 32 }, .{ c.CBMImplTrait, 16 }, .{ c.CBMResolvedCall, 56 }, + .{ FileResult, 4480 }, + }; + for (sizes) |entry| { + if (@sizeOf(entry[0]) != entry[1]) { + @compileError("update result_spill.zig for " ++ @typeName(entry[0])); + } + } + } +} + +const SPILL_NULL: u32 = 0xFFFFFFFF; +const SPILL_MAGIC: u32 = 0x31535243; // "CRS1" + +// ── Encoder ───────────────────────────────────────────────────────── + +const SpillBuf = struct { + data: MutStr = null, + len: usize = 0, + cap: usize = 0, + failed: bool = false, +}; + +fn put(b: *SpillBuf, bytes: []const u8) void { + const n = bytes.len; + if (b.failed or n == 0) return; + if (n > std.math.maxInt(usize) - b.len) { + b.failed = true; + return; + } + if (b.len + n > b.cap) { + var cap: usize = if (b.cap != 0) b.cap else 4096; + while (cap < b.len + n) cap *= 2; + const data: MutStr = @ptrCast(c.realloc(b.data, cap)); + if (data == null) { + b.failed = true; + return; + } + b.data = data; + b.cap = cap; + } + @memcpy(b.data[b.len..][0..n], bytes); + b.len += n; +} + +fn putU32(b: *SpillBuf, v: u32) void { + put(b, std.mem.asBytes(&v)); +} + +fn putBytes(b: *SpillBuf, s: CStr, n: usize) void { + if (s == null) { + putU32(b, SPILL_NULL); + return; + } + if (n >= SPILL_NULL) { + b.failed = true; + return; + } + putU32(b, @intCast(n)); + put(b, s[0..n]); +} + +fn putStr(b: *SpillBuf, s: CStr) void { + putBytes(b, s, if (s != null) c.strlen(s) else 0); +} + +/// NULL-terminated string array. +fn putStrv(b: *SpillBuf, v: [*c]const CStr) void { + if (v == null) { + putU32(b, SPILL_NULL); + return; + } + var n: u32 = 0; + while (v[n] != null) n += 1; + putU32(b, n); + for (0..n) |i| putStr(b, v[i]); +} + +// ── Decoder ───────────────────────────────────────────────────────── + +const SpillRd = struct { + p: [*]const u8, + left: usize, + arena: *c.CBMArena, + failed: bool = false, +}; + +fn take(r: *SpillRd, out: []u8) void { + const n = out.len; + if (r.failed or n > r.left) { + r.failed = true; + @memset(out, 0); + return; + } + @memcpy(out, r.p[0..n]); + r.p += n; + r.left -= n; +} + +fn takeU32(r: *SpillRd) u32 { + var v: u32 = 0; + take(r, std.mem.asBytes(&v)); + return v; +} + +/// A length-prefixed byte string copied into the arena and NUL-terminated. +fn takeBytes(r: *SpillRd, out_len: ?*c_int) CStr { + const n = takeU32(r); + if (out_len) |ol| ol.* = 0; + if (r.failed or n == SPILL_NULL) return null; + if (n > r.left) { + r.failed = true; + return null; + } + const s: MutStr = @ptrCast(c.cbm_arena_alloc(r.arena, @as(usize, n) + 1)); + if (s == null) { + r.failed = true; + return null; + } + @memcpy(s[0..n], r.p[0..n]); + s[n] = 0; + r.p += n; + r.left -= n; + if (out_len) |ol| ol.* = @bitCast(n); + return s; +} + +fn takeStr(r: *SpillRd) CStr { + return takeBytes(r, null); +} + +fn takeStrv(r: *SpillRd) [*c]CStr { + const n = takeU32(r); + if (r.failed or n == SPILL_NULL) return null; + // Each entry costs at least its 4-byte length. + if (n > r.left / @sizeOf(u32)) { + r.failed = true; + return null; + } + const v: [*c]CStr = @ptrCast(@alignCast(c.cbm_arena_alloc(r.arena, (@as(usize, n) + 1) * @sizeOf(CStr)))); + if (v == null) { + r.failed = true; + return null; + } + for (0..n) |i| v[i] = takeStr(r); + v[n] = null; + return v; +} + +/// Array header: the item count, then that many raw records. Returns arena +/// storage for the records (NULL for zero) with the count in out_count. +fn takeItems(r: *SpillRd, item_size: usize, out_count: *c_int) ?*anyopaque { + const n = takeU32(r); + out_count.* = 0; + if (r.failed or n == 0) return null; + if (n > std.math.maxInt(i32) or n > r.left / item_size) { + r.failed = true; + return null; + } + const items = c.cbm_arena_alloc(r.arena, @as(usize, n) * item_size) orelse { + r.failed = true; + return null; + }; + take(r, @as([*]u8, @ptrCast(items))[0 .. @as(usize, n) * item_size]); + out_count.* = @intCast(n); + return items; +} + +// ── Records ───────────────────────────────────────────────────────── + +// Each record type lists its pointer fields once. An array is written as its +// count, then every record's raw bytes with the pointer fields nulled (so +// encoding is a pure function of the content), then each record's strings in +// field order. + +const DEF_STRS = [_][]const u8{ + "name", "qualified_name", "label", "file_path", "signature", + "return_type", "receiver", "docstring", "parent_class", "route_path", + "route_method", "structural_profile", "body_tokens", "variants", "alias_value", + "http_client", "http_base_url", +}; +const DEF_STRVS = [_][]const u8{ "decorators", "base_classes", "param_names", "param_types", "return_types" }; + +const Def = struct { + const T = c.CBMDefinition; + const Fingerprint = @typeInfo(@FieldType(T, "fingerprint")).pointer.child; + + fn nullFields(x: *T) void { + inline for (DEF_STRS) |f| @field(x, f) = null; + inline for (DEF_STRVS) |f| @field(x, f) = null; + x.fingerprint = null; + } + + fn enc(b: *SpillBuf, x: *const T) void { + inline for (DEF_STRS) |f| putStr(b, @field(x, f)); + inline for (DEF_STRVS) |f| putStrv(b, @ptrCast(@field(x, f))); + if (x.fingerprint != null and x.fingerprint_k > 0) { + const k = ix(x.fingerprint_k); + putU32(b, @intCast(k)); + put(b, std.mem.sliceAsBytes(x.fingerprint[0..k])); + } else { + putU32(b, SPILL_NULL); + } + } + + fn dec(r: *SpillRd, x: *T) void { + inline for (DEF_STRS) |f| @field(x, f) = takeStr(r); + inline for (DEF_STRVS) |f| @field(x, f) = @ptrCast(takeStrv(r)); + const k = takeU32(r); + x.fingerprint = null; + if (r.failed or k == SPILL_NULL) return; + if (k > r.left / @sizeOf(Fingerprint)) { + r.failed = true; + return; + } + x.fingerprint = @ptrCast(@alignCast(c.cbm_arena_alloc(r.arena, @as(usize, k) * @sizeOf(Fingerprint)))); + if (x.fingerprint == null) { + r.failed = true; + return; + } + take(r, std.mem.sliceAsBytes(x.fingerprint[0..k])); + } +}; + +const CALL_STRS = [_][]const u8{ "callee_name", "enclosing_func_qn", "first_string_arg", "second_arg_name" }; +const ARG_STRS = [_][]const u8{ "expr", "value", "url_value", "keyword" }; + +const Call = struct { + const T = c.CBMCall; + + fn nullFields(x: *T) void { + inline for (CALL_STRS) |f| @field(x, f) = null; + for (&x.args) |*arg| { + inline for (ARG_STRS) |f| @field(arg, f) = null; + } + } + + fn enc(b: *SpillBuf, x: *const T) void { + inline for (CALL_STRS) |f| putStr(b, @field(x, f)); + for (&x.args) |*arg| { + inline for (ARG_STRS) |f| putStr(b, @field(arg, f)); + } + } + + fn dec(r: *SpillRd, x: *T) void { + inline for (CALL_STRS) |f| @field(x, f) = takeStr(r); + for (&x.args) |*arg| { + inline for (ARG_STRS) |f| @field(arg, f) = takeStr(r); + } + } +}; + +/// Records whose pointer fields are only strings. +fn StringRecord(comptime Rec: type, comptime fields: []const []const u8) type { + return struct { + const T = Rec; + + fn nullFields(x: *T) void { + inline for (fields) |f| @field(x, f) = null; + } + + fn enc(b: *SpillBuf, x: *const T) void { + inline for (fields) |f| putStr(b, @field(x, f)); + } + + fn dec(r: *SpillRd, x: *T) void { + inline for (fields) |f| @field(x, f) = takeStr(r); + } + }; +} + +const Import = StringRecord(c.CBMImport, &.{ "local_name", "module_path", "member" }); +const Usage = StringRecord(c.CBMUsage, &.{ "ref_name", "enclosing_func_qn" }); +const Throw = StringRecord(c.CBMThrow, &.{ "exception_name", "enclosing_func_qn" }); +const ReadWrite = StringRecord(c.CBMReadWrite, &.{ "var_name", "enclosing_func_qn" }); +const TypeRef = StringRecord(c.CBMTypeRef, &.{ "type_name", "enclosing_func_qn" }); +const EnvAccess = StringRecord(c.CBMEnvAccess, &.{ "env_key", "enclosing_func_qn" }); +const TypeAssign = StringRecord(c.CBMTypeAssign, &.{ "var_name", "type_name", "enclosing_func_qn" }); +const ImplTrait = StringRecord(c.CBMImplTrait, &.{ "trait_name", "struct_name" }); +const ResolvedCall = StringRecord(c.CBMResolvedCall, &.{ "caller_qn", "callee_qn", "strategy", "reason" }); +const StringRef = StringRecord(c.CBMStringRef, &.{ "value", "enclosing_func_qn", "key_path" }); +const InfraBinding = StringRecord(c.CBMInfraBinding, &.{ "source_name", "target_url", "broker" }); +const Channel = StringRecord(c.CBMChannel, &.{ "channel_name", "transport", "enclosing_func_qn" }); + +fn encodeArray(b: *SpillBuf, arr: anytype, comptime R: type) void { + putU32(b, @bitCast(arr.count)); + const count: usize = if (arr.count > 0) ix(arr.count) else 0; + for (0..count) |i| { + var raw: R.T = undefined; + @memcpy(std.mem.asBytes(&raw), std.mem.asBytes(util.at(arr.items, i))); + R.nullFields(&raw); + put(b, std.mem.asBytes(&raw)); + } + for (0..count) |i| { + R.enc(b, util.at(arr.items, i)); + } +} + +fn decodeArray(r: *SpillRd, arr: anytype, comptime R: type) void { + arr.items = @ptrCast(@alignCast(takeItems(r, @sizeOf(R.T), &arr.count))); + arr.cap = arr.count; + var i: usize = 0; + while (i < arr.count and !r.failed) : (i += 1) { + R.dec(r, util.at(arr.items, i)); + } +} + +/// Scalar result fields, written as one block so a decode restores them in +/// one step. +const SpillScalars = extern struct { + has_error: bool, + parse_incomplete: bool, + parse_unusable: bool, + is_test_file: bool, + error_region_count: i32, + imports_count: i32, + cached_lang: i32, +}; + +pub export fn cbm_result_spill_encode(result: *const FileResult, out_len: *usize) MutStr { + var b: SpillBuf = .{}; + putU32(&b, SPILL_MAGIC); + encodeArray(&b, &result.defs, Def); + encodeArray(&b, &result.calls, Call); + encodeArray(&b, &result.imports, Import); + encodeArray(&b, &result.usages, Usage); + encodeArray(&b, &result.throws, Throw); + encodeArray(&b, &result.rw, ReadWrite); + encodeArray(&b, &result.type_refs, TypeRef); + encodeArray(&b, &result.env_accesses, EnvAccess); + encodeArray(&b, &result.type_assigns, TypeAssign); + encodeArray(&b, &result.impl_traits, ImplTrait); + encodeArray(&b, &result.resolved_calls, ResolvedCall); + encodeArray(&b, &result.string_refs, StringRef); + encodeArray(&b, &result.infra_bindings, InfraBinding); + encodeArray(&b, &result.channels, Channel); + putStr(&b, result.module_qn); + putStr(&b, result.namespace_name); + putStrv(&b, @ptrCast(result.exports)); + putStrv(&b, @ptrCast(result.constants)); + putStrv(&b, @ptrCast(result.global_vars)); + putStrv(&b, @ptrCast(result.macros)); + putStr(&b, result.error_msg); + putStr(&b, result.error_ranges); + const scalars: SpillScalars = .{ + .has_error = result.has_error, + .parse_incomplete = result.parse_incomplete, + .parse_unusable = result.parse_unusable, + .is_test_file = result.is_test_file, + .error_region_count = result.error_region_count, + .imports_count = result.imports_count, + .cached_lang = @bitCast(result.cached_lang), + }; + put(&b, std.mem.asBytes(&scalars)); + putBytes(&b, result.source, if (result.source != null) ix(result.source_len) else 0); + if (b.failed) { + c.free(b.data); + return null; + } + out_len.* = b.len; + return b.data; +} + +pub export fn cbm_result_spill_decode(data: [*]const u8, len: usize) ?*FileResult { + const result: *FileResult = @as(?*FileResult, interop.callocN(FileResult, 1)) orelse return null; + c.cbm_arena_init(&result.arena); + var r: SpillRd = .{ .p = data, .left = len, .arena = &result.arena }; + if (takeU32(&r) != SPILL_MAGIC) r.failed = true; + decodeArray(&r, &result.defs, Def); + decodeArray(&r, &result.calls, Call); + decodeArray(&r, &result.imports, Import); + decodeArray(&r, &result.usages, Usage); + decodeArray(&r, &result.throws, Throw); + decodeArray(&r, &result.rw, ReadWrite); + decodeArray(&r, &result.type_refs, TypeRef); + decodeArray(&r, &result.env_accesses, EnvAccess); + decodeArray(&r, &result.type_assigns, TypeAssign); + decodeArray(&r, &result.impl_traits, ImplTrait); + decodeArray(&r, &result.resolved_calls, ResolvedCall); + decodeArray(&r, &result.string_refs, StringRef); + decodeArray(&r, &result.infra_bindings, InfraBinding); + decodeArray(&r, &result.channels, Channel); + result.module_qn = takeStr(&r); + result.namespace_name = takeStr(&r); + result.exports = @ptrCast(takeStrv(&r)); + result.constants = @ptrCast(takeStrv(&r)); + result.global_vars = @ptrCast(takeStrv(&r)); + result.macros = @ptrCast(takeStrv(&r)); + result.error_msg = takeStr(&r); + result.error_ranges = takeStr(&r); + var scalars: SpillScalars = undefined; + take(&r, std.mem.asBytes(&scalars)); + result.has_error = scalars.has_error; + result.parse_incomplete = scalars.parse_incomplete; + result.parse_unusable = scalars.parse_unusable; + result.is_test_file = scalars.is_test_file; + result.error_region_count = scalars.error_region_count; + result.imports_count = scalars.imports_count; + result.cached_lang = @bitCast(scalars.cached_lang); + result.source = takeBytes(&r, &result.source_len); + if (r.failed or r.left != 0) { + c.cbm_free_result(result); + return null; + } + return result; +} + +// ── Store ─────────────────────────────────────────────────────────── + +const SpillSlot = extern struct { + offset: i64, // -1: empty + len: i64, + infra_bindings: c_int, + string_refs: c_int, +}; + +const Spill = struct { + path: [1024]u8, + file: ?*c.FILE, // opened on the first park + open_failed: bool, + end: i64, + mu: c.cbm_mutex_t, + slots: [*]SpillSlot, + slot_count: c_int, + parked: AtomicI64, + bytes: AtomicI64, + loads: AtomicI64, + load_failures: AtomicI64, +}; + +extern fn _getpid() c_int; +extern fn _fseeki64(file: ?*c.FILE, offset: i64, whence: c_int) c_int; + +fn currentPid() c_int { + return if (is_windows) _getpid() else c.getpid(); +} + +fn spillFseek(file: ?*c.FILE, offset: i64, whence: c_int) c_int { + return if (is_windows) _fseeki64(file, offset, whence) else c.fseeko(file, offset, whence); +} + +var sequence: std.atomic.Value(c_int) = .init(0); + +pub export fn cbm_result_spill_open(dir: CStr, slots: c_int) ?*Spill { + if (dir == null or slots <= 0) return null; + const sp: ?*Spill = interop.callocN(Spill, 1); + const table: ?[*]SpillSlot = interop.callocN(SpillSlot, ix(slots)); + if (sp == null or table == null) { + c.free(sp); + c.free(@ptrCast(table)); + return null; + } + const n = c.snprintf( + &sp.?.path, + sp.?.path.len, + "%s/spill-%d-%d.bin", + dir, + currentPid(), + sequence.fetchAdd(1, .seq_cst), + ); + if (n < 0 or @as(usize, @intCast(n)) >= sp.?.path.len) { + c.free(sp); + c.free(@ptrCast(table)); + return null; + } + for (table.?[0..ix(slots)]) |*slot| slot.offset = -1; + sp.?.slots = table.?; + sp.?.slot_count = slots; + c.cbm_mutex_init(&sp.?.mu); + return sp; +} + +/// Under sp.mu. Creates and the file on first use. +fn spillEnsureFile(sp: *Spill) bool { + if (sp.file != null or sp.open_failed) return sp.file != null; + var dir: [1024]u8 = undefined; + _ = c.snprintf(&dir, dir.len, "%s", @as(CStr, &sp.path)); + const slash: MutStr = c.strrchr(&dir, '/'); + if (slash != null) { + slash[0] = 0; + _ = c.cbm_mkdir_p(&dir, 0o700); + } + sp.file = c.cbm_fopen(&sp.path, "w+b"); + if (sp.file == null) { + sp.open_failed = true; + interop.logWarn("mem.spill.open_failed", .{ @as(CStr, "path"), @as(CStr, &sp.path) }); + } + return sp.file != null; +} + +pub export fn cbm_result_spill_park(sp_c: ?*Spill, slot: c_int, result_c: ?*FileResult) bool { + const sp = sp_c orelse return false; + const result = result_c orelse return false; + if (slot < 0 or slot >= sp.slot_count) return false; + var len: usize = 0; + const data = cbm_result_spill_encode(result, &len); + if (data == null) return false; + var ok = false; + c.cbm_mutex_lock(&sp.mu); + if (sp.slots[ix(slot)].offset < 0 and spillEnsureFile(sp) and + spillFseek(sp.file, sp.end, c.SEEK_SET) == 0 and c.fwrite(data, 1, len, sp.file) == len) + { + sp.slots[ix(slot)] = .{ + .offset = sp.end, + .len = @intCast(len), + .infra_bindings = result.infra_bindings.count, + .string_refs = result.string_refs.count, + }; + sp.end += @intCast(len); + ok = true; + } + c.cbm_mutex_unlock(&sp.mu); + c.free(data); + if (ok) { + _ = sp.parked.fetchAdd(1, .seq_cst); + _ = sp.bytes.fetchAdd(@intCast(len), .seq_cst); + c.cbm_free_result(result); + } + return ok; +} + +pub export fn cbm_result_spill_has(sp_c: ?*const Spill, slot: c_int) bool { + // A slot is written before its result leaves the cache and read only by + // later phases, so no lock is needed here. + const sp = sp_c orelse return false; + return slot >= 0 and slot < sp.slot_count and sp.slots[ix(slot)].offset >= 0; +} + +pub export fn cbm_result_spill_infra_binding_count(sp: ?*const Spill, slot: c_int) c_int { + return if (cbm_result_spill_has(sp, slot)) sp.?.slots[ix(slot)].infra_bindings else 0; +} + +pub export fn cbm_result_spill_string_ref_count(sp: ?*const Spill, slot: c_int) c_int { + return if (cbm_result_spill_has(sp, slot)) sp.?.slots[ix(slot)].string_refs else 0; +} + +pub export fn cbm_result_spill_load(sp_c: ?*Spill, slot: c_int) ?*FileResult { + if (!cbm_result_spill_has(sp_c, slot)) return null; + const sp = sp_c.?; + const entry = sp.slots[ix(slot)]; + const entry_len: usize = @intCast(entry.len); + const data: MutStr = @ptrCast(c.malloc(entry_len)); + if (data == null) return null; + c.cbm_mutex_lock(&sp.mu); + const read = c.fflush(sp.file) == 0 and spillFseek(sp.file, entry.offset, c.SEEK_SET) == 0 and + c.fread(data, 1, entry_len, sp.file) == entry_len; + c.cbm_mutex_unlock(&sp.mu); + const result = if (read) cbm_result_spill_decode(data, entry_len) else null; + c.free(data); + if (result != null) { + _ = sp.loads.fetchAdd(1, .seq_cst); + } else { + _ = sp.load_failures.fetchAdd(1, .seq_cst); + interop.log(c.CBM_LOG_ERROR, "mem.spill.load_failed", .{ @as(CStr, "path"), @as(CStr, &sp.path) }); + } + return result; +} + +pub export fn cbm_result_spill_load_failures(sp: ?*const Spill) i64 { + return if (sp) |s| s.load_failures.load(.seq_cst) else 0; +} + +pub export fn cbm_result_spill_stats(sp: ?*const Spill, parked: *i64, bytes: *i64, loads: *i64) void { + parked.* = if (sp) |s| s.parked.load(.seq_cst) else 0; + bytes.* = if (sp) |s| s.bytes.load(.seq_cst) else 0; + loads.* = if (sp) |s| s.loads.load(.seq_cst) else 0; +} + +pub export fn cbm_result_spill_close(sp_c: ?*Spill) void { + const sp = sp_c orelse return; + if (sp.file != null) { + _ = c.fclose(sp.file); + _ = c.cbm_unlink(&sp.path); + } + c.cbm_mutex_destroy(&sp.mu); + c.free(@ptrCast(sp.slots)); + c.free(sp); +} + +// ── Cache access ──────────────────────────────────────────────────── + +pub export fn cbm_result_cache_acquire(cache: [*]?*FileResult, sp: ?*Spill, i: c_int) ?*FileResult { + if (cache[ix(i)]) |cached| return cached; + return cbm_result_spill_load(sp, i); +} + +pub export fn cbm_result_cache_release(cache: [*]?*FileResult, i: c_int, result: ?*FileResult) void { + if (result != null and result != cache[ix(i)]) { + c.cbm_free_result(result); + } +} + +pub export fn cbm_result_cache_present(cache: [*]const ?*FileResult, sp: ?*const Spill, i: c_int) bool { + return cache[ix(i)] != null or cbm_result_spill_has(sp, i); +} diff --git a/src/pipeline/service_bindings.c b/src/pipeline/service_bindings.c deleted file mode 100644 index ea50bc11..00000000 --- a/src/pipeline/service_bindings.c +++ /dev/null @@ -1,319 +0,0 @@ -/* - * service_bindings.c — Classify a call as a service call through what its - * receiver or callee name is bound to. - * - * cbm_pipeline_http_client_call_url a call on an axios.create() instance - * (#1916): the instance's baseURL joined - * with the request path. - * cbm_pipeline_external_import_service a callee an import binds to an external - * client library (`send` under - * `from paho.mqtt import publish as send`). - * - * Used by the CALLS resolution in pass_parallel.c. - */ -#include "foundation/constants.h" -#include "pipeline/pipeline.h" -#include "pipeline/pipeline_internal.h" -#include "graph_buffer/graph_buffer.h" -#include "foundation/limits.h" -#include "cbm.h" -#include "service_patterns.h" - -#include -#include -#include -#include - -/* The graph node whose QN is cbm_pipeline_fqn_compute(project, rel_path, name), - * or NULL. */ -static const cbm_gbuf_node_t *pc_find_by_computed_qn(const cbm_gbuf_t *gbuf, const char *project, - const char *rel_path, const char *name) { - char *qn = cbm_pipeline_fqn_compute(project, rel_path, name); - const cbm_gbuf_node_t *n = qn ? cbm_gbuf_find_by_qn(gbuf, qn) : NULL; - free(qn); - return n; -} - -/* ── #1916: axios client-instance calls ─────────────────────────── */ - -/* Copy the plain (unescaped) JSON string value of "key" in props into out. - * False when absent, escaped, or too long — an unreadable base is unknown. */ -static bool pc_json_str_prop(const char *props, const char *key, char *out, size_t out_sz) { - if (!props) { - return false; - } - char pat[CBM_SZ_64]; - snprintf(pat, sizeof(pat), "\"%s\":\"", key); - const char *p = strstr(props, pat); - if (!p) { - return false; - } - p += strlen(pat); - const char *e = strchr(p, '"'); - if (!e) { - return false; - } - size_t len = (size_t)(e - p); - if (len >= out_sz || memchr(p, '\\', len) != NULL) { - return false; - } - memcpy(out, p, len); - out[len] = '\0'; - return true; -} - -/* A node the extractor marked as an axios instance. `base` receives the - * literal baseURL ("" when unknown). */ -static bool pc_node_is_http_client(const cbm_gbuf_node_t *n, char *base, size_t base_sz) { - char client[CBM_SZ_32]; - if (!n || !pc_json_str_prop(n->properties_json, "http_client", client, sizeof(client))) { - return false; - } - if (!pc_json_str_prop(n->properties_json, "http_base_url", base, base_sz)) { - base[0] = '\0'; - } - return true; -} - -/* The module-level Variable `name` declared in file `file_rel`. */ -static const cbm_gbuf_node_t *pc_module_var(const cbm_gbuf_t *gbuf, const char *project, - const char *file_rel, const char *name) { - if (!file_rel) { - return NULL; - } - const cbm_gbuf_node_t *n = pc_find_by_computed_qn(gbuf, project, file_rel, name); - return (n && n->label && strcmp(n->label, "Variable") == 0) ? n : NULL; -} - -/* Resolve the binding `recv` names in the calling file to a client node: - * a same-file declaration first, then its ES import (default import → the - * module's default export; named import → that module's Variable). */ -static bool pc_receiver_client_base(const cbm_gbuf_t *gbuf, const char *project, const char *rel, - const CBMFileResult *result, const char **imp_keys, - const char **imp_vals, int imp_count, const char *recv, - char *base, size_t base_sz) { - if (pc_node_is_http_client(pc_module_var(gbuf, project, rel, recv), base, base_sz)) { - return true; - } - const CBMImport *imp = NULL; - for (int i = 0; result && i < result->imports.count; i++) { - const CBMImport *it = &result->imports.items[i]; - if (it->local_name && strcmp(it->local_name, recv) == 0) { - imp = it; - break; - } - } - if (!imp) { - return false; - } - for (int i = 0; i < imp_count; i++) { - if (!imp_keys[i] || strcmp(imp_keys[i], recv) != 0 || !imp_vals[i]) { - continue; - } - const cbm_gbuf_node_t *t = cbm_gbuf_find_by_qn(gbuf, imp_vals[i]); - if (!t || !t->label) { - return false; - } - if (strcmp(t->label, "Variable") == 0) { - return pc_node_is_http_client(t, base, base_sz); - } - if (strcmp(t->label, "Module") != 0) { - return false; - } - if (imp->is_default) { - return pc_node_is_http_client(t, base, base_sz); - } - return pc_node_is_http_client(pc_module_var(gbuf, project, t->file_path, recv), base, - base_sz); - } - return false; -} - -/* axios instance request methods (the verb after the receiver). */ -static bool pc_is_axios_verb(const char *verb) { - static const char *const verbs[] = {"get", "post", "put", "delete", - "patch", "head", "options", NULL}; - for (int i = 0; verbs[i]; i++) { - if (strcmp(verb, verbs[i]) == 0) { - return true; - } - } - return false; -} - -/* Copy the plain-identifier receiver of `callee` (text before `dot`) into - * recv. False for `this.api.get`, `a().get` and other non-binding receivers. */ -static bool pc_plain_receiver(const char *callee, const char *dot, char *recv, size_t recv_sz) { - size_t rlen = (size_t)(dot - callee); - if (rlen == 0 || rlen >= recv_sz) { - return false; - } - for (size_t i = 0; i < rlen; i++) { - char c = callee[i]; - bool ident = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || - c == '_' || c == '$'; - if (!ident) { - return false; - } - } - memcpy(recv, callee, rlen); - recv[rlen] = '\0'; - return true; -} - -bool cbm_pipeline_http_client_call_url(const cbm_gbuf_t *gbuf, const char *project, const char *rel, - const CBMFileResult *result, const char **imp_keys, - const char **imp_vals, int imp_count, const CBMCall *call, - char *out, size_t out_sz) { - const char *callee = call ? call->callee_name : NULL; - const char *url = call ? call->first_string_arg : NULL; - if (!gbuf || !project || !callee || !url || (url[0] != '/' && !strstr(url, "://"))) { - return false; - } - const char *dot = strrchr(callee, '.'); - char recv[CBM_SZ_128]; - if (!dot || !pc_is_axios_verb(dot + SKIP_ONE) || - !pc_plain_receiver(callee, dot, recv, sizeof(recv))) { - return false; - } - char base[CBM_SZ_256]; - if (!pc_receiver_client_base(gbuf, project, rel, result, imp_keys, imp_vals, imp_count, recv, - base, sizeof(base))) { - return false; - } - /* axios combineURLs: an absolute request URL ignores baseURL; otherwise - * the base (trailing '/' trimmed) is joined with the '/'-leading path. */ - if (strstr(url, "://") || base[0] == '\0') { - int n = snprintf(out, out_sz, "%s", url); - return n > 0 && (size_t)n < out_sz; - } - size_t blen = strlen(base); - while (blen > 0 && base[blen - SKIP_ONE] == '/') { - base[--blen] = '\0'; - } - int n = snprintf(out, out_sz, "%s%s", base, url); - return n > 0 && (size_t)n < out_sz; -} - -/* ── Calls through an import of an external library ───────────────── - * See cbm_pipeline_external_import_service in pipeline_internal.h. */ - -static bool pc_external_import_lang(CBMLanguage lang) { - return lang == CBM_LANG_PYTHON || lang == CBM_LANG_JAVASCRIPT || lang == CBM_LANG_TYPESCRIPT || - lang == CBM_LANG_TSX; -} - -/* Whether an import can name an external module at all: a relative path - * never does, nor (JS/TS) an absolute path or a project-root alias. */ -static bool pc_import_may_be_external(CBMLanguage lang, const CBMImport *imp) { - const char *m = imp->module_path; - if (!m || !m[0] || m[0] == '.' || strstr(m, " as ")) { - return false; - } - if (lang == CBM_LANG_PYTHON) { - return true; - } - return m[0] != '/' && m[0] != '~' && m[0] != '#' && strncmp(m, "@/", PAIR_LEN) != 0; -} - -/* Two imports bind the same name to the same thing. */ -static bool pc_same_binding(const CBMImport *a, const CBMImport *b) { - return strcmp(a->module_path, b->module_path) == 0 && - (a->member == b->member || - (a->member && b->member && strcmp(a->member, b->member) == 0)); -} - -/* The callee as its import makes it; false when the import adds nothing to - * the callee's own spelling. - * Python binds `local` to module_path: `from m import x as y` and - * `import m.x as y` both store y -> m.x, so `y.f` is m.x.f. A plain - * `import a` binds `a` and the callee already spells its path. - * JS/TS: a named import binds the module's member, alias or not - * (`navigate` under `import { get as navigate } from 'axios'` is - * axios.get); a default or namespace import binds the module itself, so - * `ax.get` is axios.get. */ -static bool pc_spell_through_import(CBMLanguage lang, const CBMImport *imp, const char *callee, - size_t root_len, char *out, size_t out_sz) { - static const char node_scheme[] = "node:"; - const char *module = imp->module_path; - const char *rest = callee + root_len; - int n; - if (lang == CBM_LANG_PYTHON) { - size_t local_len = strlen(imp->local_name); - if (strncmp(module, imp->local_name, local_len) == 0 && - (module[local_len] == '\0' || module[local_len] == '.')) { - return false; - } - n = snprintf(out, out_sz, "%s%s", module, rest); - } else { - if (strncmp(module, node_scheme, sizeof(node_scheme) - SKIP_ONE) == 0) { - module += sizeof(node_scheme) - SKIP_ONE; - } - n = imp->member ? snprintf(out, out_sz, "%s.%s%s", module, imp->member, rest) - : snprintf(out, out_sz, "%s%s", module, rest); - } - return n > 0 && (size_t)n < out_sz; -} - -cbm_svc_kind_t cbm_pipeline_external_import_service(CBMLanguage lang, - const CBMImportArray *imports, - const char **imp_keys, int imp_count, - const CBMCall *call, char *spelled, - size_t spelled_sz) { - if (!spelled || spelled_sz == 0) { - return CBM_SVC_NONE; - } - spelled[0] = '\0'; - const char *callee = call ? call->callee_name : NULL; - const char *u = call ? call->first_string_arg : NULL; - if (!imports || imports->count <= 0 || !callee || !callee[0] || !u || !u[0] || - !pc_external_import_lang(lang) || - (lang == CBM_LANG_PYTHON && call->callee_is_locally_bound) || - cbm_service_pattern_route_method(callee) != NULL || - cbm_service_pattern_is_global_fetch(callee)) { - return CBM_SVC_NONE; - } - const char *dot = strchr(callee, '.'); - size_t root_len = dot ? (size_t)(dot - callee) : strlen(callee); - char root[CBM_SZ_256]; - if (root_len == 0 || root_len >= sizeof(root)) { - return CBM_SVC_NONE; - } - memcpy(root, callee, root_len); - root[root_len] = '\0'; - /* The import map holds exactly the imports that resolved to a project - * node: such a name is a project module, whatever its spelling. */ - for (int i = 0; i < imp_count; i++) { - if (imp_keys && imp_keys[i] && strcmp(imp_keys[i], root) == 0) { - return CBM_SVC_NONE; - } - } - /* One external binding only: a relative import of the name, or a second - * import that binds it to something else, leaves it ambiguous. */ - const CBMImport *imp = NULL; - for (int i = 0; i < imports->count; i++) { - const CBMImport *cand = &imports->items[i]; - if (!cand->local_name || strcmp(cand->local_name, root) != 0) { - continue; - } - if (!pc_import_may_be_external(lang, cand) || (imp && !pc_same_binding(imp, cand))) { - return CBM_SVC_NONE; - } - imp = cand; - } - if (!imp || !pc_spell_through_import(lang, imp, callee, root_len, spelled, spelled_sz)) { - spelled[0] = '\0'; - return CBM_SVC_NONE; - } - cbm_svc_kind_t svc = cbm_service_pattern_match(spelled); - bool is_url = u[0] == '/' || strstr(u, "://") != NULL; - bool is_topic = svc == CBM_SVC_ASYNC && strlen(u) > PAIR_LEN; - /* An HTTP library also exports helpers that take a URL without sending - * one (urllib.parse.urljoin): only a request verb is a request. */ - bool is_request = svc == CBM_SVC_HTTP && cbm_service_pattern_http_method(spelled) != NULL; - if ((is_request && is_url) || (svc == CBM_SVC_ASYNC && (is_url || is_topic))) { - return svc; - } - spelled[0] = '\0'; - return CBM_SVC_NONE; -} diff --git a/src/pipeline/service_bindings.zig b/src/pipeline/service_bindings.zig new file mode 100644 index 00000000..a2556f39 --- /dev/null +++ b/src/pipeline/service_bindings.zig @@ -0,0 +1,303 @@ +//! service_bindings.zig — Classify a call as a service call through what its +//! receiver or callee name is bound to. +//! +//! cbm_pipeline_http_client_call_url a call on an axios.create() instance +//! (#1916): the instance's baseURL joined +//! with the request path. +//! cbm_pipeline_external_import_service a callee an import binds to an external +//! client library (`send` under +//! `from paho.mqtt import publish as send`). +//! +//! Used by the CALLS resolution in pass_parallel. + +const std = @import("std"); +const c = @import("c"); +const util = @import("util.zig"); + +const CStr = util.CStr; +const MutStr = util.MutStr; +const streq = util.streq; +const Node = c.cbm_gbuf_node_t; + +/// The graph node whose QN is cbm_pipeline_fqn_compute(project, rel_path, name), +/// or NULL. +fn pcFindByComputedQn(gbuf: ?*const c.cbm_gbuf_t, project: CStr, rel_path: CStr, name: CStr) ?*const Node { + const qn: MutStr = c.cbm_pipeline_fqn_compute(project, rel_path, name); + const n: ?*const Node = if (qn != null) c.cbm_gbuf_find_by_qn(gbuf, qn) else null; + c.free(qn); + return n; +} + +// ── #1916: axios client-instance calls ────────────────────────────── + +/// Copy the plain (unescaped) JSON string value of "key" in props into out. +/// False when absent, escaped, or too long — an unreadable base is unknown. +fn pcJsonStrProp(props: CStr, key: CStr, out: []u8) bool { + if (props == null) return false; + var pat: [c.CBM_SZ_64]u8 = undefined; + _ = c.snprintf(&pat, pat.len, "\"%s\":\"", key); + var p: CStr = c.strstr(props, &pat); + if (p == null) return false; + p += c.strlen(&pat); + const e: CStr = c.strchr(p, '"'); + if (e == null) return false; + const len = util.ptrDiff(e, p); + if (len >= out.len or c.memchr(p, '\\', len) != null) return false; + @memcpy(out[0..len], p[0..len]); + out[len] = 0; + return true; +} + +/// A node the extractor marked as an axios instance. `base` receives the +/// literal baseURL ("" when unknown). +fn pcNodeIsHttpClient(n_c: ?*const Node, base: []u8) bool { + var client: [c.CBM_SZ_32]u8 = undefined; + const n = n_c orelse return false; + if (!pcJsonStrProp(n.properties_json, "http_client", &client)) return false; + if (!pcJsonStrProp(n.properties_json, "http_base_url", base)) base[0] = 0; + return true; +} + +/// The module-level Variable `name` declared in file `file_rel`. +fn pcModuleVar(gbuf: ?*const c.cbm_gbuf_t, project: CStr, file_rel: CStr, name: CStr) ?*const Node { + if (file_rel == null) return null; + const n = pcFindByComputedQn(gbuf, project, file_rel, name) orelse return null; + return if (n.label != null and streq(n.label, "Variable")) n else null; +} + +/// Resolve the binding `recv` names in the calling file to a client node: +/// a same-file declaration first, then its ES import (default import → the +/// module's default export; named import → that module's Variable). +fn pcReceiverClientBase( + gbuf: ?*const c.cbm_gbuf_t, + project: CStr, + rel: CStr, + result: ?*const c.CBMFileResult, + imp_keys: [*c]const CStr, + imp_vals: [*c]const CStr, + imp_count: c_int, + recv: CStr, + base: []u8, +) bool { + if (pcNodeIsHttpClient(pcModuleVar(gbuf, project, rel, recv), base)) return true; + var imp: ?*const c.CBMImport = null; + if (result) |res| { + var i: usize = 0; + while (i < res.imports.count) : (i += 1) { + const it = util.at(res.imports.items, i); + if (it.local_name != null and streq(it.local_name, recv)) { + imp = it; + break; + } + } + } + const found = imp orelse return false; + var i: usize = 0; + while (i < imp_count) : (i += 1) { + if (imp_keys[i] == null or !streq(imp_keys[i], recv) or imp_vals[i] == null) continue; + const t: *const Node = @as(?*const Node, c.cbm_gbuf_find_by_qn(gbuf, imp_vals[i])) orelse return false; + if (t.label == null) return false; + if (streq(t.label, "Variable")) return pcNodeIsHttpClient(t, base); + if (!streq(t.label, "Module")) return false; + if (found.is_default) return pcNodeIsHttpClient(t, base); + return pcNodeIsHttpClient(pcModuleVar(gbuf, project, t.file_path, recv), base); + } + return false; +} + +/// axios instance request methods (the verb after the receiver). +fn pcIsAxiosVerb(verb: CStr) bool { + const verbs = [_][*:0]const u8{ "get", "post", "put", "delete", "patch", "head", "options" }; + for (verbs) |v| { + if (streq(verb, v)) return true; + } + return false; +} + +/// Copy the plain-identifier receiver of `callee` (text before `dot`) into +/// recv. False for `this.api.get`, `a().get` and other non-binding receivers. +fn pcPlainReceiver(callee: CStr, dot: CStr, recv: []u8) bool { + const rlen = util.ptrDiff(dot, callee); + if (rlen == 0 or rlen >= recv.len) return false; + for (callee[0..rlen]) |ch| { + const ident = (ch >= 'a' and ch <= 'z') or (ch >= 'A' and ch <= 'Z') or + (ch >= '0' and ch <= '9') or ch == '_' or ch == '$'; + if (!ident) return false; + } + @memcpy(recv[0..rlen], callee[0..rlen]); + recv[rlen] = 0; + return true; +} + +pub export fn cbm_pipeline_http_client_call_url( + gbuf: ?*const c.cbm_gbuf_t, + project: CStr, + rel: CStr, + result: ?*const c.CBMFileResult, + imp_keys: [*c]CStr, + imp_vals: [*c]CStr, + imp_count: c_int, + call: ?*const c.CBMCall, + out: MutStr, + out_sz: usize, +) bool { + const callee: CStr = if (call) |cl| cl.callee_name else null; + const url: CStr = if (call) |cl| cl.first_string_arg else null; + if (gbuf == null or project == null or callee == null or url == null or + (url[0] != '/' and c.strstr(url, "://") == null)) + { + return false; + } + const dot: CStr = c.strrchr(callee, '.'); + var recv: [c.CBM_SZ_128]u8 = undefined; + if (dot == null or !pcIsAxiosVerb(dot + 1) or !pcPlainReceiver(callee, dot, &recv)) { + return false; + } + var base: [c.CBM_SZ_256]u8 = undefined; + if (!pcReceiverClientBase(gbuf, project, rel, result, imp_keys, imp_vals, imp_count, &recv, &base)) { + return false; + } + // axios combineURLs: an absolute request URL ignores baseURL; otherwise + // the base (trailing '/' trimmed) is joined with the '/'-leading path. + if (c.strstr(url, "://") != null or base[0] == 0) { + const n = c.snprintf(out, out_sz, "%s", url); + return n > 0 and @as(usize, @intCast(n)) < out_sz; + } + var blen = c.strlen(&base); + while (blen > 0 and base[blen - 1] == '/') { + blen -= 1; + base[blen] = 0; + } + const n = c.snprintf(out, out_sz, "%s%s", @as(CStr, &base), url); + return n > 0 and @as(usize, @intCast(n)) < out_sz; +} + +// ── Calls through an import of an external library ────────────────── +// See cbm_pipeline_external_import_service in pipeline_internal.h. + +fn pcExternalImportLang(lang: c.CBMLanguage) bool { + return lang == c.CBM_LANG_PYTHON or lang == c.CBM_LANG_JAVASCRIPT or + lang == c.CBM_LANG_TYPESCRIPT or lang == c.CBM_LANG_TSX; +} + +/// Whether an import can name an external module at all: a relative path +/// never does, nor (JS/TS) an absolute path or a project-root alias. +fn pcImportMayBeExternal(lang: c.CBMLanguage, imp: *const c.CBMImport) bool { + const m: CStr = imp.module_path; + if (m == null or m[0] == 0 or m[0] == '.' or c.strstr(m, " as ") != null) return false; + if (lang == c.CBM_LANG_PYTHON) return true; + return m[0] != '/' and m[0] != '~' and m[0] != '#' and c.strncmp(m, "@/", 2) != 0; +} + +/// Two imports bind the same name to the same thing. +fn pcSameBinding(a: *const c.CBMImport, b: *const c.CBMImport) bool { + return streq(a.module_path, b.module_path) and + (a.member == b.member or + (a.member != null and b.member != null and streq(a.member, b.member))); +} + +/// The callee as its import makes it; false when the import adds nothing to +/// the callee's own spelling. +/// Python binds `local` to module_path: `from m import x as y` and +/// `import m.x as y` both store y -> m.x, so `y.f` is m.x.f. A plain +/// `import a` binds `a` and the callee already spells its path. +/// JS/TS: a named import binds the module's member, alias or not +/// (`navigate` under `import { get as navigate } from 'axios'` is +/// axios.get); a default or namespace import binds the module itself, so +/// `ax.get` is axios.get. +fn pcSpellThroughImport( + lang: c.CBMLanguage, + imp: *const c.CBMImport, + callee: CStr, + root_len: usize, + out: MutStr, + out_sz: usize, +) bool { + const node_scheme = "node:"; + var module: CStr = imp.module_path; + const rest: CStr = callee + root_len; + var n: c_int = undefined; + if (lang == c.CBM_LANG_PYTHON) { + const local_len = c.strlen(imp.local_name); + if (c.strncmp(module, imp.local_name, local_len) == 0 and + (module[local_len] == 0 or module[local_len] == '.')) + { + return false; + } + n = c.snprintf(out, out_sz, "%s%s", module, rest); + } else { + if (c.strncmp(module, node_scheme, node_scheme.len) == 0) { + module += node_scheme.len; + } + n = if (imp.member != null) + c.snprintf(out, out_sz, "%s.%s%s", module, imp.member, rest) + else + c.snprintf(out, out_sz, "%s%s", module, rest); + } + return n > 0 and @as(usize, @intCast(n)) < out_sz; +} + +pub export fn cbm_pipeline_external_import_service( + lang: c.CBMLanguage, + imports_c: ?*const c.CBMImportArray, + imp_keys: [*c]CStr, + imp_count: c_int, + call: ?*const c.CBMCall, + spelled: MutStr, + spelled_sz: usize, +) c.cbm_svc_kind_t { + if (spelled == null or spelled_sz == 0) return c.CBM_SVC_NONE; + spelled[0] = 0; + const callee: CStr = if (call) |cl| cl.callee_name else null; + const u: CStr = if (call) |cl| cl.first_string_arg else null; + const imports = imports_c orelse return c.CBM_SVC_NONE; + if (imports.count <= 0 or callee == null or callee[0] == 0 or u == null or u[0] == 0 or + !pcExternalImportLang(lang) or + (lang == c.CBM_LANG_PYTHON and call.?.callee_is_locally_bound) or + c.cbm_service_pattern_route_method(callee) != null or + c.cbm_service_pattern_is_global_fetch(callee)) + { + return c.CBM_SVC_NONE; + } + const dot: CStr = c.strchr(callee, '.'); + const root_len: usize = if (dot != null) util.ptrDiff(dot, callee) else c.strlen(callee); + var root: [c.CBM_SZ_256]u8 = undefined; + if (root_len == 0 or root_len >= root.len) return c.CBM_SVC_NONE; + @memcpy(root[0..root_len], callee[0..root_len]); + root[root_len] = 0; + // The import map holds exactly the imports that resolved to a project + // node: such a name is a project module, whatever its spelling. + var i: usize = 0; + while (i < imp_count) : (i += 1) { + if (imp_keys != null and imp_keys[i] != null and streq(imp_keys[i], &root)) { + return c.CBM_SVC_NONE; + } + } + // One external binding only: a relative import of the name, or a second + // import that binds it to something else, leaves it ambiguous. + var imp: ?*const c.CBMImport = null; + i = 0; + while (i < imports.count) : (i += 1) { + const cand = util.at(imports.items, i); + if (cand.local_name == null or !streq(cand.local_name, &root)) continue; + if (!pcImportMayBeExternal(lang, cand) or (imp != null and !pcSameBinding(imp.?, cand))) { + return c.CBM_SVC_NONE; + } + imp = cand; + } + if (imp == null or !pcSpellThroughImport(lang, imp.?, callee, root_len, spelled, spelled_sz)) { + spelled[0] = 0; + return c.CBM_SVC_NONE; + } + const svc = c.cbm_service_pattern_match(spelled); + const is_url = u[0] == '/' or c.strstr(u, "://") != null; + const is_topic = svc == c.CBM_SVC_ASYNC and c.strlen(u) > 2; + // An HTTP library also exports helpers that take a URL without sending + // one (urllib.parse.urljoin): only a request verb is a request. + const is_request = svc == c.CBM_SVC_HTTP and c.cbm_service_pattern_http_method(spelled) != null; + if ((is_request and is_url) or (svc == c.CBM_SVC_ASYNC and (is_url or is_topic))) { + return svc; + } + spelled[0] = 0; + return c.CBM_SVC_NONE; +} diff --git a/src/pipeline/util.zig b/src/pipeline/util.zig new file mode 100644 index 00000000..daf7e547 --- /dev/null +++ b/src/pipeline/util.zig @@ -0,0 +1,160 @@ +//! util.zig — general C-interop helpers used by the pipeline port. +//! +//! These belong in src/foundation/interop.zig. They live here only while the +//! foundation port is in flight on another branch, and move there once both +//! land. + +const std = @import("std"); +const c = @import("c"); + +pub const CStr = [*c]const u8; +pub const MutStr = [*c]u8; + +// ── Indexing ──────────────────────────────────────────────────────── + +/// A C int used as an array index or element count. A negative value gives +/// 0, so `for (0..ix(n))` runs as often as C's `for (i = 0; i < n; i++)`. +pub inline fn ix(i: c_int) usize { + return @intCast(@max(i, 0)); +} + +// ── foundation/dyn_array.h ────────────────────────────────────────── + +/// CBM_DYN_ARRAY(T): the layout every dyn_array.h macro works on. +pub fn DynArray(comptime T: type) type { + return extern struct { + items: [*c]T = null, + count: c_int = 0, + cap: c_int = 0, + }; +} + +fn DaItem(comptime Da: type) type { + return @typeInfo(@FieldType(@typeInfo(Da).pointer.child, "items")).pointer.child; +} + +/// cbm_da_push: appends item, doubling the capacity when full. Drops the item +/// when realloc fails, as the macro does. +pub fn daPush(da: anytype, item: DaItem(@TypeOf(da))) void { + const Item = DaItem(@TypeOf(da)); + if (da.count >= da.cap) { + const new_cap: c_int = if (da.cap != 0) da.cap * 2 else 8; + const new = c.realloc(@ptrCast(da.items), ix(new_cap) * @sizeOf(Item)) orelse return; + da.items = @ptrCast(@alignCast(new)); + da.cap = new_cap; + } + da.items[ix(da.count)] = item; + da.count += 1; +} + +/// cbm_da_push_ptr: grows the array by one slot and returns a pointer to it. +/// Like the macro, it does not check the realloc result. +pub fn daPushPtr(da: anytype) *DaItem(@TypeOf(da)) { + const Item = DaItem(@TypeOf(da)); + if (da.count >= da.cap) { + da.cap = if (da.cap != 0) da.cap * 2 else 8; + da.items = @ptrCast(@alignCast(c.realloc(@ptrCast(da.items), ix(da.cap) * @sizeOf(Item)))); + } + const slot = &da.items[ix(da.count)]; + da.count += 1; + return slot; +} + +/// cbm_da_free: releases the items and resets the array to empty. +pub fn daFree(da: anytype) void { + c.free(@ptrCast(da.items)); + da.items = null; + da.count = 0; + da.cap = 0; +} + +/// cbm_da_reserve: grows the capacity to at least n; keeps the array as it is +/// when realloc fails. +pub fn daReserve(da: anytype, n: c_int) void { + const Item = DaItem(@TypeOf(da)); + if (n > da.cap) { + if (c.realloc(@ptrCast(da.items), ix(n) * @sizeOf(Item))) |new| { + da.items = @ptrCast(@alignCast(new)); + da.cap = n; + } + } +} + +// ── C strings ─────────────────────────────────────────────────────── + +/// strcmp(a, b) == 0. +pub inline fn streq(a: CStr, b: CStr) bool { + return c.strcmp(a, b) == 0; +} + +/// The bytes between two pointers into the same C string. +pub inline fn ptrDiff(end: anytype, start: anytype) usize { + return @intFromPtr(end) - @intFromPtr(start); +} + +/// &p[i] for a C pointer. Plain indexing gives a `*allowzero T`, which does +/// not coerce to `*T`. +pub inline fn at(p: anytype, i: usize) @TypeOf(&p[0..1][0]) { + // An array would be copied into the parameter: take its element address + // with plain `&array[i]` instead. + if (@typeInfo(@TypeOf(p)) != .pointer) @compileError("at() takes a pointer, not " ++ @typeName(@TypeOf(p))); + return &p[i .. i + 1][0]; +} + +// ── foundation/profile.h ──────────────────────────────────────────── + +/// CBM_PROF_START: the start time of a profiled phase, read only while +/// profiling is active. +pub inline fn profStart() c.struct_timespec { + var ts: c.struct_timespec = undefined; + if (c.cbm_profile_active) c.cbm_profile_now(&ts); + return ts; +} + +/// CBM_PROF_END_N: logs the time since `start` and the item count while +/// profiling is active. CBM_PROF_END is the same with zero items. +pub inline fn profEnd(phase: CStr, sub: CStr, start: *const c.struct_timespec, items: anytype) void { + if (c.cbm_profile_active) { + c.cbm_profile_log_elapsed(phase, sub, start, @intCast(items)); + } +} + +// ── foundation/compat_thread.h ────────────────────────────────────── + +/// cbm_once for Zig callers. A cbm_once_t needs CBM_ONCE_INIT, a C +/// initializer macro that translate-c cannot express on every platform +/// (PTHREAD_ONCE_INIT is a brace initializer on macOS, INIT_ONCE_STATIC_INIT +/// on Windows). Like cbm_once, callers that arrive while the function runs +/// wait for it to finish; they spin instead of blocking. +pub const Once = struct { + state: std.atomic.Value(u8) = .init(not_started), + + const not_started = 0; + const running = 1; + const done = 2; + + pub fn call(self: *Once, comptime init: fn () void) void { + if (self.state.load(.acquire) == done) return; + if (self.state.cmpxchgStrong(not_started, running, .acq_rel, .acquire) == null) { + init(); + self.state.store(done, .release); + return; + } + while (self.state.load(.acquire) != done) std.atomic.spinLoopHint(); + } +}; + +// ── sys/stat.h ────────────────────────────────────────────────────── + +/// S_ISDIR(st.st_mode). +pub inline fn isDirMode(st: *const c.struct_stat) bool { + return (st.st_mode & c.S_IFMT) == c.S_IFDIR; +} + +// ── pipeline_internal.h inline functions ──────────────────────────── + +/// cbm_pipeline_check_cancel. The header's inline version reads a C11 atomic, +/// which translate-c cannot express, so it is written out here. +pub inline fn checkCancel(ctx: *const c.cbm_pipeline_ctx_t) c_int { + return if (@atomicLoad(c_int, ctx.cancelled, .seq_cst) != 0) c.CBM_NOT_FOUND else 0; +} diff --git a/src/pipeline/worker_pool.c b/src/pipeline/worker_pool.c deleted file mode 100644 index 4fa33237..00000000 --- a/src/pipeline/worker_pool.c +++ /dev/null @@ -1,115 +0,0 @@ -/* - * worker_pool.c — Parallel-for dispatch with pthreads. - * - * Uses pthreads with 8MB stacks and atomic work-stealing index. - * GCD is avoided because its worker threads have 512KB stacks, - * which overflows on deeply nested ASTs (tree-sitter + walk_defs). - * - * Each worker pulls indices from a shared atomic counter — zero - * contention, natural load balancing across heterogeneous cores. - */ -#include "pipeline/worker_pool.h" -#include "foundation/constants.h" - -enum { WP_TRUE = 1, WP_MIN = 1, WP_STEP = 1 }; -#include "foundation/platform.h" -#include "foundation/compat_thread.h" - -#include -#include - -/* CBM_WORKER_STACK_SIZE (8 MB per worker) lives in worker_pool.h. */ - -/* ── Serial fallback ─────────────────────────────────────────────── */ - -static void run_serial(int count, cbm_parallel_fn fn, void *ctx) { - for (int i = 0; i < count; i++) { - fn(i, ctx); - } -} - -/* ── pthreads backend ────────────────────────────────────────────── */ - -typedef struct { - cbm_parallel_fn fn; - void *ctx; - _Atomic int *next_idx; - int count; -} pthread_worker_arg_t; - -static void *pthread_worker(void *arg) { - pthread_worker_arg_t *wa = arg; - while (WP_TRUE) { - int idx = atomic_fetch_add_explicit(wa->next_idx, WP_STEP, memory_order_relaxed); - if (idx >= wa->count) { - break; - } - wa->fn(idx, wa->ctx); - } - return NULL; -} - -static void run_pthreads(int count, cbm_parallel_fn fn, void *ctx, int nworkers) { - _Atomic int next_idx = 0; - - pthread_worker_arg_t wa = { - .fn = fn, - .ctx = ctx, - .next_idx = &next_idx, - .count = count, - }; - - cbm_thread_t *threads = (cbm_thread_t *)malloc((size_t)nworkers * sizeof(cbm_thread_t)); - if (!threads) { - run_serial(count, fn, ctx); - return; - } - - for (int i = 0; i < nworkers; i++) { - if (cbm_thread_create(&threads[i], CBM_WORKER_STACK_SIZE, pthread_worker, &wa) != 0) { - /* Failed to create thread — let remaining work run in main thread */ - nworkers = i; - break; - } - } - - /* Main thread also participates */ - while (WP_TRUE) { - int idx = atomic_fetch_add_explicit(&next_idx, WP_STEP, memory_order_relaxed); - if (idx >= count) { - break; - } - fn(idx, ctx); - } - - for (int i = 0; i < nworkers; i++) { - cbm_thread_join(&threads[i]); - } - - free(threads); -} - -/* ── Public API ──────────────────────────────────────────────────── */ - -void cbm_parallel_for(int count, cbm_parallel_fn fn, void *ctx, cbm_parallel_for_opts_t opts) { - if (count <= 0 || !fn) { - return; - } - - /* Determine worker count */ - int nworkers = opts.max_workers; - if (nworkers <= 0) { - nworkers = cbm_default_worker_count(true); - } - if (nworkers < WP_MIN) { - nworkers = SKIP_ONE; - } - - /* Serial fallback: single worker or trivially small workload */ - if (nworkers <= WP_MIN || count <= WP_MIN) { - run_serial(count, fn, ctx); - return; - } - - run_pthreads(count, fn, ctx, nworkers); -} diff --git a/src/pipeline/worker_pool.zig b/src/pipeline/worker_pool.zig new file mode 100644 index 00000000..a53c1c54 --- /dev/null +++ b/src/pipeline/worker_pool.zig @@ -0,0 +1,116 @@ +//! worker_pool.zig — Parallel-for dispatch with pthreads. +//! +//! Implements src/pipeline/worker_pool.h. Uses pthreads with 8MB stacks and +//! an atomic work-stealing index. GCD is avoided because its worker threads +//! have 512KB stacks, which overflows on deeply nested ASTs (tree-sitter + +//! walk_defs). +//! +//! Each worker pulls indices from a shared atomic counter — zero +//! contention, natural load balancing across heterogeneous cores. + +const std = @import("std"); +const c = @import("c"); +const interop = @import("../foundation/interop.zig"); + +const AtomicInt = std.atomic.Value(c_int); + +/// 8 MB per worker: CBM_WORKER_STACK_SIZE in worker_pool.h. +const WORKER_STACK_SIZE: usize = 8 * c.CBM_SZ_1K * c.CBM_SZ_1K; + +const ParallelFn = *const fn (c_int, ?*anyopaque) callconv(.c) void; + +// ── Serial fallback ───────────────────────────────────────────────── + +fn runSerial(count: c_int, func: ParallelFn, ctx: ?*anyopaque) void { + var i: c_int = 0; + while (i < count) : (i += 1) { + func(i, ctx); + } +} + +// ── pthreads backend ──────────────────────────────────────────────── + +const PthreadWorkerArg = struct { + func: ParallelFn, + ctx: ?*anyopaque, + next_idx: *AtomicInt, + count: c_int, +}; + +fn pthreadWorker(arg: ?*anyopaque) callconv(.c) ?*anyopaque { + const wa: *PthreadWorkerArg = @ptrCast(@alignCast(arg)); + while (true) { + const idx = wa.next_idx.fetchAdd(1, .monotonic); + if (idx >= wa.count) break; + wa.func(idx, wa.ctx); + } + return null; +} + +fn runPthreads(count: c_int, func: ParallelFn, ctx: ?*anyopaque, nworkers_in: c_int) void { + var nworkers: usize = @intCast(nworkers_in); + var next_idx: AtomicInt = .init(0); + + var wa: PthreadWorkerArg = .{ + .func = func, + .ctx = ctx, + .next_idx = &next_idx, + .count = count, + }; + + const threads = interop.mallocN(c.cbm_thread_t, nworkers); + if (threads == null) { + runSerial(count, func, ctx); + return; + } + + for (0..nworkers) |i| { + if (c.cbm_thread_create(&threads[i], WORKER_STACK_SIZE, pthreadWorker, &wa) != 0) { + // Failed to create thread — let remaining work run in main thread + nworkers = i; + break; + } + } + + // Main thread also participates + while (true) { + const idx = next_idx.fetchAdd(1, .monotonic); + if (idx >= count) break; + func(idx, ctx); + } + + for (0..nworkers) |i| { + _ = c.cbm_thread_join(&threads[i]); + } + + c.free(@ptrCast(threads)); +} + +// ── Public API ────────────────────────────────────────────────────── + +pub export fn cbm_parallel_for( + count: c_int, + func_c: ?ParallelFn, + ctx: ?*anyopaque, + opts: c.cbm_parallel_for_opts_t, +) void { + const func = func_c orelse return; + if (count <= 0) return; + + // Determine worker count + var nworkers = opts.max_workers; + if (nworkers <= 0) { + nworkers = c.cbm_default_worker_count(true); + } + if (nworkers < 1) { + nworkers = 1; + } + + // Serial fallback: single worker or trivially small workload + if (nworkers <= 1 or count <= 1) { + runSerial(count, func, ctx); + return; + } + + runPthreads(count, func, ctx, nworkers); +} diff --git a/src/semantic/ast_profile.c b/src/semantic/ast_profile.c deleted file mode 100644 index 283180d8..00000000 --- a/src/semantic/ast_profile.c +++ /dev/null @@ -1,417 +0,0 @@ -/* - * ast_profile.c — Extraction-time AST structural signals. - * - * Walks a function body AST (alongside MinHash) to compute: - * Signal 8: Control flow counts, nesting depth, expression types - * Signal 9: Approximate data flow (params in returns/conditions) - * Signal 11: Halstead-lite (unique/total operators/operands) - * - * Pure functions — thread-safe, no shared state. - */ -#include "semantic/ast_profile.h" -#include "foundation/constants.h" -#include "tree_sitter/api.h" - -#include -#include -#include -#include -#include - -/* ── Node type classification ────────────────────────────────────── */ - -enum { - WALK_STACK_CAP = 2048, - HALSTEAD_SET_SIZE = 512, - HALSTEAD_SET_MASK = 511, - PROFILE_FIELD_COUNT = 25, - DEPTH_SCALE = 10, - BASE_DECIMAL_AST = 10, - HALSTEAD_HASH_MUL = 31, -}; - -typedef struct { - TSNode node; - int depth; -} profile_frame_t; - -static bool is_control_if(const char *k) { - return strcmp(k, "if_statement") == 0 || strcmp(k, "if_expression") == 0 || - strcmp(k, "elif_clause") == 0; -} - -static bool is_control_for(const char *k) { - return strcmp(k, "for_statement") == 0 || strcmp(k, "for_range_loop") == 0 || - strcmp(k, "for_expression") == 0 || strcmp(k, "for_in_clause") == 0; -} - -static bool is_control_while(const char *k) { - return strcmp(k, "while_statement") == 0 || strcmp(k, "while_expression") == 0 || - strcmp(k, "do_statement") == 0; -} - -static bool is_control_switch(const char *k) { - return strcmp(k, "switch_statement") == 0 || strcmp(k, "switch_expression") == 0 || - strcmp(k, "match_expression") == 0 || strcmp(k, "type_switch_statement") == 0; -} - -static bool is_control_try(const char *k) { - return strcmp(k, "try_statement") == 0 || strcmp(k, "try_expression") == 0 || - strcmp(k, "catch_clause") == 0 || strcmp(k, "except_clause") == 0; -} - -static bool is_return(const char *k) { - return strcmp(k, "return_statement") == 0 || strcmp(k, "return_expression") == 0; -} - -static bool is_comparison(const char *k) { - return strcmp(k, "binary_expression") == 0 || strcmp(k, "comparison_operator") == 0 || - strcmp(k, "boolean_operator") == 0; -} - -static bool is_arithmetic(const char *k) { - return strcmp(k, "unary_expression") == 0 || strcmp(k, "update_expression") == 0; -} - -static bool is_assignment(const char *k) { - return strcmp(k, "assignment_expression") == 0 || strcmp(k, "assignment_statement") == 0 || - strcmp(k, "augmented_assignment") == 0 || strcmp(k, "short_var_declaration") == 0; -} - -static bool is_string_lit(const char *k) { - return strcmp(k, "string") == 0 || strcmp(k, "string_literal") == 0 || - strcmp(k, "interpreted_string_literal") == 0 || strcmp(k, "raw_string_literal") == 0; -} - -static bool is_number_lit(const char *k) { - return strcmp(k, "number") == 0 || strcmp(k, "integer") == 0 || strcmp(k, "float") == 0 || - strcmp(k, "integer_literal") == 0 || strcmp(k, "float_literal") == 0; -} - -static bool is_bool_lit(const char *k) { - return strcmp(k, "true") == 0 || strcmp(k, "false") == 0; -} - -static bool is_operator_node(const char *k) { - /* Named nodes that represent operations (not data). */ - return is_control_if(k) || is_control_for(k) || is_control_while(k) || is_control_switch(k) || - is_control_try(k) || is_return(k) || is_comparison(k) || is_arithmetic(k) || - is_assignment(k) || strcmp(k, "call_expression") == 0 || - strcmp(k, "member_expression") == 0 || strcmp(k, "subscript_expression") == 0; -} - -static bool is_identifier(const char *k) { - return strcmp(k, "identifier") == 0 || strcmp(k, "field_identifier") == 0 || - strcmp(k, "property_identifier") == 0 || strcmp(k, "type_identifier") == 0; -} - -/* Simple hash set for Halstead unique counting (open addressing). */ -static bool halstead_insert(uint32_t *set, const char *key) { - uint32_t h = 0; - for (const char *p = key; *p; p++) { - h = (h * HALSTEAD_HASH_MUL) + (uint32_t)*p; - } - uint32_t idx = h & HALSTEAD_SET_MASK; - for (int probe = 0; probe < HALSTEAD_SET_SIZE; probe++) { - uint32_t slot = (idx + (uint32_t)probe) & HALSTEAD_SET_MASK; - if (set[slot] == 0) { - set[slot] = h | SKIP_ONE; - return true; /* new */ - } - if (set[slot] == (h | SKIP_ONE)) { - return false; /* existing */ - } - } - return false; -} - -/* Check if an identifier matches any parameter name. */ -static bool is_param_name(const char *ident, const char *source, const char **param_names, - int param_count) { - if (!ident || !source || !param_names || param_count == 0) { - return false; - } - for (int i = 0; i < param_count; i++) { - if (param_names[i] && strcmp(ident, param_names[i]) == 0) { - return true; - } - } - return false; -} - -/* ── Main computation ────────────────────────────────────────────── */ - -/* Count control-flow statement kinds (if/for/while/switch/try/return). */ -static void accumulate_control_flow(const char *kind, cbm_ast_profile_t *out, bool *in_return) { - if (is_control_if(kind)) { - out->if_count++; - } - if (is_control_for(kind)) { - out->for_count++; - } - if (is_control_while(kind)) { - out->while_count++; - } - if (is_control_switch(kind)) { - out->switch_count++; - } - if (is_control_try(kind)) { - out->try_count++; - } - if (is_return(kind)) { - out->return_count++; - *in_return = true; - } -} - -/* Count expression- and literal-class counters for one node. */ -static void accumulate_expressions(const char *kind, cbm_ast_profile_t *out) { - if (is_comparison(kind)) { - out->comparison_ops++; - } - if (is_arithmetic(kind)) { - out->arithmetic_ops++; - } - if (strcmp(kind, "not_operator") == 0 || strcmp(kind, "boolean_operator") == 0) { - out->logical_ops++; - } - if (is_assignment(kind)) { - out->assignment_count++; - out->variable_reassigns++; - } - if (is_string_lit(kind)) { - out->string_literals++; - } - if (is_number_lit(kind)) { - out->number_literals++; - } - if (is_bool_lit(kind)) { - out->bool_literals++; - } -} - -/* Accumulate Halstead operator/operand counts. Leaf identifiers, literals, - * and booleans are "operands"; recognized statement/expression kinds are - * "operators". Uses an open-addressed hash set for unique counting. */ -static void accumulate_halstead(const char *kind, uint32_t child_count, uint32_t *op_set, - uint32_t *operand_set, cbm_ast_profile_t *out) { - if (is_operator_node(kind)) { - out->total_operators++; - if (halstead_insert(op_set, kind)) { - out->unique_operators++; - } - } - if (child_count == 0 && - (is_identifier(kind) || is_string_lit(kind) || is_number_lit(kind) || is_bool_lit(kind))) { - out->total_operands++; - if (halstead_insert(operand_set, kind)) { - out->unique_operands++; - } - out->body_tokens++; - } -} - -/* Data-flow counter update: bump params_in_returns / params_in_conditions - * when the current leaf identifier names a parameter and we're inside the - * corresponding syntactic scope. */ -static void accumulate_data_flow(TSNode node, const char *kind, uint32_t child_count, - const char *source, const char **param_names, int param_count, - bool in_return, bool in_condition, cbm_ast_profile_t *out) { - if (!(child_count == 0 && is_identifier(kind) && source)) { - return; - } - uint32_t start = ts_node_start_byte(node); - uint32_t end = ts_node_end_byte(node); - if (end <= start || (end - start) >= CBM_SZ_128) { - return; - } - char ident_buf[CBM_SZ_128]; - int ilen = (int)(end - start); - memcpy(ident_buf, source + start, (size_t)ilen); - ident_buf[ilen] = '\0'; - if (!is_param_name(ident_buf, source, param_names, param_count)) { - return; - } - if (in_return) { - out->params_in_returns++; - } - if (in_condition) { - out->params_in_conditions++; - } -} - -bool cbm_ast_profile_compute(TSNode func_body, const char *source, const char **param_names, - int param_count, cbm_ast_profile_t *out) { - if (ts_node_is_null(func_body)) { - return false; - } - - memset(out, 0, sizeof(*out)); - out->param_count = (uint16_t)param_count; - - uint32_t op_set[HALSTEAD_SET_SIZE]; - uint32_t operand_set[HALSTEAD_SET_SIZE]; - memset(op_set, 0, sizeof(op_set)); - memset(operand_set, 0, sizeof(operand_set)); - - int total_depth = 0; - int node_count = 0; - bool in_return = false; - bool in_condition = false; - - profile_frame_t stack[WALK_STACK_CAP]; - int top = 0; - stack[top++] = (profile_frame_t){func_body, 0}; - - while (top > 0) { - profile_frame_t frame = stack[--top]; - TSNode node = frame.node; - int depth = frame.depth; - uint32_t child_count = ts_node_child_count(node); - const char *kind = ts_node_type(node); - - if (!ts_node_is_named(node) && child_count == 0) { - /* Anonymous leaf (punctuation, keywords) — skip. */ - goto push_children; - } - - node_count++; - total_depth += depth; - - if ((uint16_t)depth > out->max_nesting_depth) { - out->max_nesting_depth = (uint16_t)depth; - } - - accumulate_control_flow(kind, out, &in_return); - accumulate_expressions(kind, out); - accumulate_halstead(kind, child_count, op_set, operand_set, out); - accumulate_data_flow(node, kind, child_count, source, param_names, param_count, in_return, - in_condition, out); - - /* Track context for data flow: are we inside a condition? */ - if (is_control_if(kind) || is_control_while(kind)) { - in_condition = true; - } - - push_children: - /* Reset context flags when leaving return/condition scope */ - if (is_return(kind)) { - in_return = false; - } - if (child_count > 0 && (is_control_if(kind) || is_control_while(kind))) { - in_condition = false; - } - - /* Push children in reverse order */ - for (int i = (int)child_count - SKIP_ONE; i >= 0 && top < WALK_STACK_CAP; i--) { - stack[top++] = (profile_frame_t){ts_node_child(node, (uint32_t)i), depth + SKIP_ONE}; - } - } - - /* Compute averages */ - if (node_count > 0) { - out->avg_nesting_depth_x10 = (uint16_t)((total_depth * DEPTH_SCALE) / node_count); - } - - return node_count > 0; -} - -/* ── Serialization ───────────────────────────────────────────────── */ - -void cbm_ast_profile_to_str(const cbm_ast_profile_t *p, char *buf, int bufsize) { - if (!p || !buf || bufsize < SKIP_ONE) { - if (buf && bufsize > 0) { - buf[0] = '\0'; - } - return; - } - snprintf(buf, (size_t)bufsize, - "%u,%u,%u,%u,%u,%u,%u,%u," - "%u,%u,%u,%u," - "%u,%u,%u," - "%u,%u,%u,%u," - "%u,%u,%u,%u," - "%u,%u", - p->if_count, p->for_count, p->while_count, p->switch_count, p->try_count, - p->return_count, p->max_nesting_depth, p->avg_nesting_depth_x10, p->comparison_ops, - p->arithmetic_ops, p->logical_ops, p->assignment_count, p->string_literals, - p->number_literals, p->bool_literals, p->param_count, p->params_in_returns, - p->params_in_conditions, p->variable_reassigns, p->unique_operators, - p->unique_operands, p->total_operators, p->total_operands, p->body_lines, - p->body_tokens); -} - -bool cbm_ast_profile_from_str(const char *str, cbm_ast_profile_t *out) { - if (!str || !out) { - return false; - } - memset(out, 0, sizeof(*out)); - /* Field layout must match cbm_ast_profile_to_str() exactly. */ - uint16_t *fields[PROFILE_FIELD_COUNT] = { - &out->if_count, &out->for_count, - &out->while_count, &out->switch_count, - &out->try_count, &out->return_count, - &out->max_nesting_depth, &out->avg_nesting_depth_x10, - &out->comparison_ops, &out->arithmetic_ops, - &out->logical_ops, &out->assignment_count, - &out->string_literals, &out->number_literals, - &out->bool_literals, &out->param_count, - &out->params_in_returns, &out->params_in_conditions, - &out->variable_reassigns, &out->unique_operators, - &out->unique_operands, &out->total_operators, - &out->total_operands, &out->body_lines, - &out->body_tokens, - }; - const char *p = str; - for (int i = 0; i < PROFILE_FIELD_COUNT; i++) { - char *end = NULL; - unsigned long parsed = strtoul(p, &end, BASE_DECIMAL_AST); - if (end == p) { - return false; - } - *fields[i] = (uint16_t)parsed; - p = end; - /* Skip the comma separator between fields. */ - if (i + SKIP_ONE < PROFILE_FIELD_COUNT) { - if (*p != ',') { - return false; - } - p++; - } - } - return true; -} - -void cbm_ast_profile_to_vector(const cbm_ast_profile_t *p, float *out) { - if (!p || !out) { - return; - } - /* Normalize each field to [0,1] range using reasonable maximums. */ - enum { MAX_COUNT = 100, MAX_DEPTH = 20, MAX_HALSTEAD = 200, MAX_TOKENS = 2000 }; - int i = 0; - out[i++] = (float)p->if_count / MAX_COUNT; - out[i++] = (float)p->for_count / MAX_COUNT; - out[i++] = (float)p->while_count / MAX_COUNT; - out[i++] = (float)p->switch_count / MAX_COUNT; - out[i++] = (float)p->try_count / MAX_COUNT; - out[i++] = (float)p->return_count / MAX_COUNT; - out[i++] = (float)p->max_nesting_depth / MAX_DEPTH; - out[i++] = (float)p->avg_nesting_depth_x10 / (MAX_DEPTH * DEPTH_SCALE); - out[i++] = (float)p->comparison_ops / MAX_COUNT; - out[i++] = (float)p->arithmetic_ops / MAX_COUNT; - out[i++] = (float)p->logical_ops / MAX_COUNT; - out[i++] = (float)p->assignment_count / MAX_COUNT; - out[i++] = (float)p->string_literals / MAX_COUNT; - out[i++] = (float)p->number_literals / MAX_COUNT; - out[i++] = (float)p->bool_literals / MAX_COUNT; - out[i++] = (float)p->param_count / MAX_DEPTH; - out[i++] = (float)p->params_in_returns / MAX_COUNT; - out[i++] = (float)p->params_in_conditions / MAX_COUNT; - out[i++] = (float)p->variable_reassigns / MAX_COUNT; - out[i++] = (float)p->unique_operators / MAX_HALSTEAD; - out[i++] = (float)p->unique_operands / MAX_HALSTEAD; - out[i++] = (float)p->total_operators / MAX_HALSTEAD; - out[i++] = (float)p->total_operands / MAX_HALSTEAD; - out[i++] = (float)p->body_lines / MAX_TOKENS; - out[i++] = (float)p->body_tokens / MAX_TOKENS; -} diff --git a/src/semantic/ast_profile.zig b/src/semantic/ast_profile.zig new file mode 100644 index 00000000..cf47e2f0 --- /dev/null +++ b/src/semantic/ast_profile.zig @@ -0,0 +1,393 @@ +//! ast_profile.zig — Extraction-time AST structural signals. +//! +//! Implements src/semantic/ast_profile.h. Walks a function body AST +//! (alongside MinHash) to compute: +//! Signal 8: Control flow counts, nesting depth, expression types +//! Signal 9: Approximate data flow (params in returns/conditions) +//! Signal 11: Halstead-lite (unique/total operators/operands) +//! +//! Pure functions — thread-safe, no shared state. + +const std = @import("std"); +const c = @import("c"); + +const Profile = c.cbm_ast_profile_t; +const Str = [*:0]const u8; + +// ── Node type classification ──────────────────────────────────────── + +const WALK_STACK_CAP = 2048; +const HALSTEAD_SET_SIZE = 512; +const HALSTEAD_SET_MASK = 511; +const PROFILE_FIELD_COUNT = 25; +const DEPTH_SCALE = 10; +const BASE_DECIMAL_AST = 10; +const HALSTEAD_HASH_MUL = 31; + +const ProfileFrame = struct { + node: c.TSNode, + depth: c_int, +}; + +fn isOneOf(kind: []const u8, comptime names: []const []const u8) bool { + inline for (names) |name| { + if (std.mem.eql(u8, kind, name)) return true; + } + return false; +} + +fn isControlIf(k: []const u8) bool { + return isOneOf(k, &.{ "if_statement", "if_expression", "elif_clause" }); +} + +fn isControlFor(k: []const u8) bool { + return isOneOf(k, &.{ "for_statement", "for_range_loop", "for_expression", "for_in_clause" }); +} + +fn isControlWhile(k: []const u8) bool { + return isOneOf(k, &.{ "while_statement", "while_expression", "do_statement" }); +} + +fn isControlSwitch(k: []const u8) bool { + return isOneOf(k, &.{ + "switch_statement", "switch_expression", "match_expression", "type_switch_statement", + }); +} + +fn isControlTry(k: []const u8) bool { + return isOneOf(k, &.{ "try_statement", "try_expression", "catch_clause", "except_clause" }); +} + +fn isReturn(k: []const u8) bool { + return isOneOf(k, &.{ "return_statement", "return_expression" }); +} + +fn isComparison(k: []const u8) bool { + return isOneOf(k, &.{ "binary_expression", "comparison_operator", "boolean_operator" }); +} + +fn isArithmetic(k: []const u8) bool { + return isOneOf(k, &.{ "unary_expression", "update_expression" }); +} + +fn isAssignment(k: []const u8) bool { + return isOneOf(k, &.{ + "assignment_expression", "assignment_statement", "augmented_assignment", + "short_var_declaration", + }); +} + +fn isStringLit(k: []const u8) bool { + return isOneOf(k, &.{ + "string", "string_literal", "interpreted_string_literal", "raw_string_literal", + }); +} + +fn isNumberLit(k: []const u8) bool { + return isOneOf(k, &.{ "number", "integer", "float", "integer_literal", "float_literal" }); +} + +fn isBoolLit(k: []const u8) bool { + return isOneOf(k, &.{ "true", "false" }); +} + +fn isOperatorNode(k: []const u8) bool { + // Named nodes that represent operations (not data). + return isControlIf(k) or isControlFor(k) or isControlWhile(k) or isControlSwitch(k) or + isControlTry(k) or isReturn(k) or isComparison(k) or isArithmetic(k) or + isAssignment(k) or + isOneOf(k, &.{ "call_expression", "member_expression", "subscript_expression" }); +} + +fn isIdentifier(k: []const u8) bool { + return isOneOf(k, &.{ + "identifier", "field_identifier", "property_identifier", "type_identifier", + }); +} + +/// Simple hash set for Halstead unique counting (open addressing). +fn halsteadInsert(set: *[HALSTEAD_SET_SIZE]u32, key: []const u8) bool { + var h: u32 = 0; + for (key) |ch| { + h = (h *% HALSTEAD_HASH_MUL) +% ch; + } + const idx = h & HALSTEAD_SET_MASK; + for (0..HALSTEAD_SET_SIZE) |probe| { + const slot = (idx +% @as(u32, @intCast(probe))) & HALSTEAD_SET_MASK; + if (set[slot] == 0) { + set[slot] = h | 1; + return true; // new + } + if (set[slot] == (h | 1)) { + return false; // existing + } + } + return false; +} + +/// Check if an identifier matches any parameter name. +fn isParamName(ident: Str, param_names: ?[*]?Str, param_count: c_int) bool { + const names = param_names orelse return false; + if (param_count <= 0) return false; + for (names[0..@intCast(param_count)]) |name| { + if (name != null and c.strcmp(ident, name) == 0) return true; + } + return false; +} + +// ── Main computation ──────────────────────────────────────────────── + +/// Count control-flow statement kinds (if/for/while/switch/try/return). +fn accumulateControlFlow(kind: []const u8, out: *Profile, in_return: *bool) void { + if (isControlIf(kind)) out.if_count +%= 1; + if (isControlFor(kind)) out.for_count +%= 1; + if (isControlWhile(kind)) out.while_count +%= 1; + if (isControlSwitch(kind)) out.switch_count +%= 1; + if (isControlTry(kind)) out.try_count +%= 1; + if (isReturn(kind)) { + out.return_count +%= 1; + in_return.* = true; + } +} + +/// Count expression- and literal-class counters for one node. +fn accumulateExpressions(kind: []const u8, out: *Profile) void { + if (isComparison(kind)) out.comparison_ops +%= 1; + if (isArithmetic(kind)) out.arithmetic_ops +%= 1; + if (isOneOf(kind, &.{ "not_operator", "boolean_operator" })) out.logical_ops +%= 1; + if (isAssignment(kind)) { + out.assignment_count +%= 1; + out.variable_reassigns +%= 1; + } + if (isStringLit(kind)) out.string_literals +%= 1; + if (isNumberLit(kind)) out.number_literals +%= 1; + if (isBoolLit(kind)) out.bool_literals +%= 1; +} + +/// Accumulate Halstead operator/operand counts. Leaf identifiers, literals, +/// and booleans are "operands"; recognized statement/expression kinds are +/// "operators". Uses an open-addressed hash set for unique counting. +fn accumulateHalstead( + kind: []const u8, + child_count: u32, + op_set: *[HALSTEAD_SET_SIZE]u32, + operand_set: *[HALSTEAD_SET_SIZE]u32, + out: *Profile, +) void { + if (isOperatorNode(kind)) { + out.total_operators +%= 1; + if (halsteadInsert(op_set, kind)) out.unique_operators +%= 1; + } + if (child_count == 0 and + (isIdentifier(kind) or isStringLit(kind) or isNumberLit(kind) or isBoolLit(kind))) + { + out.total_operands +%= 1; + if (halsteadInsert(operand_set, kind)) out.unique_operands +%= 1; + out.body_tokens +%= 1; + } +} + +/// Data-flow counter update: bump params_in_returns / params_in_conditions +/// when the current leaf identifier names a parameter and we're inside the +/// corresponding syntactic scope. +fn accumulateDataFlow( + node: c.TSNode, + kind: []const u8, + child_count: u32, + source_c: ?[*]const u8, + param_names: ?[*]?Str, + param_count: c_int, + in_return: bool, + in_condition: bool, + out: *Profile, +) void { + const source = source_c orelse return; + if (!(child_count == 0 and isIdentifier(kind))) return; + const start = c.ts_node_start_byte(node); + const end = c.ts_node_end_byte(node); + if (end <= start or (end - start) >= c.CBM_SZ_128) return; + var ident_buf: [c.CBM_SZ_128]u8 = undefined; + const ilen: usize = end - start; + @memcpy(ident_buf[0..ilen], source[start..end]); + ident_buf[ilen] = 0; + if (!isParamName(@ptrCast(&ident_buf), param_names, param_count)) return; + if (in_return) out.params_in_returns +%= 1; + if (in_condition) out.params_in_conditions +%= 1; +} + +pub export fn cbm_ast_profile_compute( + func_body: c.TSNode, + source: ?[*]const u8, + param_names: ?[*]?Str, + param_count: c_int, + out: *Profile, +) bool { + if (c.ts_node_is_null(func_body)) return false; + + out.* = std.mem.zeroes(Profile); + out.param_count = @truncate(@as(c_uint, @bitCast(param_count))); + + var op_set: [HALSTEAD_SET_SIZE]u32 = @splat(0); + var operand_set: [HALSTEAD_SET_SIZE]u32 = @splat(0); + + var total_depth: c_int = 0; + var node_count: c_int = 0; + var in_return = false; + var in_condition = false; + + var stack: [WALK_STACK_CAP]ProfileFrame = undefined; + var top: usize = 0; + stack[top] = .{ .node = func_body, .depth = 0 }; + top += 1; + + while (top > 0) { + top -= 1; + const frame = stack[top]; + const node = frame.node; + const depth = frame.depth; + const child_count = c.ts_node_child_count(node); + const kind = std.mem.span(@as(Str, @ptrCast(c.ts_node_type(node)))); + + // Anonymous leaves (punctuation, keywords) are skipped. + if (c.ts_node_is_named(node) or child_count != 0) { + node_count += 1; + total_depth +%= depth; + + const depth16: u16 = @truncate(@as(c_uint, @bitCast(depth))); + if (depth16 > out.max_nesting_depth) out.max_nesting_depth = depth16; + + accumulateControlFlow(kind, out, &in_return); + accumulateExpressions(kind, out); + accumulateHalstead(kind, child_count, &op_set, &operand_set, out); + accumulateDataFlow(node, kind, child_count, source, param_names, param_count, in_return, in_condition, out); + + // Track context for data flow: are we inside a condition? + if (isControlIf(kind) or isControlWhile(kind)) in_condition = true; + } + + // Reset context flags when leaving return/condition scope. + if (isReturn(kind)) in_return = false; + if (child_count > 0 and (isControlIf(kind) or isControlWhile(kind))) in_condition = false; + + // Push children in reverse order. + var i = child_count; + while (i > 0 and top < WALK_STACK_CAP) { + i -= 1; + stack[top] = .{ .node = c.ts_node_child(node, i), .depth = depth + 1 }; + top += 1; + } + } + + // Compute averages. + if (node_count > 0) { + const avg = @divTrunc(total_depth *% DEPTH_SCALE, node_count); + out.avg_nesting_depth_x10 = @truncate(@as(c_uint, @bitCast(avg))); + } + + return node_count > 0; +} + +// ── Serialization ─────────────────────────────────────────────────── + +/// Serialization order of the profile fields. Must match the argument order +/// in cbm_ast_profile_to_str(). +const field_order = [PROFILE_FIELD_COUNT][]const u8{ + "if_count", "for_count", "while_count", "switch_count", + "try_count", "return_count", "max_nesting_depth", "avg_nesting_depth_x10", + "comparison_ops", "arithmetic_ops", "logical_ops", "assignment_count", + "string_literals", "number_literals", "bool_literals", "param_count", + "params_in_returns", "params_in_conditions", "variable_reassigns", "unique_operators", + "unique_operands", "total_operators", "total_operands", "body_lines", + "body_tokens", +}; + +pub export fn cbm_ast_profile_to_str(p_c: ?*const Profile, buf_c: ?[*]u8, bufsize: c_int) void { + const buf = buf_c orelse return; + const p = p_c orelse { + if (bufsize > 0) buf[0] = 0; + return; + }; + if (bufsize < 1) return; + const u = struct { + fn u(v: u16) c_uint { + return v; + } + }.u; + _ = c.snprintf( + buf, + @intCast(bufsize), + "%u,%u,%u,%u,%u,%u,%u,%u," ++ + "%u,%u,%u,%u," ++ + "%u,%u,%u," ++ + "%u,%u,%u,%u," ++ + "%u,%u,%u,%u," ++ + "%u,%u", + u(p.if_count), + u(p.for_count), + u(p.while_count), + u(p.switch_count), + u(p.try_count), + u(p.return_count), + u(p.max_nesting_depth), + u(p.avg_nesting_depth_x10), + u(p.comparison_ops), + u(p.arithmetic_ops), + u(p.logical_ops), + u(p.assignment_count), + u(p.string_literals), + u(p.number_literals), + u(p.bool_literals), + u(p.param_count), + u(p.params_in_returns), + u(p.params_in_conditions), + u(p.variable_reassigns), + u(p.unique_operators), + u(p.unique_operands), + u(p.total_operators), + u(p.total_operands), + u(p.body_lines), + u(p.body_tokens), + ); +} + +pub export fn cbm_ast_profile_from_str(str: ?Str, out_c: ?*Profile) bool { + const out = out_c orelse return false; + var p: Str = str orelse return false; + out.* = std.mem.zeroes(Profile); + // Field layout must match cbm_ast_profile_to_str() exactly. + inline for (field_order, 0..) |name, i| { + var end: [*c]u8 = null; + const parsed = c.strtoul(p, &end, BASE_DECIMAL_AST); + if (@intFromPtr(end) == @intFromPtr(p)) return false; + @field(out, name) = @truncate(parsed); + p = @ptrCast(end); + // Skip the comma separator between fields. + if (i + 1 < PROFILE_FIELD_COUNT) { + if (p[0] != ',') return false; + p += 1; + } + } + return true; +} + +pub export fn cbm_ast_profile_to_vector(p_c: ?*const Profile, out_c: ?[*]f32) void { + const p = p_c orelse return; + const out = out_c orelse return; + // Normalize each field to [0,1] range using reasonable maximums. + const MAX_COUNT = 100; + const MAX_DEPTH = 20; + const MAX_HALSTEAD = 200; + const MAX_TOKENS = 2000; + const maxima = [PROFILE_FIELD_COUNT]f32{ + MAX_COUNT, MAX_COUNT, MAX_COUNT, MAX_COUNT, + MAX_COUNT, MAX_COUNT, MAX_DEPTH, MAX_DEPTH * DEPTH_SCALE, + MAX_COUNT, MAX_COUNT, MAX_COUNT, MAX_COUNT, + MAX_COUNT, MAX_COUNT, MAX_COUNT, MAX_DEPTH, + MAX_COUNT, MAX_COUNT, MAX_COUNT, MAX_HALSTEAD, + MAX_HALSTEAD, MAX_HALSTEAD, MAX_HALSTEAD, MAX_TOKENS, + MAX_TOKENS, + }; + inline for (field_order, maxima, 0..) |name, max, i| { + out[i] = @as(f32, @floatFromInt(@field(p, name))) / max; + } +} diff --git a/src/semantic/fp.zig b/src/semantic/fp.zig new file mode 100644 index 00000000..eeeb19f9 --- /dev/null +++ b/src/semantic/fp.zig @@ -0,0 +1,28 @@ +//! fp.zig — floating-point contraction that matches the C build. +//! +//! The C sources were compiled by clang with its default -ffp-contract=on: +//! where the target CPU has a fused multiply-add instruction, `a * b + c` +//! written in one expression becomes a single FMA with one rounding. Zig +//! never fuses on its own, so a port that writes `a * b + c` rounds twice and +//! produces scores that differ in the last bit. Those differences move +//! similarity scores across edge thresholds. +//! +//! mulAdd fuses exactly when clang would: on targets with FMA. Use it for +//! every expression the C code wrote as a product plus (or minus) a term in +//! one expression, including `x += a * b`. + +const std = @import("std"); +const builtin = @import("builtin"); + +/// True when the target has a fused multiply-add instruction, which is when +/// clang lowers llvm.fmuladd to FMA. +pub const has_fma = switch (builtin.cpu.arch) { + .x86_64, .x86 => std.Target.x86.featureSetHas(builtin.cpu.features, .fma), + .aarch64, .aarch64_be => true, + else => false, +}; + +/// `a * b + c` the way clang compiles that C expression. +pub inline fn mulAdd(comptime T: type, a: T, b: T, c: T) T { + return if (has_fma) @mulAdd(T, a, b, c) else a * b + c; +} diff --git a/src/semantic/rotsq.c b/src/semantic/rotsq.c deleted file mode 100644 index c067b5d9..00000000 --- a/src/semantic/rotsq.c +++ /dev/null @@ -1,129 +0,0 @@ -/* - * rotsq.c — see rotsq.h. From-paper implementation, no dependencies. - */ -#include "semantic/rotsq.h" - -#include - -#include "foundation/compat_thread.h" - -#define XXH_INLINE_ALL -#include "xxhash/xxhash.h" - -/* ── Deterministic ±1 diagonal (seeded, generated once) ─────────────── */ - -static float g_rsq_diag[CBM_RSQ_DIM]; -static cbm_once_t g_rsq_diag_once = CBM_ONCE_INIT; - -static void rsq_build_diag(void) { - for (int d = 0; d < CBM_RSQ_DIM; d++) { - uint64_t h = XXH3_64bits_withSeed(&d, sizeof(d), 0x5bd1e995u); - g_rsq_diag[d] = (h & 1u) ? 1.0F : -1.0F; - } -} - -static void rsq_init_diag(void) { - cbm_once(&g_rsq_diag_once, rsq_build_diag); -} - -/* ── Fast Walsh–Hadamard Transform (in place, unnormalized) ─────────── */ - -static void rsq_fwht(float *v) { - for (int len = 1; len < CBM_RSQ_DIM; len <<= 1) { - for (int i = 0; i < CBM_RSQ_DIM; i += len << 1) { - for (int j = i; j < i + len; j++) { - float a = v[j]; - float b = v[j + len]; - v[j] = a + b; - v[j + len] = a - b; - } - } - } -} - -/* ── Encode ─────────────────────────────────────────────────────────── */ - -void cbm_rsq_encode(const float *v, cbm_rsq_code_t *out) { - rsq_init_diag(); - - float rot[CBM_RSQ_DIM]; - for (int d = 0; d < CBM_RSQ_IN_DIM; d++) { - rot[d] = v[d] * g_rsq_diag[d]; - } - for (int d = CBM_RSQ_IN_DIM; d < CBM_RSQ_DIM; d++) { - rot[d] = 0.0F; - } - rsq_fwht(rot); - /* Normalize the transform so the rotation is orthonormal (H/√D): keeps - * the coordinates in a data-independent range and makes the estimated IP - * directly comparable to the pre-rotation IP. */ - const float inv_sqrt_d = 1.0F / 32.0F; /* 1/√1024 */ - float lo = rot[0] * inv_sqrt_d; - float hi = lo; - for (int d = 0; d < CBM_RSQ_DIM; d++) { - rot[d] *= inv_sqrt_d; - if (rot[d] < lo) { - lo = rot[d]; - } - if (rot[d] > hi) { - hi = rot[d]; - } - } - - /* Per-vector scalar quantization over [lo, hi] at CBM_RSQ_BITS. */ - float range = hi - lo; - float step = range > 0.0F ? range / (float)CBM_RSQ_LEVELS : 1.0F; - out->offset = lo; - out->scale = step; - - int32_t sum = 0; - memset(out->codes, 0, sizeof(out->codes)); - for (int d = 0; d < CBM_RSQ_DIM; d++) { - float q = (rot[d] - lo) / step; - int32_t c = (int32_t)(q + 0.5F); - if (c < 0) { - c = 0; - } - if (c > CBM_RSQ_LEVELS) { - c = CBM_RSQ_LEVELS; - } - sum += c; - if (d & 1) { - out->codes[d >> 1] |= (uint8_t)(c << 4); - } else { - out->codes[d >> 1] |= (uint8_t)c; - } - } - out->code_sum = sum; -} - -/* ── Estimated inner product from two codes ─────────────────────────── */ - -float cbm_rsq_ip(const cbm_rsq_code_t *a, const cbm_rsq_code_t *b) { - /* Integer dot of the 4-bit codes. */ - int64_t dot = 0; - for (int i = 0; i < CBM_RSQ_CODE_BYTES; i++) { - uint8_t ba = a->codes[i]; - uint8_t bb = b->codes[i]; - dot += (int64_t)(ba & 0x0F) * (bb & 0x0F); - dot += (int64_t)(ba >> 4) * (bb >> 4); - } - /* x_i ≈ oa + sa·ca_i, y_i ≈ ob + sb·cb_i (in the rotated space, where the - * IP equals the original IP because the rotation is orthonormal). */ - double d = (double)CBM_RSQ_DIM; - double ip = d * (double)a->offset * (double)b->offset + - (double)a->offset * (double)b->scale * (double)b->code_sum + - (double)b->offset * (double)a->scale * (double)a->code_sum + - (double)a->scale * (double)b->scale * (double)dot; - return (float)ip; -} - -/* ── Dequantize into the rotated basis ──────────────────────────────── */ - -void cbm_rsq_decode(const cbm_rsq_code_t *c, float *out) { - for (int i = 0; i < CBM_RSQ_CODE_BYTES; i++) { - uint8_t b = c->codes[i]; - out[i * 2] = c->offset + c->scale * (float)(b & 0x0F); - out[i * 2 + 1] = c->offset + c->scale * (float)(b >> 4); - } -} diff --git a/src/semantic/rotsq.zig b/src/semantic/rotsq.zig new file mode 100644 index 00000000..ebdefda0 --- /dev/null +++ b/src/semantic/rotsq.zig @@ -0,0 +1,121 @@ +//! rotsq.zig — see rotsq.h. From-paper implementation, no dependencies. + +const std = @import("std"); +const c = @import("c"); +const fp = @import("fp.zig"); + +const XxHash3 = std.hash.XxHash3; + +const IN_DIM: comptime_int = c.CBM_RSQ_IN_DIM; +const DIM: comptime_int = c.CBM_RSQ_DIM; +const LEVELS: comptime_int = c.CBM_RSQ_LEVELS; + +// ── Deterministic ±1 diagonal ────────────────────────────────────── + +/// The C code generated this table once at first use. It is a pure function +/// of the index, so it is computed at compile time here. +const rsq_diag: [DIM]f32 = blk: { + @setEvalBranchQuota(4_000_000); + var diag: [DIM]f32 = undefined; + for (&diag, 0..) |*v, d| { + const h = XxHash3.hash(0x5bd1e995, &std.mem.toBytes(@as(c_int, @intCast(d)))); + v.* = if (h & 1 != 0) 1.0 else -1.0; + } + break :blk diag; +}; + +// ── Fast Walsh–Hadamard Transform (in place, unnormalized) ───────── + +fn rsqFwht(v: *[DIM]f32) void { + var len: usize = 1; + while (len < DIM) : (len <<= 1) { + var i: usize = 0; + while (i < DIM) : (i += len << 1) { + for (i..i + len) |j| { + const a = v[j]; + const b = v[j + len]; + v[j] = a + b; + v[j + len] = a - b; + } + } + } +} + +// ── Encode ───────────────────────────────────────────────────────── + +pub export fn cbm_rsq_encode(v: [*]const f32, out: *c.cbm_rsq_code_t) void { + var rot: [DIM]f32 = undefined; + for (rot[0..IN_DIM], v[0..IN_DIM], rsq_diag[0..IN_DIM]) |*r, x, sign| { + r.* = x * sign; + } + @memset(rot[IN_DIM..], 0.0); + rsqFwht(&rot); + // Normalize the transform so the rotation is orthonormal (H/√D): keeps + // the coordinates in a data-independent range and makes the estimated IP + // directly comparable to the pre-rotation IP. + const inv_sqrt_d: f32 = 1.0 / 32.0; // 1/√1024 + var lo: f32 = rot[0] * inv_sqrt_d; + var hi: f32 = lo; + for (&rot) |*r| { + r.* *= inv_sqrt_d; + if (r.* < lo) lo = r.*; + if (r.* > hi) hi = r.*; + } + + // Per-vector scalar quantization over [lo, hi] at CBM_RSQ_BITS. + const range = hi - lo; + const step: f32 = if (range > 0.0) range / @as(f32, LEVELS) else 1.0; + out.offset = lo; + out.scale = step; + + var sum: i32 = 0; + @memset(&out.codes, 0); + for (rot, 0..) |r, d| { + const q = (r - lo) / step; + var code: i32 = @intFromFloat(q + 0.5); + if (code < 0) code = 0; + if (code > LEVELS) code = LEVELS; + sum += code; + const nibble: u8 = @intCast(code); + if (d & 1 != 0) { + out.codes[d >> 1] |= nibble << 4; + } else { + out.codes[d >> 1] |= nibble; + } + } + out.code_sum = sum; +} + +// ── Estimated inner product from two codes ───────────────────────── + +pub export fn cbm_rsq_ip(a: *const c.cbm_rsq_code_t, b: *const c.cbm_rsq_code_t) f32 { + // Integer dot of the 4-bit codes. + var dot: i64 = 0; + for (a.codes, b.codes) |ba, bb| { + dot += @as(i64, ba & 0x0F) * (bb & 0x0F); + dot += @as(i64, ba >> 4) * (bb >> 4); + } + // x_i ≈ oa + sa·ca_i, y_i ≈ ob + sb·cb_i (in the rotated space, where the + // IP equals the original IP because the rotation is orthonormal). + const d: f64 = DIM; + const oa: f64 = a.offset; + const ob: f64 = b.offset; + const sa: f64 = a.scale; + const sb: f64 = b.scale; + // C: d*oa*ob + oa*sb*Σcb + ob*sa*Σca + sa*sb*dot, summed left to right. + // The first sum fuses with its left product, the later ones with their + // right product. + var ip = fp.mulAdd(f64, d * oa, ob, oa * sb * @as(f64, @floatFromInt(b.code_sum))); + ip = fp.mulAdd(f64, ob * sa, @as(f64, @floatFromInt(a.code_sum)), ip); + ip = fp.mulAdd(f64, sa * sb, @as(f64, @floatFromInt(dot)), ip); + return @floatCast(ip); +} + +// ── Dequantize into the rotated basis ────────────────────────────── + +pub export fn cbm_rsq_decode(code: *const c.cbm_rsq_code_t, out: [*]f32) void { + for (code.codes, 0..) |b, i| { + out[i * 2] = fp.mulAdd(f32, code.scale, @floatFromInt(b & 0x0F), code.offset); + out[i * 2 + 1] = fp.mulAdd(f32, code.scale, @floatFromInt(b >> 4), code.offset); + } +} diff --git a/src/semantic/semantic.c b/src/semantic/semantic.c deleted file mode 100644 index 8023e70c..00000000 --- a/src/semantic/semantic.c +++ /dev/null @@ -1,1683 +0,0 @@ -/* - * semantic.c — Algorithmic code embeddings: TF-IDF, Random Indexing, - * API/Type/Decorator signatures, combined scoring, graph diffusion. - * - * All signals computed from graph buffer metadata — no source file reads. - * Uses xxHash for deterministic random vectors. Pure C, zero dependencies. - */ -#include "semantic/semantic.h" -#include "foundation/constants.h" -#include "foundation/hash_table.h" -#include "foundation/log.h" -#include "foundation/profile.h" -#include "foundation/platform.h" -#include "foundation/compat_thread.h" -#include "pipeline/worker_pool.h" -#include "simhash/minhash.h" -#include "nomic/code_vectors.h" - -#define XXH_INLINE_ALL -#include "xxhash/xxhash.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -/* ── Constants ───────────────────────────────────────────────────── */ - -enum { - TOKEN_BUF_LEN = 128, - CORPUS_INIT_CAP = 4096, - DOC_TOKENS_INIT = 64, - RI_SEED_BASE = 0x52494E44, /* "RIND" */ -}; - -/* Default signal weights for cbm_sem_combined_score. Must sum to ~1.0; - * proximity is a multiplier applied on top. */ -#define CBM_SEM_W_TFIDF 0.20F -#define CBM_SEM_W_RI 0.25F -#define CBM_SEM_W_MINHASH 0.10F -#define CBM_SEM_W_API 0.15F -#define CBM_SEM_W_TYPE 0.10F -#define CBM_SEM_W_DECORATOR 0.05F -#define CBM_SEM_W_STRUCT_PROFILE 0.10F -#define CBM_SEM_W_DATAFLOW 0.05F - -/* Threshold bounds for CBM_SEMANTIC_THRESHOLD env override. */ -#define CBM_SEM_THRESHOLD_MIN 0.0F -#define CBM_SEM_THRESHOLD_MAX 1.0F - -/* Epsilon for denominator guards in cosine math. */ -#define CBM_SEM_DENOM_EPS 1e-10F - -/* Int8 quantization bounds for vector storage (maps [-1.0, 1.0] to [-127, 127]). */ -#define CBM_SEM_INT8_MAX 127.0F -#define CBM_SEM_INT8_MIN (-127.0F) - -/* Unit vector bounds (normalized similarity / cosine value limits). */ -#define CBM_SEM_UNIT_POS 1.0F -#define CBM_SEM_UNIT_NEG (-1.0F) - -/* Proximity boost: same-file gets 1.0 + CBM_SEM_PROX_MAX_BOOST, distant gets 1.0. */ -#define CBM_SEM_PROX_MAX_BOOST 0.10F - -/* Reflective Random Indexing blend factors for pass2 (context mixing). */ -#define CBM_SEM_RRI_ALPHA 0.3F -#define CBM_SEM_RRI_BETA 0.7F - -/* IDF smoothing constant (log base): log2(1 + docs / doc_freq). */ -#define CBM_SEM_IDF_BASE 0.5F - -/* Strings of exactly two parts / dimensions used for co-occurrence window math. */ -#define CBM_SEM_COOCCUR_STRIDE 2 - -/* Worker tile/chunk sizes for parallel cooccurrence passes. */ -enum { - CBM_SEM_WORKER_STACK_CAP = 256, - CBM_SEM_TILE_SIZE = 40, - CBM_SEM_SEEN_INIT_CAP = 256, - CBM_SEM_RESOLVE_CHUNK = 64, - CBM_SEM_COOCCUR_CHUNK = 32, - CBM_SEM_RRI_TILE = 128, - CBM_SEM_INT8_I_LO = -128, - CBM_SEM_INT8_I_HI = 127, - CBM_SEM_ATOMIC_INC = 1, - CBM_SEM_RI_NONZERO_COUNT = 8, -}; - -/* Mutex-init state machine for the lazy-initialized pretrained token map. - * State 0 = not started, 1 = initializing, 2 = initialized. */ -enum { - MTX_STATE_UNINIT = 0, - MTX_STATE_INITIALIZING = 1, - MTX_STATE_READY = 2, -}; - -/* Pretrained map ready flag (signals a one-shot atomic transition 0 → 1). */ -enum { MAP_READY = 1 }; - -/* Numeric conversion radix for strtol (base 10 decimal). */ -enum { BASE_DECIMAL = 10 }; - -/* ── Configuration ───────────────────────────────────────────────── */ - -cbm_sem_config_t cbm_sem_get_config(void) { - cbm_sem_config_t cfg = { - .w_tfidf = CBM_SEM_W_TFIDF, - .w_ri = CBM_SEM_W_RI, - .w_minhash = CBM_SEM_W_MINHASH, - .w_api = CBM_SEM_W_API, - .w_type = CBM_SEM_W_TYPE, - .w_decorator = CBM_SEM_W_DECORATOR, - .w_struct_profile = CBM_SEM_W_STRUCT_PROFILE, - .w_dataflow = CBM_SEM_W_DATAFLOW, - .threshold = (float)CBM_SEM_EDGE_THRESHOLD, - .max_edges = CBM_SEM_MAX_EDGES, - }; - const char *thresh = getenv("CBM_SEMANTIC_THRESHOLD"); - if (thresh) { - /* strtod reports errors via endptr; reject non-numeric input silently. */ - char *end = NULL; - double parsed = strtod(thresh, &end); - if (end != thresh && parsed > CBM_SEM_THRESHOLD_MIN && parsed <= CBM_SEM_THRESHOLD_MAX) { - cfg.threshold = (float)parsed; - } - } - return cfg; -} - -bool cbm_sem_is_enabled(void) { - const char *val = getenv("CBM_SEMANTIC_ENABLED"); - return val && val[0] == '1'; -} - -/* ── Token extraction ────────────────────────────────────────────── */ - -/* True for characters that terminate a token regardless of case. */ -static bool is_token_delim(char c) { - return c == '.' || c == '/' || c == '_' || c == '-' || c == ' ' || c == '(' || c == ')' || - c == ',' || c == ':'; -} - -/* True for a camelCase transition: uppercase letter preceded by a lowercase. */ -static bool is_camel_break(const char *name, int i) { - if (i <= 0) { - return false; - } - char c = name[i]; - char p = name[i - SKIP_ONE]; - return c >= 'A' && c <= 'Z' && p >= 'a' && p <= 'z'; -} - -/* Flush the current buffer as a token into out[]. */ -static void flush_token(char *buf, int *blen, char **out, int *count, int max_out) { - if (*blen > 0 && *count < max_out) { - buf[*blen] = '\0'; - out[(*count)++] = strdup(buf); - } - *blen = 0; -} - -int cbm_sem_tokenize(const char *name, char **out, int max_out) { - if (!name || !out || max_out <= 0) { - return 0; - } - int count = 0; - char buf[TOKEN_BUF_LEN]; - int blen = 0; - - for (int i = 0; name[i] && count < max_out; i++) { - char c = name[i]; - bool split = is_token_delim(c); - bool camel = is_camel_break(name, i); - if (split || camel) { - flush_token(buf, &blen, out, &count, max_out); - if (split) { - continue; - } - } - if (blen < TOKEN_BUF_LEN - SKIP_ONE && isalnum((unsigned char)c)) { - buf[blen++] = (char)tolower((unsigned char)c); - } - } - flush_token(buf, &blen, out, &count, max_out); - - /* Abbreviation expansion: add expanded forms for common code abbreviations. - * "err" → also add "error", "ctx" → "context", etc. */ - /* Cross-language abbreviation table — covers Go, Python, JS/TS, Rust, - * Java, C/C++, Ruby, PHP, Kotlin, Swift, Scala, C#, and common patterns. */ - static const struct { - const char *abbrev; - const char *expanded; - } abbrevs[] = { - /* Error/exception handling */ - {"err", "error"}, - {"exc", "exception"}, - {"ex", "exception"}, - /* Context/config */ - {"ctx", "context"}, - {"cfg", "config"}, - {"conf", "configuration"}, - {"env", "environment"}, - {"opt", "option"}, - {"opts", "options"}, - /* Request/response (HTTP, RPC) */ - {"req", "request"}, - {"res", "response"}, - {"resp", "response"}, - {"rsp", "response"}, - {"hdr", "header"}, - {"hdrs", "headers"}, - /* Strings/formatting */ - {"str", "string"}, - {"fmt", "format"}, - {"msg", "message"}, - {"txt", "text"}, - {"lbl", "label"}, - {"desc", "description"}, - /* Data structures */ - {"buf", "buffer"}, - {"arr", "array"}, - {"vec", "vector"}, - {"lst", "list"}, - {"dict", "dictionary"}, - {"tbl", "table"}, - {"stk", "stack"}, - {"que", "queue"}, - /* Functions/callbacks */ - {"fn", "function"}, - {"func", "function"}, - {"cb", "callback"}, - {"proc", "procedure"}, - {"ctor", "constructor"}, - {"dtor", "destructor"}, - /* Database/storage */ - {"db", "database"}, - {"col", "column"}, - {"tbl", "table"}, - {"stmt", "statement"}, - {"txn", "transaction"}, - {"trx", "transaction"}, - {"repo", "repository"}, - /* Auth/security */ - {"auth", "authentication"}, - {"authz", "authorization"}, - {"perm", "permission"}, - {"cred", "credential"}, - {"tok", "token"}, - {"pwd", "password"}, - /* Values/types */ - {"val", "value"}, - {"num", "number"}, - {"int", "integer"}, - {"bool", "boolean"}, - {"flt", "float"}, - {"dbl", "double"}, - /* Indexing/iteration */ - {"idx", "index"}, - {"iter", "iterator"}, - {"elem", "element"}, - {"cnt", "count"}, - {"len", "length"}, - {"sz", "size"}, - {"pos", "position"}, - {"off", "offset"}, - {"cap", "capacity"}, - /* Lifecycle */ - {"init", "initialize"}, - {"deinit", "deinitialize"}, - {"alloc", "allocate"}, - {"dealloc", "deallocate"}, - {"del", "delete"}, - {"rm", "remove"}, - /* Implementation/interface */ - {"impl", "implementation"}, - {"iface", "interface"}, - {"abs", "abstract"}, - {"decl", "declaration"}, - /* Parameters/attributes */ - {"param", "parameter"}, - {"arg", "argument"}, - {"attr", "attribute"}, - {"prop", "property"}, - {"ret", "return"}, - /* Source/destination */ - {"src", "source"}, - {"dst", "destination"}, - {"tgt", "target"}, - {"orig", "original"}, - {"prev", "previous"}, - {"cur", "current"}, - {"tmp", "temporary"}, - {"temp", "temporary"}, - /* Networking/IO */ - {"conn", "connection"}, - {"sess", "session"}, - {"sock", "socket"}, - {"addr", "address"}, - {"url", "uniform"}, - {"srv", "server"}, - {"cli", "client"}, - {"svc", "service"}, - {"ep", "endpoint"}, - /* Management */ - {"mgr", "manager"}, - {"ctrl", "controller"}, - {"hdlr", "handler"}, - {"sched", "scheduler"}, - {"disp", "dispatcher"}, - {"reg", "registry"}, - /* Async/concurrent */ - {"chan", "channel"}, - {"sem", "semaphore"}, - {"mtx", "mutex"}, - {"wg", "waitgroup"}, - {"sig", "signal"}, - {"evt", "event"}, - {"sub", "subscriber"}, - {"pub", "publisher"}, - /* Testing */ - {"spec", "specification"}, - {"mock", "mock"}, - {"stub", "stub"}, - {"assert", "assertion"}, - /* Logging/monitoring */ - {"log", "logging"}, - {"lvl", "level"}, - {"dbg", "debug"}, - {"wrn", "warning"}, - {"inf", "info"}, - /* Time */ - {"ts", "timestamp"}, - {"dur", "duration"}, - {"ttl", "timetolive"}, - /* Miscellaneous */ - {"ver", "version"}, - {"ns", "namespace"}, - {"pkg", "package"}, - {"mod", "module"}, - {"lib", "library"}, - {"dep", "dependency"}, - {"ref", "reference"}, - {"ptr", "pointer"}, - {"obj", "object"}, - {"doc", "document"}, - {"cmd", "command"}, - {"ops", "operations"}, - {"util", "utility"}, - {"hlp", "helper"}, - {"ext", "extension"}, - {NULL, NULL}, - }; - int orig_count = count; - for (int t = 0; t < orig_count && count < max_out; t++) { - for (int a = 0; abbrevs[a].abbrev; a++) { - if (strcmp(out[t], abbrevs[a].abbrev) == 0) { - out[count++] = strdup(abbrevs[a].expanded); - break; - } - } - } - - return count; -} - -/* ── Dense vector operations ─────────────────────────────────────── */ - -float cbm_sem_cosine(const cbm_sem_vec_t *a, const cbm_sem_vec_t *b) { - if (!a || !b) { - return 0.0F; - } - float dot = 0.0F; - float mag_a = 0.0F; - float mag_b = 0.0F; - for (int i = 0; i < CBM_SEM_DIM; i++) { - dot += a->v[i] * b->v[i]; - mag_a += a->v[i] * a->v[i]; - mag_b += b->v[i] * b->v[i]; - } - float denom = sqrtf(mag_a) * sqrtf(mag_b); - if (denom < CBM_SEM_DENOM_EPS) { - return 0.0F; - } - return dot / denom; -} - -/* Pretrained token lookup table — built lazily on first use. */ -static CBMHashTable *g_pretrained_map = NULL; -static _Atomic int g_pretrained_ready = 0; -static cbm_mutex_t g_pretrained_mtx; -static _Atomic int g_pretrained_mtx_init = 0; - -/* Thread-safe lazy init of the pretrained token lookup map. - * Uses double-checked locking: fast path reads an atomic flag. */ -static void ensure_pretrained_map(void) { - if (atomic_load_explicit(&g_pretrained_ready, memory_order_acquire)) { - return; - } - /* First-time init of the mutex itself (also needs to be thread-safe) */ - int expected = MTX_STATE_UNINIT; - if (atomic_compare_exchange_strong_explicit(&g_pretrained_mtx_init, &expected, - MTX_STATE_INITIALIZING, memory_order_acq_rel, - memory_order_acquire)) { - cbm_mutex_init(&g_pretrained_mtx); - atomic_store_explicit(&g_pretrained_mtx_init, MTX_STATE_READY, memory_order_release); - } else { - /* Spin until another thread finishes initializing the mutex */ - while (atomic_load_explicit(&g_pretrained_mtx_init, memory_order_acquire) != - MTX_STATE_READY) { - /* brief spin */ - } - } - cbm_mutex_lock(&g_pretrained_mtx); - if (!atomic_load_explicit(&g_pretrained_ready, memory_order_acquire)) { - g_pretrained_map = cbm_ht_create(PRETRAINED_TOKEN_COUNT); - char idx_buf[CBM_SZ_16]; - for (int i = 0; i < PRETRAINED_TOKEN_COUNT; i++) { - const char *tok = PRETRAINED_TOKENS[i]; - if (tok && tok[0]) { - snprintf(idx_buf, sizeof(idx_buf), "%d", i); - cbm_ht_set(g_pretrained_map, strdup(tok), strdup(idx_buf)); - } - } - atomic_store_explicit(&g_pretrained_ready, MAP_READY, memory_order_release); - } - cbm_mutex_unlock(&g_pretrained_mtx); -} - -void cbm_sem_ensure_ready(void) { - ensure_pretrained_map(); -} - -void cbm_sem_random_index(const char *token, cbm_sem_vec_t *out) { - memset(out, 0, sizeof(*out)); - if (!token) { - return; - } - - /* Try pretrained nomic-embed-code vector first (768d, distilled from 7B). */ - ensure_pretrained_map(); - const char *idx_str = cbm_ht_get(g_pretrained_map, token); - if (idx_str) { - char *end = NULL; - long idx = strtol(idx_str, &end, BASE_DECIMAL); - if (end != idx_str && idx >= 0 && idx < PRETRAINED_TOKEN_COUNT) { - const int8_t *pvec = pretrained_vec_at((int)idx); - for (int d = 0; d < CBM_SEM_DIM && d < PRETRAINED_DIM; d++) { - out->v[d] = (float)pvec[d] / CBM_SEM_INT8_MAX; - } - return; - } - } - - /* Fallback: sparse random vector for tokens not in pretrained vocab. */ - uint64_t seed = XXH3_64bits(token, strlen(token)); - for (int i = 0; i < CBM_SEM_SPARSE_NNZE; i++) { - uint64_t h = XXH3_64bits_withSeed(&i, sizeof(i), seed + RI_SEED_BASE); - int pos = (int)(h % CBM_SEM_DIM); - float sign = (h & SKIP_ONE) ? CBM_SEM_UNIT_POS : -CBM_SEM_UNIT_POS; - out->v[pos] += sign; - } -} - -void cbm_sem_normalize(cbm_sem_vec_t *v) { - if (!v) { - return; - } - float mag = 0.0F; - for (int i = 0; i < CBM_SEM_DIM; i++) { - mag += v->v[i] * v->v[i]; - } - mag = sqrtf(mag); - if (mag < CBM_SEM_DENOM_EPS) { - return; - } - float inv = CBM_SEM_UNIT_POS / mag; - for (int i = 0; i < CBM_SEM_DIM; i++) { - v->v[i] *= inv; - } -} - -void cbm_sem_vec_add_scaled(cbm_sem_vec_t *dst, const cbm_sem_vec_t *src, float scale) { - if (!dst || !src) { - return; - } - for (int i = 0; i < CBM_SEM_DIM; i++) { - dst->v[i] += scale * src->v[i]; - } -} - -/* ── Corpus (IDF + Random Indexing enrichment) ───────────────────── */ - -typedef struct { - char *token; - int doc_freq; - cbm_sem_vec_t enriched_vec; /* context-enriched via co-occurrence */ -} corpus_entry_t; - -struct cbm_sem_corpus { - CBMHashTable *token_map; /* token → index into entries[] */ - corpus_entry_t *entries; - int entry_count; - int entry_cap; - int doc_count; - bool finalized; - - /* Per-document token lists for co-occurrence pass */ - int **doc_token_ids; - int *doc_token_counts; - int doc_cap; -}; - -static int corpus_get_or_add(cbm_sem_corpus_t *c, const char *token) { - char idx_buf[CBM_SZ_16]; - const char *existing = cbm_ht_get(c->token_map, token); - if (existing) { - char *end = NULL; - long parsed = strtol(existing, &end, BASE_DECIMAL); - return (end != existing) ? (int)parsed : CBM_NOT_FOUND; - } - if (c->entry_count >= c->entry_cap) { - int new_cap = c->entry_cap < CORPUS_INIT_CAP ? CORPUS_INIT_CAP : c->entry_cap * PAIR_LEN; - corpus_entry_t *grown = realloc(c->entries, (size_t)new_cap * sizeof(corpus_entry_t)); - if (!grown) { - return CBM_NOT_FOUND; - } - c->entries = grown; - c->entry_cap = new_cap; - } - int idx = c->entry_count++; - c->entries[idx].token = strdup(token); - c->entries[idx].doc_freq = 0; - memset(&c->entries[idx].enriched_vec, 0, sizeof(cbm_sem_vec_t)); - snprintf(idx_buf, sizeof(idx_buf), "%d", idx); - cbm_ht_set(c->token_map, strdup(token), strdup(idx_buf)); - return idx; -} - -cbm_sem_corpus_t *cbm_sem_corpus_new(void) { - cbm_sem_corpus_t *c = calloc(SKIP_ONE, sizeof(cbm_sem_corpus_t)); - if (c) { - c->token_map = cbm_ht_create(CORPUS_INIT_CAP); - } - return c; -} - -void cbm_sem_corpus_add_doc(cbm_sem_corpus_t *corpus, const char **tokens, int count) { - if (!corpus || !tokens || count <= 0) { - return; - } - /* Track document for co-occurrence pass */ - if (corpus->doc_count >= corpus->doc_cap) { - int new_cap = - corpus->doc_cap < DOC_TOKENS_INIT ? DOC_TOKENS_INIT : corpus->doc_cap * PAIR_LEN; - int **grown_ids = realloc(corpus->doc_token_ids, (size_t)new_cap * sizeof(int *)); - int *grown_counts = realloc(corpus->doc_token_counts, (size_t)new_cap * sizeof(int)); - if (!grown_ids || !grown_counts) { - free(grown_ids); - free(grown_counts); - return; - } - corpus->doc_token_ids = grown_ids; - corpus->doc_token_counts = grown_counts; - corpus->doc_cap = new_cap; - } - int doc_idx = corpus->doc_count++; - corpus->doc_token_ids[doc_idx] = malloc((size_t)count * sizeof(int)); - corpus->doc_token_counts[doc_idx] = count; - - /* Per-doc unique set for IDF */ - int *seen = calloc((size_t)corpus->entry_cap + (size_t)count + CORPUS_INIT_CAP, sizeof(int)); - int seen_count = 0; - - for (int i = 0; i < count; i++) { - int tid = corpus_get_or_add(corpus, tokens[i]); - corpus->doc_token_ids[doc_idx][i] = tid; - if (tid < 0) { - continue; - } - /* Check uniqueness for IDF (simple linear scan — tokens per doc is small) */ - bool is_new = true; - for (int j = 0; j < seen_count; j++) { - if (seen[j] == tid) { - is_new = false; - break; - } - } - if (is_new) { - seen[seen_count++] = tid; - corpus->entries[tid].doc_freq++; - } - } - free(seen); -} - -/* ── Parallel corpus batch build ──────────────────────────────────── */ -/* Strategy: - * Phase A (SEQUENTIAL): Scan all documents once to build the global - * token_map (inserts unique tokens, assigns global IDs). This is - * inherently sequential (hash table mutation), but much faster than - * the current per-doc add_doc because we avoid the per-doc malloc of - * the `seen` array and per-doc bookkeeping. - * Phase B (PARALLEL): Each worker processes a chunk of docs, translates - * tokens → global IDs via read-only token_map lookups, fills - * doc_token_ids[d], and accumulates doc_freq contributions via atomics. - */ - -typedef struct { - cbm_sem_corpus_t *corpus; - char **all_tokens; - const int *token_counts; - int max_tokens; - int doc_count; - _Atomic int *doc_freq_atomic; /* per-entry atomic counter (entry_count long) */ - _Atomic int next_idx; -} batch_resolve_ctx_t; - -/* Resolve one document: look up each token's global ID, fill the corpus - * doc_token_ids[d], and bump the per-token doc_freq counter atomically. The - * caller is responsible for ensuring `seen` has capacity for `count` ints - * before calling (the worker grows its per-thread scratch buffer). */ -static void batch_resolve_one_doc(batch_resolve_ctx_t *bc, int doc_index, int *seen) { - int count = bc->token_counts[doc_index]; - if (count <= 0) { - bc->corpus->doc_token_ids[doc_index] = NULL; - bc->corpus->doc_token_counts[doc_index] = 0; - return; - } - int *ids = malloc((size_t)count * sizeof(int)); - bc->corpus->doc_token_ids[doc_index] = ids; - bc->corpus->doc_token_counts[doc_index] = count; - - int seen_count = 0; - char **tokens = &bc->all_tokens[(ptrdiff_t)doc_index * bc->max_tokens]; - for (int i = 0; i < count; i++) { - const char *idx_str = cbm_ht_get(bc->corpus->token_map, tokens[i]); - int tid = CBM_NOT_FOUND; - if (idx_str) { - char *end = NULL; - long parsed = strtol(idx_str, &end, BASE_DECIMAL); - if (end != idx_str) { - tid = (int)parsed; - } - } - ids[i] = tid; - if (tid < 0) { - continue; - } - /* Unique-per-doc check for IDF */ - bool is_new = true; - for (int j = 0; j < seen_count; j++) { - if (seen[j] == tid) { - is_new = false; - break; - } - } - if (is_new) { - seen[seen_count++] = tid; - atomic_fetch_add_explicit(&bc->doc_freq_atomic[tid], CBM_SEM_ATOMIC_INC, - memory_order_relaxed); - } - } -} - -static void batch_resolve_worker(int worker_id, void *ctx_ptr) { - (void)worker_id; - batch_resolve_ctx_t *bc = ctx_ptr; - /* Per-worker scratch for unique-per-doc tracking */ - int local_seen_cap = CBM_SEM_SEEN_INIT_CAP; - int *seen = malloc((size_t)local_seen_cap * sizeof(int)); - if (!seen) { - return; - } - - while (true) { - int start = - atomic_fetch_add_explicit(&bc->next_idx, CBM_SEM_RESOLVE_CHUNK, memory_order_relaxed); - if (start >= bc->doc_count) { - break; - } - int end = start + CBM_SEM_RESOLVE_CHUNK; - if (end > bc->doc_count) { - end = bc->doc_count; - } - for (int d = start; d < end; d++) { - int count = bc->token_counts[d]; - if (count > local_seen_cap) { - int *grown = realloc(seen, (size_t)count * sizeof(int)); - if (!grown) { - continue; - } - seen = grown; - local_seen_cap = count; - } - batch_resolve_one_doc(bc, d, seen); - } - } - free(seen); -} - -void cbm_sem_corpus_add_docs_batch(cbm_sem_corpus_t *corpus, char **all_tokens, - const int *token_counts, int doc_count, int max_tokens_per_doc) { - if (!corpus || !all_tokens || !token_counts || doc_count <= 0) { - return; - } - - /* Phase A (SEQUENTIAL): Build token_map and allocate doc arrays. - * Hash table mutation can't be parallelized; strdup+insert is the cost. */ - if (corpus->doc_cap < corpus->doc_count + doc_count) { - int new_cap = corpus->doc_count + doc_count; - int **grown_ids = realloc(corpus->doc_token_ids, (size_t)new_cap * sizeof(int *)); - int *grown_counts = realloc(corpus->doc_token_counts, (size_t)new_cap * sizeof(int)); - if (!grown_ids || !grown_counts) { - free(grown_ids); - free(grown_counts); - return; - } - corpus->doc_token_ids = grown_ids; - corpus->doc_token_counts = grown_counts; - corpus->doc_cap = new_cap; - } - int base_doc = corpus->doc_count; - corpus->doc_count += doc_count; - - for (int d = 0; d < doc_count; d++) { - int count = token_counts[d]; - char **tokens = &all_tokens[(ptrdiff_t)d * max_tokens_per_doc]; - for (int i = 0; i < count; i++) { - /* Inserts token into token_map if new; we discard return here — - * Phase B will re-lookup in read-only mode to get the ID. */ - (void)corpus_get_or_add(corpus, tokens[i]); - } - } - - /* Phase B (PARALLEL): Resolve tokens → IDs and count doc_freq per entry. - * token_map is now read-only; each worker owns its doc range (no writes - * to shared state except atomic doc_freq counters). */ - _Atomic int *doc_freq_atomic = calloc((size_t)corpus->entry_count, sizeof(_Atomic int)); - if (!doc_freq_atomic) { - /* OOM fallback: sequential path. Roll back doc_count first since - * add_doc increments it itself. */ - corpus->doc_count = base_doc; - for (int d = 0; d < doc_count; d++) { - int count = token_counts[d]; - char **tokens = &all_tokens[(ptrdiff_t)d * max_tokens_per_doc]; - cbm_sem_corpus_add_doc(corpus, (const char **)tokens, count); - } - return; - } - - int worker_count = cbm_default_worker_count(false); - batch_resolve_ctx_t bc = { - .corpus = corpus, - .all_tokens = all_tokens, - .token_counts = token_counts, - .max_tokens = max_tokens_per_doc, - .doc_count = doc_count, - .doc_freq_atomic = doc_freq_atomic, - }; - atomic_init(&bc.next_idx, 0); - /* Temporarily re-base doc arrays so workers write to base_doc..base_doc+doc_count */ - corpus->doc_token_ids += base_doc; - corpus->doc_token_counts += base_doc; - cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false}; - cbm_parallel_for(worker_count, batch_resolve_worker, &bc, opts); - corpus->doc_token_ids -= base_doc; - corpus->doc_token_counts -= base_doc; - - /* Phase C (SEQUENTIAL reduce): atomic counters → entries[].doc_freq */ - for (int i = 0; i < corpus->entry_count; i++) { - corpus->entries[i].doc_freq += - atomic_load_explicit(&doc_freq_atomic[i], memory_order_relaxed); - } - free(doc_freq_atomic); -} - -/* ── Parallel corpus_finalize ─────────────────────────────────────── */ -/* Strategy: - * 1. Precompute base RI vectors into a shared array (eliminates ~333M - * redundant cbm_sem_random_index calls on kernel-scale corpora). - * 2. Co-occurrence passes: partition TARGET tokens across workers so each - * worker writes to a disjoint range of enriched_vec (zero contention). - * Each worker still scans all documents but only accumulates for targets - * in its range. Inner vector add is the parallelized work. - * 3. Normalize/blend loops are trivially parallel per-entry. - */ - -/* Reverse index: for each token id, a list of (doc_id, position_in_doc) pairs. - * Built once, reused for both cooccur passes. Eliminates the O(num_chunks × doc_count) - * redundant outer scan in the old algorithm. */ -typedef struct { - int32_t doc_id; - int32_t pos; -} cooccur_pos_t; - -typedef struct { - int *offsets; /* offsets[entry_count + 1], prefix sum of occurrences */ - cooccur_pos_t *flat; /* flat array of positions, total = offsets[entry_count] */ -} reverse_index_t; - -/* Tagged source vector: sparse (~30% of tokens, 8 inline nonzeros) or dense int8 - * reference into PRETRAINED_VECTOR_BLOB (no copy). Dense path converts int8→float - * on the fly in the hot loop. Tested with both int8 and float32 blob storage - * formats — cooccur passes are memory-bandwidth-bound, so the int8 format (4x - * less source traffic) is equivalent in wall time despite the conversion cost, - * while saving 90 MB of binary size. */ -typedef struct { - uint8_t is_sparse; /* 1 = sparse path, 0 = dense int8 reference */ - uint8_t nnz; /* number of nonzeros used in sparse path */ - uint16_t _pad; - uint16_t indices[CBM_SEM_SPARSE_NNZE]; /* 8 * 2 = 16 bytes */ - float values[CBM_SEM_SPARSE_NNZE]; /* 8 * 4 = 32 bytes */ - const int8_t *dense_int8; /* points into PRETRAINED_VECTOR_BLOB */ -} cbm_sem_src_entry_t; - -/* Inline helper: initialize a target vector from a sparse/dense source. */ -static inline void sem_target_init_from_src(cbm_sem_vec_t *dst, const cbm_sem_src_entry_t *src) { - memset(dst, 0, sizeof(*dst)); - if (src->is_sparse) { - for (int k = 0; k < src->nnz; k++) { - dst->v[src->indices[k]] = src->values[k]; - } - } else { - const int8_t *s = src->dense_int8; - const float inv127 = CBM_SEM_UNIT_POS / CBM_SEM_INT8_MAX; - for (int d = 0; d < CBM_SEM_DIM; d++) { - dst->v[d] = inv127 * (float)s[d]; - } - } -} - -/* Inline helper: add weighted source into target. - * Sparse path: ~8 operations, ~48 bytes source memory traffic. - * Dense path: 768 mul-adds with int8→float conversion, ~768 bytes traffic. */ -static inline void sem_vec_add_src_scaled(cbm_sem_vec_t *dst, const cbm_sem_src_entry_t *src, - float scale) { - if (src->is_sparse) { - for (int k = 0; k < src->nnz; k++) { - dst->v[src->indices[k]] += scale * src->values[k]; - } - } else { - const int8_t *s = src->dense_int8; - const float mul = scale * (CBM_SEM_UNIT_POS / CBM_SEM_INT8_MAX); - for (int d = 0; d < CBM_SEM_DIM; d++) { - dst->v[d] += mul * (float)s[d]; - } - } -} - -/* Pass 1 context: uses sparse/int8 tagged sources (most memory-efficient). */ -typedef struct { - corpus_entry_t *entries; - const cbm_sem_src_entry_t *src_entries; /* sparse or int8-dense per token */ - int **doc_token_ids; - const int *doc_token_counts; - const reverse_index_t *rev; - int doc_count; - int entry_count; - _Atomic int next_chunk; - int num_chunks; - int chunk_size; - - /* Cache-blocked tiling parameters */ - int tile_size; /* targets per L2-resident tile */ -} cooccur_sparse_ctx_t; - -/* Accumulate co-occurrence context for a single target token into `target`. - * Reads neighbors within ±CBM_SEM_WINDOW positions across all documents that - * reference this token id via the reverse index. */ -static void cooccur_sparse_one_target(cooccur_sparse_ctx_t *cc, int tid, cbm_sem_vec_t *target) { - int occ_start = cc->rev->offsets[tid]; - int occ_end = cc->rev->offsets[tid + SKIP_ONE]; - /* Frequent-token subsampling: stride over occurrences when a token is very - * common so the enriched vector's direction is preserved (it's normalized - * after) while bounding work to ~CBM_SEM_MAX_OCCUR samples. See semantic.h. */ - int occ_step = SKIP_ONE; - if (occ_end - occ_start > CBM_SEM_MAX_OCCUR) { - occ_step = (occ_end - occ_start) / CBM_SEM_MAX_OCCUR; - } - for (int p = occ_start; p < occ_end; p += occ_step) { - int d = cc->rev->flat[p].doc_id; - int i = cc->rev->flat[p].pos; - int *ids = cc->doc_token_ids[d]; - int len = cc->doc_token_counts[d]; - for (int w = -CBM_SEM_WINDOW; w <= CBM_SEM_WINDOW; w++) { - if (w == 0) { - continue; - } - int j = i + w; - if (j < 0 || j >= len) { - continue; - } - int nid = ids[j]; - if (nid < 0) { - continue; - } - float weight = CBM_SEM_UNIT_POS / (float)abs(w); - sem_vec_add_src_scaled(target, &cc->src_entries[nid], weight); - } - } -} - -static void cooccur_worker_sparse(int worker_id, void *ctx_ptr) { - (void)worker_id; - cooccur_sparse_ctx_t *cc = ctx_ptr; - while (true) { - int ci = - atomic_fetch_add_explicit(&cc->next_chunk, CBM_SEM_ATOMIC_INC, memory_order_relaxed); - if (ci >= cc->num_chunks) { - break; - } - int chunk_start = ci * cc->chunk_size; - int chunk_end = chunk_start + cc->chunk_size; - if (chunk_end > cc->entry_count) { - chunk_end = cc->entry_count; - } - - /* Cache-blocked target tiling: process tile_size targets at a time so - * their vectors stay resident in L2 cache during their accumulation. */ - for (int tile_start = chunk_start; tile_start < chunk_end; tile_start += cc->tile_size) { - int tile_end = tile_start + cc->tile_size; - if (tile_end > chunk_end) { - tile_end = chunk_end; - } - for (int tid = tile_start; tid < tile_end; tid++) { - sem_target_init_from_src(&cc->entries[tid].enriched_vec, &cc->src_entries[tid]); - cooccur_sparse_one_target(cc, tid, &cc->entries[tid].enriched_vec); - } - } - } -} - -/* Pass 2 (RRI) context: uses int8-quantized pass1 vectors as source. - * Pass1 outputs are dense float32 post-normalization, values in [-1,1]. - * We quantize to int8 once (×127) to cut source memory traffic 4x. */ -typedef struct { - corpus_entry_t *entries; - const int8_t *pass1_q; /* [entry_count × CBM_SEM_DIM] int8 quantized pass1 */ - int **doc_token_ids; - const int *doc_token_counts; - const reverse_index_t *rev; - int doc_count; - int entry_count; - _Atomic int next_chunk; - int num_chunks; - int chunk_size; - int tile_size; -} cooccur_int8_ctx_t; - -static inline void sem_vec_add_int8_scaled(cbm_sem_vec_t *dst, const int8_t *src, float scale) { - const float mul = scale * (CBM_SEM_UNIT_POS / CBM_SEM_INT8_MAX); - for (int d = 0; d < CBM_SEM_DIM; d++) { - dst->v[d] += mul * (float)src[d]; - } -} - -/* RRI pass 2 accumulator for a single target token, reading int8-quantized - * pass1 vectors as the source. */ -static void cooccur_int8_one_target(cooccur_int8_ctx_t *cc, int tid, cbm_sem_vec_t *target) { - int occ_start = cc->rev->offsets[tid]; - int occ_end = cc->rev->offsets[tid + SKIP_ONE]; - /* Frequent-token subsampling (see cooccur_sparse_one_target / semantic.h). */ - int occ_step = SKIP_ONE; - if (occ_end - occ_start > CBM_SEM_MAX_OCCUR) { - occ_step = (occ_end - occ_start) / CBM_SEM_MAX_OCCUR; - } - for (int p = occ_start; p < occ_end; p += occ_step) { - int d = cc->rev->flat[p].doc_id; - int i = cc->rev->flat[p].pos; - int *ids = cc->doc_token_ids[d]; - int len = cc->doc_token_counts[d]; - for (int w = -CBM_SEM_WINDOW; w <= CBM_SEM_WINDOW; w++) { - if (w == 0) { - continue; - } - int j = i + w; - if (j < 0 || j >= len) { - continue; - } - int nid = ids[j]; - if (nid < 0) { - continue; - } - float weight = CBM_SEM_UNIT_POS / (float)abs(w); - sem_vec_add_int8_scaled(target, &cc->pass1_q[(size_t)nid * CBM_SEM_DIM], weight); - } - } -} - -static void cooccur_worker_int8(int worker_id, void *ctx_ptr) { - (void)worker_id; - cooccur_int8_ctx_t *cc = ctx_ptr; - while (true) { - int ci = - atomic_fetch_add_explicit(&cc->next_chunk, CBM_SEM_ATOMIC_INC, memory_order_relaxed); - if (ci >= cc->num_chunks) { - break; - } - int chunk_start = ci * cc->chunk_size; - int chunk_end = chunk_start + cc->chunk_size; - if (chunk_end > cc->entry_count) { - chunk_end = cc->entry_count; - } - - for (int tile_start = chunk_start; tile_start < chunk_end; tile_start += cc->tile_size) { - int tile_end = tile_start + cc->tile_size; - if (tile_end > chunk_end) { - tile_end = chunk_end; - } - for (int tid = tile_start; tid < tile_end; tid++) { - /* RRI pass 2 starts from zero (no self-init) */ - memset(&cc->entries[tid].enriched_vec, 0, sizeof(cbm_sem_vec_t)); - cooccur_int8_one_target(cc, tid, &cc->entries[tid].enriched_vec); - } - } - } -} - -typedef struct { - corpus_entry_t *entries; - int entry_count; - _Atomic int next_idx; -} norm_ctx_t; - -static void normalize_worker(int worker_id, void *ctx_ptr) { - (void)worker_id; - norm_ctx_t *nc = ctx_ptr; - while (true) { - int start = atomic_fetch_add_explicit(&nc->next_idx, CBM_SEM_WORKER_STACK_CAP, - memory_order_relaxed); - if (start >= nc->entry_count) { - break; - } - int end = start + CBM_SEM_WORKER_STACK_CAP; - if (end > nc->entry_count) { - end = nc->entry_count; - } - for (int i = start; i < end; i++) { - cbm_sem_normalize(&nc->entries[i].enriched_vec); - } - } -} - -typedef struct { - corpus_entry_t *entries; - const cbm_sem_vec_t *pass1; - int entry_count; - _Atomic int next_idx; -} blend_ctx_t; - -static void blend_worker(int worker_id, void *ctx_ptr) { - (void)worker_id; - blend_ctx_t *bc = ctx_ptr; - while (true) { - int start = atomic_fetch_add_explicit(&bc->next_idx, CBM_SEM_WORKER_STACK_CAP, - memory_order_relaxed); - if (start >= bc->entry_count) { - break; - } - int end = start + CBM_SEM_WORKER_STACK_CAP; - if (end > bc->entry_count) { - end = bc->entry_count; - } - for (int i = start; i < end; i++) { - cbm_sem_normalize(&bc->entries[i].enriched_vec); - for (int d = 0; d < CBM_SEM_DIM; d++) { - bc->entries[i].enriched_vec.v[d] = - (CBM_SEM_RRI_BETA * bc->pass1[i].v[d]) + - (CBM_SEM_RRI_ALPHA * bc->entries[i].enriched_vec.v[d]); - } - } - } -} - -typedef struct { - corpus_entry_t *entries; - cbm_sem_src_entry_t *src_entries; - int entry_count; - _Atomic int next_idx; -} src_build_ctx_t; - -/* Build one src_entry for a token: dense float32 reference if in nomic vocab, - * sparse inline representation otherwise. Collisions in the sparse hash are - * merged and zeros filtered so the final representation is exactly the same - * mathematical vector that the old dense path produced. */ -static void build_src_entry(const char *token, cbm_sem_src_entry_t *out) { - memset(out, 0, sizeof(*out)); - if (!token) { - out->is_sparse = SKIP_ONE; - out->nnz = 0; - return; - } - /* Dense path: direct int8 pointer into pretrained blob (zero-copy). */ - const char *idx_str = cbm_ht_get(g_pretrained_map, token); - if (idx_str) { - char *end = NULL; - long idx = strtol(idx_str, &end, BASE_DECIMAL); - if (end != idx_str && idx >= 0 && idx < PRETRAINED_TOKEN_COUNT) { - out->is_sparse = 0; - out->dense_int8 = pretrained_vec_at((int)idx); - return; - } - } - /* Sparse path: compute 8 hash positions with collision merging. */ - out->is_sparse = SKIP_ONE; - uint16_t tmp_idx[CBM_SEM_SPARSE_NNZE]; - float tmp_val[CBM_SEM_SPARSE_NNZE]; - int count = 0; - uint64_t seed = XXH3_64bits(token, strlen(token)); - for (int i = 0; i < CBM_SEM_SPARSE_NNZE; i++) { - uint64_t h = XXH3_64bits_withSeed(&i, sizeof(i), seed + RI_SEED_BASE); - int pos = (int)(h % CBM_SEM_DIM); - float sign = (h & SKIP_ONE) ? CBM_SEM_UNIT_POS : -CBM_SEM_UNIT_POS; - /* Merge collisions */ - int found = CBM_NOT_FOUND; - for (int j = 0; j < count; j++) { - if (tmp_idx[j] == (uint16_t)pos) { - found = j; - break; - } - } - if (found >= 0) { - tmp_val[found] += sign; - } else { - tmp_idx[count] = (uint16_t)pos; - tmp_val[count] = sign; - count++; - } - } - /* Filter zeros */ - int nnz = 0; - for (int j = 0; j < count; j++) { - if (tmp_val[j] != 0.0F) { - out->indices[nnz] = tmp_idx[j]; - out->values[nnz] = tmp_val[j]; - nnz++; - } - } - out->nnz = (uint8_t)nnz; -} - -static void src_build_worker(int worker_id, void *ctx_ptr) { - (void)worker_id; - src_build_ctx_t *sc = ctx_ptr; - while (true) { - int start = atomic_fetch_add_explicit(&sc->next_idx, CBM_SEM_WORKER_STACK_CAP, - memory_order_relaxed); - if (start >= sc->entry_count) { - break; - } - int end = start + CBM_SEM_WORKER_STACK_CAP; - if (end > sc->entry_count) { - end = sc->entry_count; - } - for (int i = start; i < end; i++) { - build_src_entry(sc->entries[i].token, &sc->src_entries[i]); - } - } -} - -/* Quantize pass1 float vectors to int8 (× 127) for use as pass2 source. - * Input vectors are unit-normalized, so values are in [-1, 1] → int8 preserves - * ~99% precision. 4x less memory traffic for pass2. */ -typedef struct { - const corpus_entry_t *entries; - int8_t *pass1_q; - int entry_count; - _Atomic int next_idx; -} pass1_quant_ctx_t; - -static void pass1_quantize_worker(int worker_id, void *ctx_ptr) { - (void)worker_id; - pass1_quant_ctx_t *qc = ctx_ptr; - while (true) { - int start = - atomic_fetch_add_explicit(&qc->next_idx, CBM_SEM_RRI_TILE, memory_order_relaxed); - if (start >= qc->entry_count) { - break; - } - int end = start + CBM_SEM_RRI_TILE; - if (end > qc->entry_count) { - end = qc->entry_count; - } - for (int i = start; i < end; i++) { - const cbm_sem_vec_t *src = &qc->entries[i].enriched_vec; - int8_t *dst = &qc->pass1_q[(size_t)i * CBM_SEM_DIM]; - for (int d = 0; d < CBM_SEM_DIM; d++) { - float v = src->v[d] * CBM_SEM_INT8_MAX; - if (v > CBM_SEM_INT8_MAX) { - v = CBM_SEM_INT8_MAX; - } - if (v < -CBM_SEM_INT8_MAX) { - v = -CBM_SEM_INT8_MAX; - } - dst[d] = (int8_t)(v >= 0 ? v + CBM_SEM_IDF_BASE : v - CBM_SEM_IDF_BASE); - } - } - } -} - -/* Build reverse index: token_id → list of (doc_id, position) pairs. - * SEQUENTIAL (fast: just pointer arithmetic + flat array fill). */ -static reverse_index_t *build_reverse_index(cbm_sem_corpus_t *corpus) { - reverse_index_t *rev = calloc(SKIP_ONE, sizeof(reverse_index_t)); - if (!rev) { - return NULL; - } - /* Phase A: count occurrences per token */ - int *counts = calloc((size_t)corpus->entry_count + SKIP_ONE, sizeof(int)); - if (!counts) { - free(rev); - return NULL; - } - long total = 0; - for (int d = 0; d < corpus->doc_count; d++) { - int *ids = corpus->doc_token_ids[d]; - int len = corpus->doc_token_counts[d]; - for (int i = 0; i < len; i++) { - int tid = ids[i]; - if (tid >= 0 && tid < corpus->entry_count) { - counts[tid]++; - total++; - } - } - } - /* Phase B: exclusive prefix sum → offsets[] */ - rev->offsets = malloc(((size_t)corpus->entry_count + SKIP_ONE) * sizeof(int)); - if (!rev->offsets) { - free(counts); - free(rev); - return NULL; - } - int running = 0; - for (int t = 0; t < corpus->entry_count; t++) { - rev->offsets[t] = running; - running += counts[t]; - counts[t] = 0; /* reuse as per-token fill cursor */ - } - rev->offsets[corpus->entry_count] = running; - /* Phase C: fill flat array. Ensure allocation size > 0 even for empty - * corpora (avoids malloc(0) which is implementation-defined). */ - size_t flat_bytes = (total > 0 ? (size_t)total : SKIP_ONE) * sizeof(cooccur_pos_t); - rev->flat = malloc(flat_bytes); - if (!rev->flat) { - free(rev->offsets); - free(counts); - free(rev); - return NULL; - } - for (int d = 0; d < corpus->doc_count; d++) { - int *ids = corpus->doc_token_ids[d]; - int len = corpus->doc_token_counts[d]; - for (int i = 0; i < len; i++) { - int tid = ids[i]; - if (tid >= 0 && tid < corpus->entry_count) { - int slot = rev->offsets[tid] + counts[tid]++; - rev->flat[slot].doc_id = (int32_t)d; - rev->flat[slot].pos = (int32_t)i; - } - } - } - free(counts); - return rev; -} - -static void free_reverse_index(reverse_index_t *rev) { - if (!rev) { - return; - } - free(rev->offsets); - free(rev->flat); - free(rev); -} - -/* Bundle of parameters shared by the finalize sub-phases. */ -typedef struct { - cbm_sem_corpus_t *corpus; - reverse_index_t *rev; - cbm_sem_src_entry_t *src_entries; - int worker_count; - int num_chunks; - int chunk_size; - int tile_size; - cbm_parallel_for_opts_t opts; -} finalize_params_t; - -/* Sub-phase 1: build tagged source vectors (sparse or dense-int8) in parallel. */ -static void finalize_build_sources(finalize_params_t *p) { - src_build_ctx_t sc = { - .entries = p->corpus->entries, - .src_entries = p->src_entries, - .entry_count = p->corpus->entry_count, - }; - atomic_init(&sc.next_idx, 0); - cbm_parallel_for(p->worker_count, src_build_worker, &sc, p->opts); -} - -/* Sub-phases 2+3: co-occurrence pass 1 + normalize. */ -static void finalize_pass1(finalize_params_t *p) { - cooccur_sparse_ctx_t cc = { - .entries = p->corpus->entries, - .src_entries = p->src_entries, - .doc_token_ids = p->corpus->doc_token_ids, - .doc_token_counts = p->corpus->doc_token_counts, - .rev = p->rev, - .doc_count = p->corpus->doc_count, - .entry_count = p->corpus->entry_count, - .num_chunks = p->num_chunks, - .chunk_size = p->chunk_size, - .tile_size = p->tile_size, - }; - atomic_init(&cc.next_chunk, 0); - cbm_parallel_for(p->worker_count, cooccur_worker_sparse, &cc, p->opts); - - norm_ctx_t nc = {.entries = p->corpus->entries, .entry_count = p->corpus->entry_count}; - atomic_init(&nc.next_idx, 0); - cbm_parallel_for(p->worker_count, normalize_worker, &nc, p->opts); -} - -/* Sub-phases 4+5: quantize pass1 to int8, run RRI pass 2, blend + normalize. */ -static void finalize_pass2(finalize_params_t *p) { - int8_t *pass1_q = malloc((size_t)p->corpus->entry_count * CBM_SEM_DIM * sizeof(int8_t)); - if (pass1_q) { - pass1_quant_ctx_t qc = { - .entries = p->corpus->entries, - .pass1_q = pass1_q, - .entry_count = p->corpus->entry_count, - }; - atomic_init(&qc.next_idx, 0); - cbm_parallel_for(p->worker_count, pass1_quantize_worker, &qc, p->opts); - } - - cbm_sem_vec_t *pass1 = malloc((size_t)p->corpus->entry_count * sizeof(cbm_sem_vec_t)); - if (pass1) { - for (int i = 0; i < p->corpus->entry_count; i++) { - pass1[i] = p->corpus->entries[i].enriched_vec; - } - } - - if (pass1_q) { - cooccur_int8_ctx_t cc = { - .entries = p->corpus->entries, - .pass1_q = pass1_q, - .doc_token_ids = p->corpus->doc_token_ids, - .doc_token_counts = p->corpus->doc_token_counts, - .rev = p->rev, - .doc_count = p->corpus->doc_count, - .entry_count = p->corpus->entry_count, - .num_chunks = p->num_chunks, - .chunk_size = p->chunk_size, - .tile_size = p->tile_size, - }; - atomic_init(&cc.next_chunk, 0); - cbm_parallel_for(p->worker_count, cooccur_worker_int8, &cc, p->opts); - } - - if (pass1) { - blend_ctx_t bc = { - .entries = p->corpus->entries, - .pass1 = pass1, - .entry_count = p->corpus->entry_count, - }; - atomic_init(&bc.next_idx, 0); - cbm_parallel_for(p->worker_count, blend_worker, &bc, p->opts); - free(pass1); - } - free(pass1_q); - - norm_ctx_t nc = {.entries = p->corpus->entries, .entry_count = p->corpus->entry_count}; - atomic_init(&nc.next_idx, 0); - cbm_parallel_for(p->worker_count, normalize_worker, &nc, p->opts); -} - -void cbm_sem_corpus_finalize(cbm_sem_corpus_t *corpus) { - if (!corpus || corpus->finalized) { - return; - } - - /* Eager init before parallel dispatch to avoid lazy-init races */ - ensure_pretrained_map(); - - int worker_count = cbm_default_worker_count(false); - cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false}; - - /* Finer chunks = better load balancing for skewed token distributions. */ - int num_chunks = worker_count * CBM_SEM_COOCCUR_CHUNK; - if (num_chunks > corpus->entry_count) { - num_chunks = corpus->entry_count; - } - if (num_chunks < SKIP_ONE) { - num_chunks = SKIP_ONE; - } - int chunk_size = (corpus->entry_count + num_chunks - SKIP_ONE) / num_chunks; - - reverse_index_t *rev = build_reverse_index(corpus); - if (!rev) { - corpus->finalized = true; - return; - } - cbm_sem_src_entry_t *src_entries = - calloc((size_t)corpus->entry_count, sizeof(cbm_sem_src_entry_t)); - if (!src_entries) { - free_reverse_index(rev); - corpus->finalized = true; - return; - } - - finalize_params_t params = { - .corpus = corpus, - .rev = rev, - .src_entries = src_entries, - .worker_count = worker_count, - .num_chunks = num_chunks, - .chunk_size = chunk_size, - .tile_size = CBM_SEM_TILE_SIZE, - .opts = opts, - }; - finalize_build_sources(¶ms); - finalize_pass1(¶ms); - finalize_pass2(¶ms); - - free(src_entries); - free_reverse_index(rev); - corpus->finalized = true; -} - -/* Parse a decimal index string via strtol, returning CBM_NOT_FOUND on parse - * failure. Centralizes the atoi-replacement pattern for token_map lookups. */ -static int parse_token_index(const char *idx_str) { - if (!idx_str) { - return CBM_NOT_FOUND; - } - char *end = NULL; - long parsed = strtol(idx_str, &end, BASE_DECIMAL); - return (end != idx_str) ? (int)parsed : CBM_NOT_FOUND; -} - -float cbm_sem_corpus_idf(const cbm_sem_corpus_t *corpus, const char *token) { - if (!corpus || !token || corpus->doc_count == 0) { - return 0.0F; - } - int idx = parse_token_index(cbm_ht_get(corpus->token_map, token)); - if (idx < 0 || idx >= corpus->entry_count) { - return 0.0F; - } - int df = corpus->entries[idx].doc_freq; - if (df <= 0) { - return 0.0F; - } - return logf((float)corpus->doc_count / (float)df); -} - -const cbm_sem_vec_t *cbm_sem_corpus_ri_vec(const cbm_sem_corpus_t *corpus, const char *token) { - if (!corpus || !token) { - return NULL; - } - int idx = parse_token_index(cbm_ht_get(corpus->token_map, token)); - if (idx < 0 || idx >= corpus->entry_count) { - return NULL; - } - return &corpus->entries[idx].enriched_vec; -} - -int cbm_sem_corpus_doc_count(const cbm_sem_corpus_t *corpus) { - return corpus ? corpus->doc_count : 0; -} - -int cbm_sem_corpus_token_count(const cbm_sem_corpus_t *corpus) { - return corpus ? corpus->entry_count : 0; -} - -const char *cbm_sem_corpus_token_at(const cbm_sem_corpus_t *corpus, int index, - const cbm_sem_vec_t **out_vec, float *out_idf) { - if (!corpus || index < 0 || index >= corpus->entry_count) { - return NULL; - } - if (out_vec) { - *out_vec = &corpus->entries[index].enriched_vec; - } - if (out_idf && corpus->doc_count > 0) { - int df = corpus->entries[index].doc_freq; - *out_idf = df > 0 ? logf((float)corpus->doc_count / (float)df) : 0.0F; - } - return corpus->entries[index].token; -} - -static void free_ht_kv(const char *key, void *value, void *userdata) { - (void)userdata; - free((void *)key); - free(value); -} - -void cbm_sem_corpus_free(cbm_sem_corpus_t *corpus) { - if (!corpus) { - return; - } - for (int i = 0; i < corpus->entry_count; i++) { - free(corpus->entries[i].token); - } - free(corpus->entries); - for (int d = 0; d < corpus->doc_count; d++) { - free(corpus->doc_token_ids[d]); - } - free(corpus->doc_token_ids); - free(corpus->doc_token_counts); - if (corpus->token_map) { - cbm_ht_foreach(corpus->token_map, free_ht_kv, NULL); - cbm_ht_free(corpus->token_map); - } - free(corpus); -} - -/* ── Combined scoring ────────────────────────────────────────────── */ - -float cbm_sem_proximity(const char *path_a, const char *path_b) { - if (!path_a || !path_b) { - return CBM_SEM_UNIT_POS; - } - /* Count shared directory components */ - int shared = 0; - int total_a = 0; - int total_b = 0; - const char *a = path_a; - const char *b = path_b; - while (*a && *b && *a == *b) { - if (*a == '/') { - shared++; - } - a++; - b++; - } - for (const char *p = path_a; *p; p++) { - if (*p == '/') { - total_a++; - } - } - for (const char *p = path_b; *p; p++) { - if (*p == '/') { - total_b++; - } - } - int max_total = total_a > total_b ? total_a : total_b; - if (max_total == 0) { - return CBM_SEM_UNIT_POS; - } - float ratio = (float)shared / (float)max_total; - /* Same file = 1.10, same dir = 1.05, distant = 1.00 */ - return CBM_SEM_UNIT_POS + (ratio * CBM_SEM_PROX_MAX_BOOST); -} - -/* Cosine similarity for small float arrays (AST profile). */ -static float small_cosine(const float *a, const float *b, int dims) { - float dot = 0.0F; - float ma = 0.0F; - float mb = 0.0F; - for (int i = 0; i < dims; i++) { - dot += a[i] * b[i]; - ma += a[i] * a[i]; - mb += b[i] * b[i]; - } - float denom = sqrtf(ma) * sqrtf(mb); - return denom < CBM_SEM_DENOM_EPS ? 0.0F : dot / denom; -} - -/* Sparse cosine over two pre-sorted (index, weight) vectors. Returns 0 when - * either side is empty or the magnitude product is below the epsilon guard. */ -static float sparse_tfidf_cosine(const cbm_sem_func_t *a, const cbm_sem_func_t *b) { - if (a->tfidf_len <= 0 || b->tfidf_len <= 0) { - return 0.0F; - } - float dot = 0.0F; - float ma = 0.0F; - float mb = 0.0F; - int ia = 0; - int ib = 0; - while (ia < a->tfidf_len && ib < b->tfidf_len) { - if (a->tfidf_indices[ia] == b->tfidf_indices[ib]) { - dot += a->tfidf_weights[ia] * b->tfidf_weights[ib]; - ia++; - ib++; - } else if (a->tfidf_indices[ia] < b->tfidf_indices[ib]) { - ia++; - } else { - ib++; - } - } - for (int i = 0; i < a->tfidf_len; i++) { - ma += a->tfidf_weights[i] * a->tfidf_weights[i]; - } - for (int i = 0; i < b->tfidf_len; i++) { - mb += b->tfidf_weights[i] * b->tfidf_weights[i]; - } - float denom = sqrtf(ma) * sqrtf(mb); - return denom > CBM_SEM_DENOM_EPS ? (dot / denom) : 0.0F; -} - -float cbm_sem_combined_score(const cbm_sem_func_t *a, const cbm_sem_func_t *b, - const cbm_sem_config_t *cfg) { - if (!a || !b || !cfg) { - return 0.0F; - } - - /* Short-circuit: if MinHash Jaccard is already above the SIMILAR_TO threshold, - * the pass_similarity pipeline already emitted a SIMILAR_TO edge for this pair. - * Returning 0 here avoids flooding top-k with cross-service copy-paste boilerplate - * (logging_middleware, shared push/pull handlers) that SIMILAR_TO already covers, - * freeing the edge budget for true semantic leaps (vocabulary-bridged relations). */ - if (a->has_minhash && b->has_minhash) { - double early_j = cbm_minhash_jaccard((const cbm_minhash_t *)a->minhash, - (const cbm_minhash_t *)b->minhash); - if (early_j >= CBM_MINHASH_JACCARD_THRESHOLD) { - return 0.0F; - } - } - - float score = cfg->w_tfidf * sparse_tfidf_cosine(a, b); - - /* Signal 2: Random Indexing */ - score += cfg->w_ri * cbm_rsq_ip(&a->ri_code, &b->ri_code); - - /* Signal 3: MinHash Jaccard */ - if (a->has_minhash && b->has_minhash) { - double j = cbm_minhash_jaccard((const cbm_minhash_t *)a->minhash, - (const cbm_minhash_t *)b->minhash); - score += cfg->w_minhash * (float)j; - } - - /* Signal 4: API Signatures */ - score += cfg->w_api * cbm_rsq_ip(&a->api_code, &b->api_code); - - /* Signal 5: Type Signatures */ - score += cfg->w_type * cbm_rsq_ip(&a->type_code, &b->type_code); - - /* Signal 7: Decorator Pattern */ - score += cfg->w_decorator * cbm_rsq_ip(&a->deco_code, &b->deco_code); - - /* Signal 8+9+11: Structural profile + data flow + Halstead */ - float sp_score = small_cosine(a->struct_profile, b->struct_profile, CBM_SEM_AST_PROFILE_DIMS); - score += cfg->w_struct_profile * sp_score; - - /* Signal 6: Module proximity (multiplier, not additive). - * Proximity returns [1.0, 1.10] — a same-file/same-dir boost for ranking. - * Clamp the final product to [0, 1] so the output stays within valid - * cosine-similarity range. Without the clamp, a 0.95 base × 1.10 proximity - * would emit a 1.045 score which violates the semantic of "similarity" and - * breaks downstream consumers that expect a normalized value. */ - score *= cbm_sem_proximity(a->file_path, b->file_path); - if (score > CBM_SEM_UNIT_POS) { - score = CBM_SEM_UNIT_POS; - } - if (score < 0.0F) { - score = 0.0F; - } - - return score; -} - -/* ── Graph diffusion ─────────────────────────────────────────────── */ - -void cbm_sem_diffuse(cbm_sem_vec_t *combined, const cbm_sem_vec_t *neighbors, int neighbor_count, - float alpha) { - if (!combined || !neighbors || neighbor_count <= 0) { - return; - } - /* Blend: combined = (1-α) × combined + α × mean(neighbors) */ - cbm_sem_vec_t mean; - memset(&mean, 0, sizeof(mean)); - for (int n = 0; n < neighbor_count; n++) { - for (int i = 0; i < CBM_SEM_DIM; i++) { - mean.v[i] += neighbors[n].v[i]; - } - } - float inv_n = CBM_SEM_UNIT_POS / (float)neighbor_count; - float one_minus_alpha = CBM_SEM_UNIT_POS - alpha; - for (int i = 0; i < CBM_SEM_DIM; i++) { - combined->v[i] = (one_minus_alpha * combined->v[i]) + (alpha * mean.v[i] * inv_n); - } - cbm_sem_normalize(combined); -} diff --git a/src/semantic/semantic.zig b/src/semantic/semantic.zig new file mode 100644 index 00000000..5520cb31 --- /dev/null +++ b/src/semantic/semantic.zig @@ -0,0 +1,1599 @@ +//! semantic.zig — Algorithmic code embeddings: TF-IDF, Random Indexing, +//! API/Type/Decorator signatures, combined scoring, graph diffusion. +//! +//! Implements src/semantic/semantic.h. All signals computed from graph buffer +//! metadata — no source file reads. Uses xxHash for deterministic random +//! vectors; std.hash.XxHash3 produces the same values as the vendored xxHash. +//! +//! Float expressions go through fp.mulAdd wherever the C code wrote a +//! product plus a term in one expression: see fp.zig. + +const std = @import("std"); +const c = @import("c"); +const fp = @import("fp.zig"); +const interop = @import("../foundation/interop.zig"); + +const XxHash3 = std.hash.XxHash3; +const AtomicInt = std.atomic.Value(c_int); + +const Vec = c.cbm_sem_vec_t; +const Func = c.cbm_sem_func_t; +const Str = [*:0]const u8; +const CStr = [*c]const u8; + +// ── Constants ─────────────────────────────────────────────────────── + +const DIM: comptime_int = c.CBM_SEM_DIM; +const SPARSE_NNZE: comptime_int = c.CBM_SEM_SPARSE_NNZE; +const WINDOW: comptime_int = c.CBM_SEM_WINDOW; +const MAX_OCCUR: comptime_int = c.CBM_SEM_MAX_OCCUR; +const AST_PROFILE_DIMS: comptime_int = c.CBM_SEM_AST_PROFILE_DIMS; +const PRETRAINED_TOKEN_COUNT: comptime_int = c.PRETRAINED_TOKEN_COUNT; +const PRETRAINED_DIM: comptime_int = c.PRETRAINED_DIM; +const NOT_FOUND: c_int = c.CBM_NOT_FOUND; + +const TOKEN_BUF_LEN = 128; +const CORPUS_INIT_CAP = 4096; +const DOC_TOKENS_INIT = 64; +const RI_SEED_BASE = 0x52494E44; // "RIND" + +/// Default signal weights for cbm_sem_combined_score. Must sum to ~1.0; +/// proximity is a multiplier applied on top. +const W_TFIDF: f32 = 0.20; +const W_RI: f32 = 0.25; +const W_MINHASH: f32 = 0.10; +const W_API: f32 = 0.15; +const W_TYPE: f32 = 0.10; +const W_DECORATOR: f32 = 0.05; +const W_STRUCT_PROFILE: f32 = 0.10; +const W_DATAFLOW: f32 = 0.05; + +/// Threshold bounds for CBM_SEMANTIC_THRESHOLD env override. +const THRESHOLD_MIN: f32 = 0.0; +const THRESHOLD_MAX: f32 = 1.0; + +/// Epsilon for denominator guards in cosine math. +const DENOM_EPS: f32 = 1e-10; + +/// Int8 quantization bound for vector storage (maps [-1.0, 1.0] to [-127, 127]). +const INT8_MAX: f32 = 127.0; + +/// Unit vector bound (normalized similarity / cosine value limit). +const UNIT_POS: f32 = 1.0; + +/// Proximity boost: same-file gets 1.0 + PROX_MAX_BOOST, distant gets 1.0. +const PROX_MAX_BOOST: f32 = 0.10; + +/// Reflective Random Indexing blend factors for pass2 (context mixing). +const RRI_ALPHA: f32 = 0.3; +const RRI_BETA: f32 = 0.7; + +/// Rounding offset for the int8 quantization. +const IDF_BASE: f32 = 0.5; + +/// Worker tile/chunk sizes for parallel cooccurrence passes. +const WORKER_STACK_CAP = 256; +const TILE_SIZE = 40; +const SEEN_INIT_CAP = 256; +const RESOLVE_CHUNK = 64; +const COOCCUR_CHUNK = 32; +const RRI_TILE = 128; + +/// Mutex-init state machine for the lazy-initialized pretrained token map. +const MTX_STATE_UNINIT = 0; +const MTX_STATE_INITIALIZING = 1; +const MTX_STATE_READY = 2; + +/// Pretrained map ready flag (signals a one-shot atomic transition 0 → 1). +const MAP_READY = 1; + +/// Numeric conversion radix for strtol (base 10 decimal). +const BASE_DECIMAL = 10; + +/// A C int used as an array index. +inline fn ix(i: c_int) usize { + return @intCast(i); +} + +/// &p[i] for a C pointer. Plain indexing gives a `*allowzero T`, which does +/// not coerce to `*T`. +inline fn at(p: anytype, i: usize) @TypeOf(&p[0..1][0]) { + return &p[i .. i + 1][0]; +} + +// ── Configuration ─────────────────────────────────────────────────── + +pub export fn cbm_sem_get_config() c.cbm_sem_config_t { + var cfg: c.cbm_sem_config_t = .{ + .w_tfidf = W_TFIDF, + .w_ri = W_RI, + .w_minhash = W_MINHASH, + .w_api = W_API, + .w_type = W_TYPE, + .w_decorator = W_DECORATOR, + .w_struct_profile = W_STRUCT_PROFILE, + .w_dataflow = W_DATAFLOW, + .threshold = @floatCast(c.CBM_SEM_EDGE_THRESHOLD), + .max_edges = c.CBM_SEM_MAX_EDGES, + }; + const thresh: CStr = c.getenv("CBM_SEMANTIC_THRESHOLD"); + if (thresh != null) { + // strtod reports errors via endptr; reject non-numeric input silently. + var end: [*c]u8 = null; + const parsed = c.strtod(thresh, &end); + if (end != thresh and parsed > THRESHOLD_MIN and parsed <= THRESHOLD_MAX) { + cfg.threshold = @floatCast(parsed); + } + } + return cfg; +} + +pub export fn cbm_sem_is_enabled() bool { + const val: CStr = c.getenv("CBM_SEMANTIC_ENABLED"); + return val != null and val[0] == '1'; +} + +// ── Token extraction ──────────────────────────────────────────────── + +/// True for characters that terminate a token regardless of case. +fn isTokenDelim(ch: u8) bool { + return switch (ch) { + '.', '/', '_', '-', ' ', '(', ')', ',', ':' => true, + else => false, + }; +} + +/// True for a camelCase transition: uppercase letter preceded by a lowercase. +fn isCamelBreak(name: Str, i: usize) bool { + if (i == 0) return false; + const ch = name[i]; + const p = name[i - 1]; + return ch >= 'A' and ch <= 'Z' and p >= 'a' and p <= 'z'; +} + +/// Flush the current buffer as a token into out[]. +fn flushToken(buf: *[TOKEN_BUF_LEN]u8, blen: *usize, out: [*][*c]u8, count: *c_int, max_out: c_int) void { + if (blen.* > 0 and count.* < max_out) { + buf[blen.*] = 0; + out[ix(count.*)] = c.strdup(buf); + count.* += 1; + } + blen.* = 0; +} + +/// Cross-language abbreviation table — covers Go, Python, JS/TS, Rust, +/// Java, C/C++, Ruby, PHP, Kotlin, Swift, Scala, C#, and common patterns. +const abbrevs = [_][2][:0]const u8{ + // Error/exception handling + .{ "err", "error" }, + .{ "exc", "exception" }, + .{ "ex", "exception" }, + // Context/config + .{ "ctx", "context" }, + .{ "cfg", "config" }, + .{ "conf", "configuration" }, + .{ "env", "environment" }, + .{ "opt", "option" }, + .{ "opts", "options" }, + // Request/response (HTTP, RPC) + .{ "req", "request" }, + .{ "res", "response" }, + .{ "resp", "response" }, + .{ "rsp", "response" }, + .{ "hdr", "header" }, + .{ "hdrs", "headers" }, + // Strings/formatting + .{ "str", "string" }, + .{ "fmt", "format" }, + .{ "msg", "message" }, + .{ "txt", "text" }, + .{ "lbl", "label" }, + .{ "desc", "description" }, + // Data structures + .{ "buf", "buffer" }, + .{ "arr", "array" }, + .{ "vec", "vector" }, + .{ "lst", "list" }, + .{ "dict", "dictionary" }, + .{ "tbl", "table" }, + .{ "stk", "stack" }, + .{ "que", "queue" }, + // Functions/callbacks + .{ "fn", "function" }, + .{ "func", "function" }, + .{ "cb", "callback" }, + .{ "proc", "procedure" }, + .{ "ctor", "constructor" }, + .{ "dtor", "destructor" }, + // Database/storage + .{ "db", "database" }, + .{ "col", "column" }, + .{ "tbl", "table" }, + .{ "stmt", "statement" }, + .{ "txn", "transaction" }, + .{ "trx", "transaction" }, + .{ "repo", "repository" }, + // Auth/security + .{ "auth", "authentication" }, + .{ "authz", "authorization" }, + .{ "perm", "permission" }, + .{ "cred", "credential" }, + .{ "tok", "token" }, + .{ "pwd", "password" }, + // Values/types + .{ "val", "value" }, + .{ "num", "number" }, + .{ "int", "integer" }, + .{ "bool", "boolean" }, + .{ "flt", "float" }, + .{ "dbl", "double" }, + // Indexing/iteration + .{ "idx", "index" }, + .{ "iter", "iterator" }, + .{ "elem", "element" }, + .{ "cnt", "count" }, + .{ "len", "length" }, + .{ "sz", "size" }, + .{ "pos", "position" }, + .{ "off", "offset" }, + .{ "cap", "capacity" }, + // Lifecycle + .{ "init", "initialize" }, + .{ "deinit", "deinitialize" }, + .{ "alloc", "allocate" }, + .{ "dealloc", "deallocate" }, + .{ "del", "delete" }, + .{ "rm", "remove" }, + // Implementation/interface + .{ "impl", "implementation" }, + .{ "iface", "interface" }, + .{ "abs", "abstract" }, + .{ "decl", "declaration" }, + // Parameters/attributes + .{ "param", "parameter" }, + .{ "arg", "argument" }, + .{ "attr", "attribute" }, + .{ "prop", "property" }, + .{ "ret", "return" }, + // Source/destination + .{ "src", "source" }, + .{ "dst", "destination" }, + .{ "tgt", "target" }, + .{ "orig", "original" }, + .{ "prev", "previous" }, + .{ "cur", "current" }, + .{ "tmp", "temporary" }, + .{ "temp", "temporary" }, + // Networking/IO + .{ "conn", "connection" }, + .{ "sess", "session" }, + .{ "sock", "socket" }, + .{ "addr", "address" }, + .{ "url", "uniform" }, + .{ "srv", "server" }, + .{ "cli", "client" }, + .{ "svc", "service" }, + .{ "ep", "endpoint" }, + // Management + .{ "mgr", "manager" }, + .{ "ctrl", "controller" }, + .{ "hdlr", "handler" }, + .{ "sched", "scheduler" }, + .{ "disp", "dispatcher" }, + .{ "reg", "registry" }, + // Async/concurrent + .{ "chan", "channel" }, + .{ "sem", "semaphore" }, + .{ "mtx", "mutex" }, + .{ "wg", "waitgroup" }, + .{ "sig", "signal" }, + .{ "evt", "event" }, + .{ "sub", "subscriber" }, + .{ "pub", "publisher" }, + // Testing + .{ "spec", "specification" }, + .{ "mock", "mock" }, + .{ "stub", "stub" }, + .{ "assert", "assertion" }, + // Logging/monitoring + .{ "log", "logging" }, + .{ "lvl", "level" }, + .{ "dbg", "debug" }, + .{ "wrn", "warning" }, + .{ "inf", "info" }, + // Time + .{ "ts", "timestamp" }, + .{ "dur", "duration" }, + .{ "ttl", "timetolive" }, + // Miscellaneous + .{ "ver", "version" }, + .{ "ns", "namespace" }, + .{ "pkg", "package" }, + .{ "mod", "module" }, + .{ "lib", "library" }, + .{ "dep", "dependency" }, + .{ "ref", "reference" }, + .{ "ptr", "pointer" }, + .{ "obj", "object" }, + .{ "doc", "document" }, + .{ "cmd", "command" }, + .{ "ops", "operations" }, + .{ "util", "utility" }, + .{ "hlp", "helper" }, + .{ "ext", "extension" }, +}; + +pub export fn cbm_sem_tokenize(name_c: ?Str, out_c: ?[*][*c]u8, max_out: c_int) c_int { + const name = name_c orelse return 0; + const out = out_c orelse return 0; + if (max_out <= 0) return 0; + var count: c_int = 0; + var buf: [TOKEN_BUF_LEN]u8 = undefined; + var blen: usize = 0; + + var i: usize = 0; + while (name[i] != 0 and count < max_out) : (i += 1) { + const ch = name[i]; + const split = isTokenDelim(ch); + const camel = isCamelBreak(name, i); + if (split or camel) { + flushToken(&buf, &blen, out, &count, max_out); + if (split) continue; + } + if (blen < TOKEN_BUF_LEN - 1 and std.ascii.isAlphanumeric(ch)) { + buf[blen] = std.ascii.toLower(ch); + blen += 1; + } + } + flushToken(&buf, &blen, out, &count, max_out); + + // Abbreviation expansion: add expanded forms for common code abbreviations. + // "err" → also add "error", "ctx" → "context", etc. + const orig_count = count; + var t: c_int = 0; + while (t < orig_count and count < max_out) : (t += 1) { + const tok = std.mem.span(out[ix(t)]); + for (abbrevs) |ab| { + if (std.mem.eql(u8, tok, ab[0])) { + out[ix(count)] = c.strdup(ab[1]); + count += 1; + break; + } + } + } + + return count; +} + +// ── Dense vector operations ───────────────────────────────────────── + +pub export fn cbm_sem_cosine(a_c: ?*const Vec, b_c: ?*const Vec) f32 { + const a = a_c orelse return 0.0; + const b = b_c orelse return 0.0; + var dot: f32 = 0.0; + var mag_a: f32 = 0.0; + var mag_b: f32 = 0.0; + for (a.v, b.v) |x, y| { + dot = fp.mulAdd(f32, x, y, dot); + mag_a = fp.mulAdd(f32, x, x, mag_a); + mag_b = fp.mulAdd(f32, y, y, mag_b); + } + const denom = @sqrt(mag_a) * @sqrt(mag_b); + if (denom < DENOM_EPS) return 0.0; + return dot / denom; +} + +/// Pretrained token lookup table — built lazily on first use. +var g_pretrained_map: ?*c.CBMHashTable = null; +var g_pretrained_ready: AtomicInt = .init(0); +var g_pretrained_mtx: c.cbm_mutex_t = undefined; +var g_pretrained_mtx_init: AtomicInt = .init(MTX_STATE_UNINIT); + +/// Thread-safe lazy init of the pretrained token lookup map. +/// Uses double-checked locking: fast path reads an atomic flag. +fn ensurePretrainedMap() void { + if (g_pretrained_ready.load(.acquire) != 0) return; + // First-time init of the mutex itself (also needs to be thread-safe) + if (g_pretrained_mtx_init.cmpxchgStrong( + MTX_STATE_UNINIT, + MTX_STATE_INITIALIZING, + .acq_rel, + .acquire, + ) == null) { + c.cbm_mutex_init(&g_pretrained_mtx); + g_pretrained_mtx_init.store(MTX_STATE_READY, .release); + } else { + // Spin until another thread finishes initializing the mutex + while (g_pretrained_mtx_init.load(.acquire) != MTX_STATE_READY) { + // brief spin + } + } + c.cbm_mutex_lock(&g_pretrained_mtx); + if (g_pretrained_ready.load(.acquire) == 0) { + g_pretrained_map = c.cbm_ht_create(PRETRAINED_TOKEN_COUNT); + var idx_buf: [c.CBM_SZ_16]u8 = undefined; + for (0..PRETRAINED_TOKEN_COUNT) |i| { + const tok: CStr = c.PRETRAINED_TOKENS[i]; + if (tok != null and tok[0] != 0) { + _ = c.snprintf(&idx_buf, idx_buf.len, "%d", @as(c_int, @intCast(i))); + _ = c.cbm_ht_set(g_pretrained_map, c.strdup(tok), c.strdup(&idx_buf)); + } + } + g_pretrained_ready.store(MAP_READY, .release); + } + c.cbm_mutex_unlock(&g_pretrained_mtx); +} + +pub export fn cbm_sem_ensure_ready() void { + ensurePretrainedMap(); +} + +/// The pretrained int8 vector for a token, or null when the token is not in +/// the pretrained vocabulary. The map must be initialized. +fn pretrainedVec(token: CStr) ?[*]const i8 { + const idx_str: CStr = @ptrCast(c.cbm_ht_get(g_pretrained_map, token)); + if (idx_str == null) return null; + var end: [*c]u8 = null; + const idx = c.strtol(idx_str, &end, BASE_DECIMAL); + if (end != idx_str and idx >= 0 and idx < PRETRAINED_TOKEN_COUNT) { + return @ptrCast(c.pretrained_vec_at(@intCast(idx))); + } + return null; +} + +pub export fn cbm_sem_random_index(token_c: ?Str, out: *Vec) void { + @memset(&out.v, 0.0); + const token = token_c orelse return; + + // Try pretrained nomic-embed-code vector first (768d, distilled from 7B). + ensurePretrainedMap(); + if (pretrainedVec(token)) |pvec| { + for (out.v[0..@min(DIM, PRETRAINED_DIM)], pvec[0..@min(DIM, PRETRAINED_DIM)]) |*o, p| { + o.* = @as(f32, @floatFromInt(p)) / INT8_MAX; + } + return; + } + + // Fallback: sparse random vector for tokens not in pretrained vocab. + const seed = XxHash3.hash(0, std.mem.span(token)); + for (0..SPARSE_NNZE) |i| { + const h = XxHash3.hash(seed +% RI_SEED_BASE, &std.mem.toBytes(@as(c_int, @intCast(i)))); + const pos: usize = @intCast(h % DIM); + const sign: f32 = if (h & 1 != 0) UNIT_POS else -UNIT_POS; + out.v[pos] += sign; + } +} + +pub export fn cbm_sem_normalize(v_c: ?*Vec) void { + const v = v_c orelse return; + var mag: f32 = 0.0; + for (v.v) |x| { + mag = fp.mulAdd(f32, x, x, mag); + } + mag = @sqrt(mag); + if (mag < DENOM_EPS) return; + const inv = UNIT_POS / mag; + for (&v.v) |*x| { + x.* *= inv; + } +} + +pub export fn cbm_sem_vec_add_scaled(dst_c: ?*Vec, src_c: ?*const Vec, scale: f32) void { + const dst = dst_c orelse return; + const src = src_c orelse return; + for (&dst.v, src.v) |*d, s| { + d.* = fp.mulAdd(f32, scale, s, d.*); + } +} + +// ── Corpus (IDF + Random Indexing enrichment) ─────────────────────── + +const CorpusEntry = extern struct { + token: [*c]u8, + doc_freq: c_int, + enriched_vec: Vec, // context-enriched via co-occurrence +}; + +const Corpus = extern struct { + token_map: ?*c.CBMHashTable, // token → index into entries[] + entries: [*c]CorpusEntry, + entry_count: c_int, + entry_cap: c_int, + doc_count: c_int, + finalized: bool, + + // Per-document token lists for co-occurrence pass + doc_token_ids: [*c][*c]c_int, + doc_token_counts: [*c]c_int, + doc_cap: c_int, +}; + +/// Parse a decimal index string via strtol, returning CBM_NOT_FOUND on parse +/// failure. Centralizes the atoi-replacement pattern for token_map lookups. +fn parseTokenIndex(idx_str: CStr) c_int { + if (idx_str == null) return NOT_FOUND; + var end: [*c]u8 = null; + const parsed = c.strtol(idx_str, &end, BASE_DECIMAL); + return if (end != idx_str) @truncate(parsed) else NOT_FOUND; +} + +fn corpusGetOrAdd(cp: *Corpus, token: CStr) c_int { + var idx_buf: [c.CBM_SZ_16]u8 = undefined; + const existing: CStr = @ptrCast(c.cbm_ht_get(cp.token_map, token)); + if (existing != null) { + return parseTokenIndex(existing); + } + if (cp.entry_count >= cp.entry_cap) { + const new_cap: c_int = if (cp.entry_cap < CORPUS_INIT_CAP) CORPUS_INIT_CAP else cp.entry_cap * 2; + const grown = interop.reallocN(CorpusEntry, cp.entries, ix(new_cap)); + if (grown == null) return NOT_FOUND; + cp.entries = grown; + cp.entry_cap = new_cap; + } + const idx = cp.entry_count; + cp.entry_count += 1; + const entry = at(cp.entries, ix(idx)); + entry.token = c.strdup(token); + entry.doc_freq = 0; + @memset(&entry.enriched_vec.v, 0.0); + _ = c.snprintf(&idx_buf, idx_buf.len, "%d", idx); + _ = c.cbm_ht_set(cp.token_map, c.strdup(token), c.strdup(&idx_buf)); + return idx; +} + +pub export fn cbm_sem_corpus_new() ?*Corpus { + const cp: ?*Corpus = interop.callocN(Corpus, 1); + if (cp) |corpus| { + corpus.token_map = c.cbm_ht_create(CORPUS_INIT_CAP); + } + return cp; +} + +pub export fn cbm_sem_corpus_add_doc(corpus_c: ?*Corpus, tokens_c: ?[*]CStr, count: c_int) void { + const corpus = corpus_c orelse return; + const tokens = tokens_c orelse return; + if (count <= 0) return; + // Track document for co-occurrence pass + if (corpus.doc_count >= corpus.doc_cap) { + const new_cap: c_int = if (corpus.doc_cap < DOC_TOKENS_INIT) DOC_TOKENS_INIT else corpus.doc_cap * 2; + const grown_ids = interop.reallocN([*c]c_int, corpus.doc_token_ids, ix(new_cap)); + const grown_counts = interop.reallocN(c_int, corpus.doc_token_counts, ix(new_cap)); + if (grown_ids == null or grown_counts == null) { + c.free(@ptrCast(grown_ids)); + c.free(@ptrCast(grown_counts)); + return; + } + corpus.doc_token_ids = grown_ids; + corpus.doc_token_counts = grown_counts; + corpus.doc_cap = new_cap; + } + const doc_idx = ix(corpus.doc_count); + corpus.doc_count += 1; + corpus.doc_token_ids[doc_idx] = interop.mallocN(c_int, ix(count)); + corpus.doc_token_counts[doc_idx] = count; + + // Per-doc unique set for IDF + const seen = interop.callocN(c_int, ix(corpus.entry_cap) + ix(count) + CORPUS_INIT_CAP); + var seen_count: usize = 0; + + for (0..ix(count)) |i| { + const tid = corpusGetOrAdd(corpus, tokens[i]); + corpus.doc_token_ids[doc_idx][i] = tid; + if (tid < 0) continue; + // Check uniqueness for IDF (simple linear scan — tokens per doc is small) + var is_new = true; + for (0..seen_count) |j| { + if (seen[j] == tid) { + is_new = false; + break; + } + } + if (is_new) { + seen[seen_count] = tid; + seen_count += 1; + corpus.entries[ix(tid)].doc_freq += 1; + } + } + c.free(@ptrCast(seen)); +} + +// ── Parallel corpus batch build ───────────────────────────────────── +// Strategy: +// Phase A (SEQUENTIAL): Scan all documents once to build the global +// token_map (inserts unique tokens, assigns global IDs). This is +// inherently sequential (hash table mutation), but much faster than +// the current per-doc add_doc because we avoid the per-doc malloc of +// the `seen` array and per-doc bookkeeping. +// Phase B (PARALLEL): Each worker processes a chunk of docs, translates +// tokens → global IDs via read-only token_map lookups, fills +// doc_token_ids[d], and accumulates doc_freq contributions via atomics. + +const BatchResolveCtx = struct { + corpus: *Corpus, + all_tokens: [*][*c]u8, + token_counts: [*]const c_int, + max_tokens: c_int, + doc_count: c_int, + doc_freq_atomic: [*]AtomicInt, // per-entry atomic counter (entry_count long) + next_idx: AtomicInt, +}; + +/// Resolve one document: look up each token's global ID, fill the corpus +/// doc_token_ids[d], and bump the per-token doc_freq counter atomically. The +/// caller is responsible for ensuring `seen` has capacity for `count` ints +/// before calling (the worker grows its per-thread scratch buffer). +fn batchResolveOneDoc(bc: *BatchResolveCtx, doc_index: usize, seen: [*c]c_int) void { + const count = bc.token_counts[doc_index]; + if (count <= 0) { + bc.corpus.doc_token_ids[doc_index] = null; + bc.corpus.doc_token_counts[doc_index] = 0; + return; + } + const ids = interop.mallocN(c_int, ix(count)); + bc.corpus.doc_token_ids[doc_index] = ids; + bc.corpus.doc_token_counts[doc_index] = count; + + var seen_count: usize = 0; + const tokens = bc.all_tokens + doc_index * ix(bc.max_tokens); + for (0..ix(count)) |i| { + const tid = parseTokenIndex(@ptrCast(c.cbm_ht_get(bc.corpus.token_map, tokens[i]))); + ids[i] = tid; + if (tid < 0) continue; + // Unique-per-doc check for IDF + var is_new = true; + for (0..seen_count) |j| { + if (seen[j] == tid) { + is_new = false; + break; + } + } + if (is_new) { + seen[seen_count] = tid; + seen_count += 1; + _ = bc.doc_freq_atomic[ix(tid)].fetchAdd(1, .monotonic); + } + } +} + +fn batchResolveWorker(worker_id: c_int, ctx_ptr: ?*anyopaque) callconv(.c) void { + _ = worker_id; + const bc: *BatchResolveCtx = @ptrCast(@alignCast(ctx_ptr)); + // Per-worker scratch for unique-per-doc tracking + var local_seen_cap: c_int = SEEN_INIT_CAP; + var seen = interop.mallocN(c_int, ix(local_seen_cap)); + if (seen == null) return; + + while (true) { + const start = bc.next_idx.fetchAdd(RESOLVE_CHUNK, .monotonic); + if (start >= bc.doc_count) break; + var end = start + RESOLVE_CHUNK; + if (end > bc.doc_count) end = bc.doc_count; + for (ix(start)..ix(end)) |d| { + const count = bc.token_counts[d]; + if (count > local_seen_cap) { + const grown = interop.reallocN(c_int, seen, ix(count)); + if (grown == null) continue; + seen = grown; + local_seen_cap = count; + } + batchResolveOneDoc(bc, d, seen); + } + } + c.free(@ptrCast(seen)); +} + +pub export fn cbm_sem_corpus_add_docs_batch( + corpus_c: ?*Corpus, + all_tokens_c: ?[*][*c]u8, + token_counts_c: ?[*]const c_int, + doc_count: c_int, + max_tokens_per_doc: c_int, +) void { + const corpus = corpus_c orelse return; + const all_tokens = all_tokens_c orelse return; + const token_counts = token_counts_c orelse return; + if (doc_count <= 0) return; + + // Phase A (SEQUENTIAL): Build token_map and allocate doc arrays. + // Hash table mutation can't be parallelized; strdup+insert is the cost. + if (corpus.doc_cap < corpus.doc_count + doc_count) { + const new_cap = corpus.doc_count + doc_count; + const grown_ids = interop.reallocN([*c]c_int, corpus.doc_token_ids, ix(new_cap)); + const grown_counts = interop.reallocN(c_int, corpus.doc_token_counts, ix(new_cap)); + if (grown_ids == null or grown_counts == null) { + c.free(@ptrCast(grown_ids)); + c.free(@ptrCast(grown_counts)); + return; + } + corpus.doc_token_ids = grown_ids; + corpus.doc_token_counts = grown_counts; + corpus.doc_cap = new_cap; + } + const base_doc = corpus.doc_count; + corpus.doc_count += doc_count; + + for (0..ix(doc_count)) |d| { + const count = token_counts[d]; + const tokens = all_tokens + d * ix(max_tokens_per_doc); + var i: c_int = 0; + while (i < count) : (i += 1) { + // Inserts token into token_map if new; we discard return here — + // Phase B will re-lookup in read-only mode to get the ID. + _ = corpusGetOrAdd(corpus, tokens[ix(i)]); + } + } + + // Phase B (PARALLEL): Resolve tokens → IDs and count doc_freq per entry. + // token_map is now read-only; each worker owns its doc range (no writes + // to shared state except atomic doc_freq counters). + const doc_freq_atomic: ?[*]AtomicInt = interop.callocN(AtomicInt, ix(corpus.entry_count)); + if (doc_freq_atomic == null) { + // OOM fallback: sequential path. Roll back doc_count first since + // add_doc increments it itself. + corpus.doc_count = base_doc; + for (0..ix(doc_count)) |d| { + const count = token_counts[d]; + const tokens = all_tokens + d * ix(max_tokens_per_doc); + cbm_sem_corpus_add_doc(corpus, @ptrCast(tokens), count); + } + return; + } + + const worker_count = c.cbm_default_worker_count(false); + var bc: BatchResolveCtx = .{ + .corpus = corpus, + .all_tokens = all_tokens, + .token_counts = token_counts, + .max_tokens = max_tokens_per_doc, + .doc_count = doc_count, + .doc_freq_atomic = doc_freq_atomic.?, + .next_idx = .init(0), + }; + // Temporarily re-base doc arrays so workers write to base_doc..base_doc+doc_count + corpus.doc_token_ids += ix(base_doc); + corpus.doc_token_counts += ix(base_doc); + const opts: c.cbm_parallel_for_opts_t = .{ .max_workers = worker_count, .force_pthreads = false }; + c.cbm_parallel_for(worker_count, batchResolveWorker, &bc, opts); + corpus.doc_token_ids -= ix(base_doc); + corpus.doc_token_counts -= ix(base_doc); + + // Phase C (SEQUENTIAL reduce): atomic counters → entries[].doc_freq + for (0..ix(corpus.entry_count)) |i| { + corpus.entries[i].doc_freq += doc_freq_atomic.?[i].load(.monotonic); + } + c.free(@ptrCast(doc_freq_atomic)); +} + +// ── Parallel corpus_finalize ──────────────────────────────────────── +// Strategy: +// 1. Precompute base RI vectors into a shared array (eliminates ~333M +// redundant cbm_sem_random_index calls on kernel-scale corpora). +// 2. Co-occurrence passes: partition TARGET tokens across workers so each +// worker writes to a disjoint range of enriched_vec (zero contention). +// Each worker still scans all documents but only accumulates for targets +// in its range. Inner vector add is the parallelized work. +// 3. Normalize/blend loops are trivially parallel per-entry. + +/// Reverse index: for each token id, a list of (doc_id, position_in_doc) pairs. +/// Built once, reused for both cooccur passes. Eliminates the O(num_chunks × doc_count) +/// redundant outer scan in the old algorithm. +const CooccurPos = extern struct { + doc_id: i32, + pos: i32, +}; + +const ReverseIndex = extern struct { + offsets: [*c]c_int, // offsets[entry_count + 1], prefix sum of occurrences + flat: [*c]CooccurPos, // flat array of positions, total = offsets[entry_count] +}; + +/// Tagged source vector: sparse (~30% of tokens, 8 inline nonzeros) or dense int8 +/// reference into PRETRAINED_VECTOR_BLOB (no copy). Dense path converts int8→float +/// on the fly in the hot loop. Tested with both int8 and float32 blob storage +/// formats — cooccur passes are memory-bandwidth-bound, so the int8 format (4x +/// less source traffic) is equivalent in wall time despite the conversion cost, +/// while saving 90 MB of binary size. +const SrcEntry = extern struct { + is_sparse: u8, // 1 = sparse path, 0 = dense int8 reference + nnz: u8, // number of nonzeros used in sparse path + _pad: u16, + indices: [SPARSE_NNZE]u16, // 8 * 2 = 16 bytes + values: [SPARSE_NNZE]f32, // 8 * 4 = 32 bytes + dense_int8: ?[*]const i8, // points into PRETRAINED_VECTOR_BLOB +}; + +/// Initialize a target vector from a sparse/dense source. +inline fn semTargetInitFromSrc(dst: *Vec, src: *const SrcEntry) void { + @memset(&dst.v, 0.0); + if (src.is_sparse != 0) { + for (src.indices[0..src.nnz], src.values[0..src.nnz]) |idx, val| { + dst.v[idx] = val; + } + } else { + const s = src.dense_int8.?; + const inv127 = UNIT_POS / INT8_MAX; + for (&dst.v, s[0..DIM]) |*d, x| { + d.* = inv127 * @as(f32, @floatFromInt(x)); + } + } +} + +/// Add weighted source into target. +/// Sparse path: ~8 operations, ~48 bytes source memory traffic. +/// Dense path: 768 mul-adds with int8→float conversion, ~768 bytes traffic. +inline fn semVecAddSrcScaled(dst: *Vec, src: *const SrcEntry, scale: f32) void { + if (src.is_sparse != 0) { + for (src.indices[0..src.nnz], src.values[0..src.nnz]) |idx, val| { + dst.v[idx] = fp.mulAdd(f32, scale, val, dst.v[idx]); + } + } else { + const s = src.dense_int8.?; + const mul = scale * (UNIT_POS / INT8_MAX); + for (&dst.v, s[0..DIM]) |*d, x| { + d.* = fp.mulAdd(f32, mul, @floatFromInt(x), d.*); + } + } +} + +/// Pass 1 context: uses sparse/int8 tagged sources (most memory-efficient). +const CooccurSparseCtx = struct { + entries: [*c]CorpusEntry, + src_entries: [*c]const SrcEntry, // sparse or int8-dense per token + doc_token_ids: [*c][*c]c_int, + doc_token_counts: [*c]const c_int, + rev: *const ReverseIndex, + doc_count: c_int, + entry_count: c_int, + next_chunk: AtomicInt, + num_chunks: c_int, + chunk_size: c_int, + + // Cache-blocked tiling parameters + tile_size: c_int, // targets per L2-resident tile +}; + +/// Accumulate co-occurrence context for a single target token into `target`. +/// Reads neighbors within ±CBM_SEM_WINDOW positions across all documents that +/// reference this token id via the reverse index. +fn cooccurSparseOneTarget(cc: *CooccurSparseCtx, tid: c_int, target: *Vec) void { + const occ_start = cc.rev.offsets[ix(tid)]; + const occ_end = cc.rev.offsets[ix(tid) + 1]; + // Frequent-token subsampling: stride over occurrences when a token is very + // common so the enriched vector's direction is preserved (it's normalized + // after) while bounding work to ~CBM_SEM_MAX_OCCUR samples. See semantic.h. + var occ_step: c_int = 1; + if (occ_end - occ_start > MAX_OCCUR) { + occ_step = @divTrunc(occ_end - occ_start, MAX_OCCUR); + } + var p = occ_start; + while (p < occ_end) : (p += occ_step) { + const d = cc.rev.flat[ix(p)].doc_id; + const i = cc.rev.flat[ix(p)].pos; + const ids = cc.doc_token_ids[ix(d)]; + const len = cc.doc_token_counts[ix(d)]; + var w: c_int = -WINDOW; + while (w <= WINDOW) : (w += 1) { + if (w == 0) continue; + const j = i + w; + if (j < 0 or j >= len) continue; + const nid = ids[ix(j)]; + if (nid < 0) continue; + const weight = UNIT_POS / @as(f32, @floatFromInt(@abs(w))); + semVecAddSrcScaled(target, at(cc.src_entries, ix(nid)), weight); + } + } +} + +fn cooccurWorkerSparse(worker_id: c_int, ctx_ptr: ?*anyopaque) callconv(.c) void { + _ = worker_id; + const cc: *CooccurSparseCtx = @ptrCast(@alignCast(ctx_ptr)); + while (true) { + const ci = cc.next_chunk.fetchAdd(1, .monotonic); + if (ci >= cc.num_chunks) break; + const chunk_start = ci * cc.chunk_size; + var chunk_end = chunk_start + cc.chunk_size; + if (chunk_end > cc.entry_count) chunk_end = cc.entry_count; + + // Cache-blocked target tiling: process tile_size targets at a time so + // their vectors stay resident in L2 cache during their accumulation. + var tile_start = chunk_start; + while (tile_start < chunk_end) : (tile_start += cc.tile_size) { + var tile_end = tile_start + cc.tile_size; + if (tile_end > chunk_end) tile_end = chunk_end; + var tid = tile_start; + while (tid < tile_end) : (tid += 1) { + const target = &at(cc.entries, ix(tid)).enriched_vec; + semTargetInitFromSrc(target, at(cc.src_entries, ix(tid))); + cooccurSparseOneTarget(cc, tid, target); + } + } + } +} + +/// Pass 2 (RRI) context: uses int8-quantized pass1 vectors as source. +/// Pass1 outputs are dense float32 post-normalization, values in [-1,1]. +/// We quantize to int8 once (×127) to cut source memory traffic 4x. +const CooccurInt8Ctx = struct { + entries: [*c]CorpusEntry, + pass1_q: [*]const i8, // [entry_count × CBM_SEM_DIM] int8 quantized pass1 + doc_token_ids: [*c][*c]c_int, + doc_token_counts: [*c]const c_int, + rev: *const ReverseIndex, + doc_count: c_int, + entry_count: c_int, + next_chunk: AtomicInt, + num_chunks: c_int, + chunk_size: c_int, + tile_size: c_int, +}; + +inline fn semVecAddInt8Scaled(dst: *Vec, src: *const [DIM]i8, scale: f32) void { + const mul = scale * (UNIT_POS / INT8_MAX); + for (&dst.v, src) |*d, x| { + d.* = fp.mulAdd(f32, mul, @floatFromInt(x), d.*); + } +} + +/// RRI pass 2 accumulator for a single target token, reading int8-quantized +/// pass1 vectors as the source. +fn cooccurInt8OneTarget(cc: *CooccurInt8Ctx, tid: c_int, target: *Vec) void { + const occ_start = cc.rev.offsets[ix(tid)]; + const occ_end = cc.rev.offsets[ix(tid) + 1]; + // Frequent-token subsampling (see cooccurSparseOneTarget / semantic.h). + var occ_step: c_int = 1; + if (occ_end - occ_start > MAX_OCCUR) { + occ_step = @divTrunc(occ_end - occ_start, MAX_OCCUR); + } + var p = occ_start; + while (p < occ_end) : (p += occ_step) { + const d = cc.rev.flat[ix(p)].doc_id; + const i = cc.rev.flat[ix(p)].pos; + const ids = cc.doc_token_ids[ix(d)]; + const len = cc.doc_token_counts[ix(d)]; + var w: c_int = -WINDOW; + while (w <= WINDOW) : (w += 1) { + if (w == 0) continue; + const j = i + w; + if (j < 0 or j >= len) continue; + const nid = ids[ix(j)]; + if (nid < 0) continue; + const weight = UNIT_POS / @as(f32, @floatFromInt(@abs(w))); + semVecAddInt8Scaled(target, cc.pass1_q[ix(nid) * DIM ..][0..DIM], weight); + } + } +} + +fn cooccurWorkerInt8(worker_id: c_int, ctx_ptr: ?*anyopaque) callconv(.c) void { + _ = worker_id; + const cc: *CooccurInt8Ctx = @ptrCast(@alignCast(ctx_ptr)); + while (true) { + const ci = cc.next_chunk.fetchAdd(1, .monotonic); + if (ci >= cc.num_chunks) break; + const chunk_start = ci * cc.chunk_size; + var chunk_end = chunk_start + cc.chunk_size; + if (chunk_end > cc.entry_count) chunk_end = cc.entry_count; + + var tile_start = chunk_start; + while (tile_start < chunk_end) : (tile_start += cc.tile_size) { + var tile_end = tile_start + cc.tile_size; + if (tile_end > chunk_end) tile_end = chunk_end; + var tid = tile_start; + while (tid < tile_end) : (tid += 1) { + // RRI pass 2 starts from zero (no self-init) + const target = &at(cc.entries, ix(tid)).enriched_vec; + @memset(&target.v, 0.0); + cooccurInt8OneTarget(cc, tid, target); + } + } + } +} + +const NormCtx = struct { + entries: [*c]CorpusEntry, + entry_count: c_int, + next_idx: AtomicInt, +}; + +fn normalizeWorker(worker_id: c_int, ctx_ptr: ?*anyopaque) callconv(.c) void { + _ = worker_id; + const nc: *NormCtx = @ptrCast(@alignCast(ctx_ptr)); + while (true) { + const start = nc.next_idx.fetchAdd(WORKER_STACK_CAP, .monotonic); + if (start >= nc.entry_count) break; + var end = start + WORKER_STACK_CAP; + if (end > nc.entry_count) end = nc.entry_count; + for (ix(start)..ix(end)) |i| { + cbm_sem_normalize(&at(nc.entries, i).enriched_vec); + } + } +} + +const BlendCtx = struct { + entries: [*c]CorpusEntry, + pass1: [*]const Vec, + entry_count: c_int, + next_idx: AtomicInt, +}; + +fn blendWorker(worker_id: c_int, ctx_ptr: ?*anyopaque) callconv(.c) void { + _ = worker_id; + const bc: *BlendCtx = @ptrCast(@alignCast(ctx_ptr)); + while (true) { + const start = bc.next_idx.fetchAdd(WORKER_STACK_CAP, .monotonic); + if (start >= bc.entry_count) break; + var end = start + WORKER_STACK_CAP; + if (end > bc.entry_count) end = bc.entry_count; + for (ix(start)..ix(end)) |i| { + const vec = &at(bc.entries, i).enriched_vec; + cbm_sem_normalize(vec); + for (&vec.v, bc.pass1[i].v) |*e, p1| { + e.* = fp.mulAdd(f32, RRI_BETA, p1, RRI_ALPHA * e.*); + } + } + } +} + +const SrcBuildCtx = struct { + entries: [*c]CorpusEntry, + src_entries: [*c]SrcEntry, + entry_count: c_int, + next_idx: AtomicInt, +}; + +/// Build one src_entry for a token: dense int8 reference if in nomic vocab, +/// sparse inline representation otherwise. Collisions in the sparse hash are +/// merged and zeros filtered so the final representation is exactly the same +/// mathematical vector that the old dense path produced. +fn buildSrcEntry(token: CStr, out: *SrcEntry) void { + out.* = std.mem.zeroes(SrcEntry); + if (token == null) { + out.is_sparse = 1; + out.nnz = 0; + return; + } + // Dense path: direct int8 pointer into pretrained blob (zero-copy). + if (pretrainedVec(token)) |pvec| { + out.is_sparse = 0; + out.dense_int8 = pvec; + return; + } + // Sparse path: compute 8 hash positions with collision merging. + out.is_sparse = 1; + var tmp_idx: [SPARSE_NNZE]u16 = undefined; + var tmp_val: [SPARSE_NNZE]f32 = undefined; + var count: usize = 0; + const seed = XxHash3.hash(0, std.mem.span(token)); + for (0..SPARSE_NNZE) |i| { + const h = XxHash3.hash(seed +% RI_SEED_BASE, &std.mem.toBytes(@as(c_int, @intCast(i)))); + const pos: u16 = @intCast(h % DIM); + const sign: f32 = if (h & 1 != 0) UNIT_POS else -UNIT_POS; + // Merge collisions + var found: ?usize = null; + for (tmp_idx[0..count], 0..) |idx, j| { + if (idx == pos) { + found = j; + break; + } + } + if (found) |j| { + tmp_val[j] += sign; + } else { + tmp_idx[count] = pos; + tmp_val[count] = sign; + count += 1; + } + } + // Filter zeros + var nnz: u8 = 0; + for (tmp_idx[0..count], tmp_val[0..count]) |idx, val| { + if (val != 0.0) { + out.indices[nnz] = idx; + out.values[nnz] = val; + nnz += 1; + } + } + out.nnz = nnz; +} + +fn srcBuildWorker(worker_id: c_int, ctx_ptr: ?*anyopaque) callconv(.c) void { + _ = worker_id; + const sc: *SrcBuildCtx = @ptrCast(@alignCast(ctx_ptr)); + while (true) { + const start = sc.next_idx.fetchAdd(WORKER_STACK_CAP, .monotonic); + if (start >= sc.entry_count) break; + var end = start + WORKER_STACK_CAP; + if (end > sc.entry_count) end = sc.entry_count; + for (ix(start)..ix(end)) |i| { + buildSrcEntry(sc.entries[i].token, at(sc.src_entries, i)); + } + } +} + +/// Quantize pass1 float vectors to int8 (× 127) for use as pass2 source. +/// Input vectors are unit-normalized, so values are in [-1, 1] → int8 preserves +/// ~99% precision. 4x less memory traffic for pass2. +const Pass1QuantCtx = struct { + entries: [*c]const CorpusEntry, + pass1_q: [*]i8, + entry_count: c_int, + next_idx: AtomicInt, +}; + +fn pass1QuantizeWorker(worker_id: c_int, ctx_ptr: ?*anyopaque) callconv(.c) void { + _ = worker_id; + const qc: *Pass1QuantCtx = @ptrCast(@alignCast(ctx_ptr)); + while (true) { + const start = qc.next_idx.fetchAdd(RRI_TILE, .monotonic); + if (start >= qc.entry_count) break; + var end = start + RRI_TILE; + if (end > qc.entry_count) end = qc.entry_count; + for (ix(start)..ix(end)) |i| { + const src = &at(qc.entries, i).enriched_vec; + const dst = qc.pass1_q[i * DIM ..][0..DIM]; + for (dst, src.v) |*q, x| { + var v = x * INT8_MAX; + if (v > INT8_MAX) v = INT8_MAX; + if (v < -INT8_MAX) v = -INT8_MAX; + q.* = @intFromFloat(if (v >= 0) v + IDF_BASE else v - IDF_BASE); + } + } + } +} + +/// Build reverse index: token_id → list of (doc_id, position) pairs. +/// SEQUENTIAL (fast: just pointer arithmetic + flat array fill). +fn buildReverseIndex(corpus: *Corpus) ?*ReverseIndex { + const rev: *ReverseIndex = @as(?*ReverseIndex, interop.callocN(ReverseIndex, 1)) orelse return null; + const entry_count = ix(corpus.entry_count); + const doc_count = ix(corpus.doc_count); + // Phase A: count occurrences per token + const counts = interop.callocN(c_int, entry_count + 1); + if (counts == null) { + c.free(rev); + return null; + } + var total: c_long = 0; + for (0..doc_count) |d| { + const ids = corpus.doc_token_ids[d]; + const len = corpus.doc_token_counts[d]; + var i: c_int = 0; + while (i < len) : (i += 1) { + const tid = ids[ix(i)]; + if (tid >= 0 and tid < corpus.entry_count) { + counts[ix(tid)] += 1; + total += 1; + } + } + } + // Phase B: exclusive prefix sum → offsets[] + rev.offsets = interop.mallocN(c_int, entry_count + 1); + if (rev.offsets == null) { + c.free(@ptrCast(counts)); + c.free(rev); + return null; + } + var running: c_int = 0; + for (0..entry_count) |t| { + rev.offsets[t] = running; + running += counts[t]; + counts[t] = 0; // reuse as per-token fill cursor + } + rev.offsets[entry_count] = running; + // Phase C: fill flat array. Ensure allocation size > 0 even for empty + // corpora (avoids malloc(0) which is implementation-defined). + rev.flat = interop.mallocN(CooccurPos, if (total > 0) @intCast(total) else 1); + if (rev.flat == null) { + c.free(@ptrCast(rev.offsets)); + c.free(@ptrCast(counts)); + c.free(rev); + return null; + } + for (0..doc_count) |d| { + const ids = corpus.doc_token_ids[d]; + const len = corpus.doc_token_counts[d]; + var i: c_int = 0; + while (i < len) : (i += 1) { + const tid = ids[ix(i)]; + if (tid >= 0 and tid < corpus.entry_count) { + const slot = ix(rev.offsets[ix(tid)] + counts[ix(tid)]); + counts[ix(tid)] += 1; + rev.flat[slot].doc_id = @intCast(d); + rev.flat[slot].pos = i; + } + } + } + c.free(@ptrCast(counts)); + return rev; +} + +fn freeReverseIndex(rev_c: ?*ReverseIndex) void { + const rev = rev_c orelse return; + c.free(@ptrCast(rev.offsets)); + c.free(@ptrCast(rev.flat)); + c.free(rev); +} + +/// Bundle of parameters shared by the finalize sub-phases. +const FinalizeParams = struct { + corpus: *Corpus, + rev: *ReverseIndex, + src_entries: [*c]SrcEntry, + worker_count: c_int, + num_chunks: c_int, + chunk_size: c_int, + tile_size: c_int, + opts: c.cbm_parallel_for_opts_t, +}; + +/// Sub-phase 1: build tagged source vectors (sparse or dense-int8) in parallel. +fn finalizeBuildSources(p: *FinalizeParams) void { + var sc: SrcBuildCtx = .{ + .entries = p.corpus.entries, + .src_entries = p.src_entries, + .entry_count = p.corpus.entry_count, + .next_idx = .init(0), + }; + c.cbm_parallel_for(p.worker_count, srcBuildWorker, &sc, p.opts); +} + +/// Sub-phases 2+3: co-occurrence pass 1 + normalize. +fn finalizePass1(p: *FinalizeParams) void { + var cc: CooccurSparseCtx = .{ + .entries = p.corpus.entries, + .src_entries = p.src_entries, + .doc_token_ids = p.corpus.doc_token_ids, + .doc_token_counts = p.corpus.doc_token_counts, + .rev = p.rev, + .doc_count = p.corpus.doc_count, + .entry_count = p.corpus.entry_count, + .next_chunk = .init(0), + .num_chunks = p.num_chunks, + .chunk_size = p.chunk_size, + .tile_size = p.tile_size, + }; + c.cbm_parallel_for(p.worker_count, cooccurWorkerSparse, &cc, p.opts); + + var nc: NormCtx = .{ + .entries = p.corpus.entries, + .entry_count = p.corpus.entry_count, + .next_idx = .init(0), + }; + c.cbm_parallel_for(p.worker_count, normalizeWorker, &nc, p.opts); +} + +/// Sub-phases 4+5: quantize pass1 to int8, run RRI pass 2, blend + normalize. +fn finalizePass2(p: *FinalizeParams) void { + const entry_count = ix(p.corpus.entry_count); + const pass1_q: ?[*]i8 = interop.mallocN(i8, entry_count * DIM); + if (pass1_q) |q| { + var qc: Pass1QuantCtx = .{ + .entries = p.corpus.entries, + .pass1_q = q, + .entry_count = p.corpus.entry_count, + .next_idx = .init(0), + }; + c.cbm_parallel_for(p.worker_count, pass1QuantizeWorker, &qc, p.opts); + } + + const pass1: ?[*]Vec = interop.mallocN(Vec, entry_count); + if (pass1) |p1| { + for (0..entry_count) |i| { + p1[i] = p.corpus.entries[i].enriched_vec; + } + } + + if (pass1_q) |q| { + var cc: CooccurInt8Ctx = .{ + .entries = p.corpus.entries, + .pass1_q = q, + .doc_token_ids = p.corpus.doc_token_ids, + .doc_token_counts = p.corpus.doc_token_counts, + .rev = p.rev, + .doc_count = p.corpus.doc_count, + .entry_count = p.corpus.entry_count, + .next_chunk = .init(0), + .num_chunks = p.num_chunks, + .chunk_size = p.chunk_size, + .tile_size = p.tile_size, + }; + c.cbm_parallel_for(p.worker_count, cooccurWorkerInt8, &cc, p.opts); + } + + if (pass1) |p1| { + var bc: BlendCtx = .{ + .entries = p.corpus.entries, + .pass1 = p1, + .entry_count = p.corpus.entry_count, + .next_idx = .init(0), + }; + c.cbm_parallel_for(p.worker_count, blendWorker, &bc, p.opts); + c.free(@ptrCast(p1)); + } + c.free(@ptrCast(pass1_q)); + + var nc: NormCtx = .{ + .entries = p.corpus.entries, + .entry_count = p.corpus.entry_count, + .next_idx = .init(0), + }; + c.cbm_parallel_for(p.worker_count, normalizeWorker, &nc, p.opts); +} + +pub export fn cbm_sem_corpus_finalize(corpus_c: ?*Corpus) void { + const corpus = corpus_c orelse return; + if (corpus.finalized) return; + + // Eager init before parallel dispatch to avoid lazy-init races + ensurePretrainedMap(); + + const worker_count = c.cbm_default_worker_count(false); + const opts: c.cbm_parallel_for_opts_t = .{ .max_workers = worker_count, .force_pthreads = false }; + + // Finer chunks = better load balancing for skewed token distributions. + var num_chunks = worker_count * COOCCUR_CHUNK; + if (num_chunks > corpus.entry_count) num_chunks = corpus.entry_count; + if (num_chunks < 1) num_chunks = 1; + const chunk_size = @divTrunc(corpus.entry_count + num_chunks - 1, num_chunks); + + const rev = buildReverseIndex(corpus) orelse { + corpus.finalized = true; + return; + }; + const src_entries = interop.callocN(SrcEntry, ix(corpus.entry_count)); + if (src_entries == null) { + freeReverseIndex(rev); + corpus.finalized = true; + return; + } + + var params: FinalizeParams = .{ + .corpus = corpus, + .rev = rev, + .src_entries = src_entries, + .worker_count = worker_count, + .num_chunks = num_chunks, + .chunk_size = chunk_size, + .tile_size = TILE_SIZE, + .opts = opts, + }; + finalizeBuildSources(¶ms); + finalizePass1(¶ms); + finalizePass2(¶ms); + + c.free(@ptrCast(src_entries)); + freeReverseIndex(rev); + corpus.finalized = true; +} + +/// log(doc_count / doc_freq) through libm, as the C code computed it. +fn idfOf(doc_count: c_int, df: c_int) f32 { + return c.logf(@as(f32, @floatFromInt(doc_count)) / @as(f32, @floatFromInt(df))); +} + +pub export fn cbm_sem_corpus_idf(corpus_c: ?*const Corpus, token: ?Str) f32 { + const corpus = corpus_c orelse return 0.0; + if (token == null or corpus.doc_count == 0) return 0.0; + const idx = parseTokenIndex(@ptrCast(c.cbm_ht_get(corpus.token_map, token))); + if (idx < 0 or idx >= corpus.entry_count) return 0.0; + const df = corpus.entries[ix(idx)].doc_freq; + if (df <= 0) return 0.0; + return idfOf(corpus.doc_count, df); +} + +pub export fn cbm_sem_corpus_ri_vec(corpus_c: ?*const Corpus, token: ?Str) ?*const Vec { + const corpus = corpus_c orelse return null; + if (token == null) return null; + const idx = parseTokenIndex(@ptrCast(c.cbm_ht_get(corpus.token_map, token))); + if (idx < 0 or idx >= corpus.entry_count) return null; + return &at(corpus.entries, ix(idx)).enriched_vec; +} + +pub export fn cbm_sem_corpus_doc_count(corpus: ?*const Corpus) c_int { + return if (corpus) |cp| cp.doc_count else 0; +} + +pub export fn cbm_sem_corpus_token_count(corpus: ?*const Corpus) c_int { + return if (corpus) |cp| cp.entry_count else 0; +} + +pub export fn cbm_sem_corpus_token_at( + corpus_c: ?*const Corpus, + index: c_int, + out_vec: ?*?*const Vec, + out_idf: ?*f32, +) CStr { + const corpus = corpus_c orelse return null; + if (index < 0 or index >= corpus.entry_count) return null; + const entry = at(corpus.entries, ix(index)); + if (out_vec) |ov| { + ov.* = &entry.enriched_vec; + } + if (out_idf) |oi| { + if (corpus.doc_count > 0) { + const df = entry.doc_freq; + oi.* = if (df > 0) idfOf(corpus.doc_count, df) else 0.0; + } + } + return entry.token; +} + +fn freeHtKv(key: CStr, value: ?*anyopaque, userdata: ?*anyopaque) callconv(.c) void { + _ = userdata; + c.free(@ptrCast(@constCast(key))); + c.free(value); +} + +pub export fn cbm_sem_corpus_free(corpus_c: ?*Corpus) void { + const corpus = corpus_c orelse return; + for (0..ix(corpus.entry_count)) |i| { + c.free(corpus.entries[i].token); + } + c.free(@ptrCast(corpus.entries)); + for (0..ix(corpus.doc_count)) |d| { + c.free(@ptrCast(corpus.doc_token_ids[d])); + } + c.free(@ptrCast(corpus.doc_token_ids)); + c.free(@ptrCast(corpus.doc_token_counts)); + if (corpus.token_map) |map| { + c.cbm_ht_foreach(map, freeHtKv, null); + c.cbm_ht_free(map); + } + c.free(corpus); +} + +// ── Combined scoring ──────────────────────────────────────────────── + +pub export fn cbm_sem_proximity(path_a: ?Str, path_b: ?Str) f32 { + const pa = path_a orelse return UNIT_POS; + const pb = path_b orelse return UNIT_POS; + // Count shared directory components + var shared: c_int = 0; + var total_a: c_int = 0; + var total_b: c_int = 0; + var i: usize = 0; + while (pa[i] != 0 and pb[i] != 0 and pa[i] == pb[i]) : (i += 1) { + if (pa[i] == '/') shared += 1; + } + for (std.mem.span(pa)) |ch| { + if (ch == '/') total_a += 1; + } + for (std.mem.span(pb)) |ch| { + if (ch == '/') total_b += 1; + } + const max_total = @max(total_a, total_b); + if (max_total == 0) return UNIT_POS; + const ratio = @as(f32, @floatFromInt(shared)) / @as(f32, @floatFromInt(max_total)); + // Same file = 1.10, same dir = 1.05, distant = 1.00 + return fp.mulAdd(f32, ratio, PROX_MAX_BOOST, UNIT_POS); +} + +/// Cosine similarity for small float arrays (AST profile). +fn smallCosine(a: []const f32, b: []const f32) f32 { + var dot: f32 = 0.0; + var ma: f32 = 0.0; + var mb: f32 = 0.0; + for (a, b) |x, y| { + dot = fp.mulAdd(f32, x, y, dot); + ma = fp.mulAdd(f32, x, x, ma); + mb = fp.mulAdd(f32, y, y, mb); + } + const denom = @sqrt(ma) * @sqrt(mb); + return if (denom < DENOM_EPS) 0.0 else dot / denom; +} + +/// Sparse cosine over two pre-sorted (index, weight) vectors. Returns 0 when +/// either side is empty or the magnitude product is below the epsilon guard. +fn sparseTfidfCosine(a: *const Func, b: *const Func) f32 { + if (a.tfidf_len <= 0 or b.tfidf_len <= 0) return 0.0; + const a_len = ix(a.tfidf_len); + const b_len = ix(b.tfidf_len); + var dot: f32 = 0.0; + var ma: f32 = 0.0; + var mb: f32 = 0.0; + var ia: usize = 0; + var ib: usize = 0; + while (ia < a_len and ib < b_len) { + if (a.tfidf_indices[ia] == b.tfidf_indices[ib]) { + dot = fp.mulAdd(f32, a.tfidf_weights[ia], b.tfidf_weights[ib], dot); + ia += 1; + ib += 1; + } else if (a.tfidf_indices[ia] < b.tfidf_indices[ib]) { + ia += 1; + } else { + ib += 1; + } + } + for (a.tfidf_weights[0..a_len]) |w| { + ma = fp.mulAdd(f32, w, w, ma); + } + for (b.tfidf_weights[0..b_len]) |w| { + mb = fp.mulAdd(f32, w, w, mb); + } + const denom = @sqrt(ma) * @sqrt(mb); + return if (denom > DENOM_EPS) dot / denom else 0.0; +} + +pub export fn cbm_sem_combined_score( + a_c: ?*const Func, + b_c: ?*const Func, + cfg_c: ?*const c.cbm_sem_config_t, +) f32 { + const a = a_c orelse return 0.0; + const b = b_c orelse return 0.0; + const cfg = cfg_c orelse return 0.0; + const a_minhash: *const c.cbm_minhash_t = @ptrCast(&a.minhash); + const b_minhash: *const c.cbm_minhash_t = @ptrCast(&b.minhash); + + // Short-circuit: if MinHash Jaccard is already above the SIMILAR_TO threshold, + // the pass_similarity pipeline already emitted a SIMILAR_TO edge for this pair. + // Returning 0 here avoids flooding top-k with cross-service copy-paste boilerplate + // (logging_middleware, shared push/pull handlers) that SIMILAR_TO already covers, + // freeing the edge budget for true semantic leaps (vocabulary-bridged relations). + if (a.has_minhash and b.has_minhash) { + const early_j = c.cbm_minhash_jaccard(a_minhash, b_minhash); + if (early_j >= c.CBM_MINHASH_JACCARD_THRESHOLD) return 0.0; + } + + var score = cfg.w_tfidf * sparseTfidfCosine(a, b); + + // Signal 2: Random Indexing + score = fp.mulAdd(f32, cfg.w_ri, c.cbm_rsq_ip(&a.ri_code, &b.ri_code), score); + + // Signal 3: MinHash Jaccard + if (a.has_minhash and b.has_minhash) { + const j = c.cbm_minhash_jaccard(a_minhash, b_minhash); + score = fp.mulAdd(f32, cfg.w_minhash, @floatCast(j), score); + } + + // Signal 4: API Signatures + score = fp.mulAdd(f32, cfg.w_api, c.cbm_rsq_ip(&a.api_code, &b.api_code), score); + + // Signal 5: Type Signatures + score = fp.mulAdd(f32, cfg.w_type, c.cbm_rsq_ip(&a.type_code, &b.type_code), score); + + // Signal 7: Decorator Pattern + score = fp.mulAdd(f32, cfg.w_decorator, c.cbm_rsq_ip(&a.deco_code, &b.deco_code), score); + + // Signal 8+9+11: Structural profile + data flow + Halstead + const sp_score = smallCosine(&a.struct_profile, &b.struct_profile); + score = fp.mulAdd(f32, cfg.w_struct_profile, sp_score, score); + + // Signal 6: Module proximity (multiplier, not additive). + // Proximity returns [1.0, 1.10] — a same-file/same-dir boost for ranking. + // Clamp the final product to [0, 1] so the output stays within valid + // cosine-similarity range. Without the clamp, a 0.95 base × 1.10 proximity + // would emit a 1.045 score which violates the semantic of "similarity" and + // breaks downstream consumers that expect a normalized value. + score *= cbm_sem_proximity(a.file_path, b.file_path); + if (score > UNIT_POS) score = UNIT_POS; + if (score < 0.0) score = 0.0; + + return score; +} + +// ── Graph diffusion ───────────────────────────────────────────────── + +pub export fn cbm_sem_diffuse( + combined_c: ?*Vec, + neighbors_c: ?[*]const Vec, + neighbor_count: c_int, + alpha: f32, +) void { + const combined = combined_c orelse return; + const neighbors = neighbors_c orelse return; + if (neighbor_count <= 0) return; + // Blend: combined = (1-α) × combined + α × mean(neighbors) + var mean: Vec = undefined; + @memset(&mean.v, 0.0); + for (neighbors[0..ix(neighbor_count)]) |*neighbor| { + for (&mean.v, neighbor.v) |*m, x| { + m.* += x; + } + } + const inv_n = UNIT_POS / @as(f32, @floatFromInt(neighbor_count)); + const one_minus_alpha = UNIT_POS - alpha; + for (&combined.v, mean.v) |*x, m| { + x.* = fp.mulAdd(f32, one_minus_alpha, x.*, alpha * m * inv_n); + } + cbm_sem_normalize(combined); +}