From c0413f8cffaf1e0b7bcd3b22c02bc7423e25c9f2 Mon Sep 17 00:00:00 2001 From: professional-tester <250027118+professional-tester@users.noreply.github.com> Date: Wed, 11 Nov 2026 01:56:43 +0300 Subject: [PATCH] refactor(semantic): port src/semantic from C to Zig Replace src/semantic/{semantic,ast_profile,rotsq}.c with Zig modules in libcbm_zig. The C headers (semantic.h, ast_profile.h, rotsq.h) stay the ABI: each Zig module exports the same symbols, and cbm_zig.zig checks every pub cbm_* export against its translated header declaration. - semantic.zig: tokenizer, corpus (IDF, random indexing, co-occurrence), vector math and the combined score. It reads the pretrained nomic table through nomic/code_vectors.h, as the C code did. - ast_profile.zig: the AST structural profile and its string form. - rotsq.zig: the rotational scalar quantizer. - fp.zig: mulAdd. The C build contracts `a * b + c` into one fused multiply-add where the CPU has FMA (clang's -ffp-contract=on); Zig never fuses on its own. mulAdd fuses at the same places, so the results match the C build on the same CPU. Behavior differences from the C code: - rotsq builds its +/-1 diagonal at compile time instead of on first use under cbm_once. The values are the same (XXH3 of the dimension index with the same seed). Build wiring: Makefile.cbm and build.zig drop the C sources and list the Zig files; cbm_zig_c.h gains nomic/code_vectors.h, pipeline/worker_pool.h and the two semantic headers. scripts/extract_nomic_vectors.py names cbm_zig_c.h as the place that includes the vector table. Net change: -80 lines. Co-Authored-By: Claude Fable 5.2 Signed-off-by: professional-tester <250027118+professional-tester@users.noreply.github.com> Signed-off-by: professional-tester <250027118+professional-tester@users.noreply.github.com> Co-authored-by: Ata Sesli --- Makefile.cbm | 8 +- build.zig | 3 - scripts/extract_nomic_vectors.py | 2 +- src/cbm_zig.zig | 3 + src/cbm_zig_c.h | 4 +- src/semantic/ast_profile.c | 417 -------- src/semantic/ast_profile.zig | 393 +++++++ src/semantic/fp.zig | 28 + src/semantic/rotsq.c | 129 --- src/semantic/rotsq.zig | 121 +++ src/semantic/semantic.c | 1683 ------------------------------ src/semantic/semantic.zig | 1599 ++++++++++++++++++++++++++++ 12 files changed, 2154 insertions(+), 2236 deletions(-) delete mode 100644 src/semantic/ast_profile.c create mode 100644 src/semantic/ast_profile.zig create mode 100644 src/semantic/fp.zig delete mode 100644 src/semantic/rotsq.c create mode 100644 src/semantic/rotsq.zig delete mode 100644 src/semantic/semantic.c create mode 100644 src/semantic/semantic.zig diff --git a/Makefile.cbm b/Makefile.cbm index a58c32d9..c124cc77 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -194,8 +194,8 @@ PIPELINE_SRCS = \ # SimHash / MinHash, watcher and git context modules are Zig: see CBM_ZIG_SRC. -# Semantic embedding module -SEMANTIC_SRCS = src/semantic/semantic.c src/semantic/ast_profile.c src/semantic/rotsq.c +# Semantic embedding module: implemented in Zig (src/semantic/*.zig, part of libcbm_zig) +SEMANTIC_SRCS = # nomic-embed-code pretrained vectors (assembler blob) UNIXCODER_BLOB_SRC = vendored/nomic/code_vectors_blob.S @@ -479,6 +479,10 @@ CBM_ZIG_SRCS = $(CBM_ZIG_SRC) src/cbm_zig_c.h src/cli_zig/cbm_cli_zig.zig src/cl src/discover/discover.h src/discover/userconfig.h \ src/git/git_context.zig src/git/git_context.h \ src/graph_buffer/graph_buffer.zig src/graph_buffer/graph_buffer.h \ + src/semantic/ast_profile.zig src/semantic/fp.zig src/semantic/rotsq.zig \ + src/semantic/semantic.zig \ + src/semantic/ast_profile.h src/semantic/rotsq.h src/semantic/semantic.h \ + vendored/nomic/code_vectors.h vendored/nomic/code_tokens.h \ src/simhash/minhash.zig src/simhash/minhash.h \ src/store/store.zig src/store/store.h \ src/watcher/watcher.zig src/watcher/watcher.h \ diff --git a/build.zig b/build.zig index c1023226..8bc26830 100644 --- a/build.zig +++ b/build.zig @@ -564,9 +564,6 @@ const native_sources = [_][]const u8{ "src/pipeline/pass_complexity.c", "src/pipeline/pass_importance.c", "src/pipeline/pass_pkgmap.c", - "src/semantic/semantic.c", - "src/semantic/ast_profile.c", - "src/semantic/rotsq.c", "vendored/yyjson/yyjson.c", "internal/cbm/cbm.c", "internal/cbm/extract_defs.c", diff --git a/scripts/extract_nomic_vectors.py b/scripts/extract_nomic_vectors.py index ef38c2ca..60c5bd5e 100755 --- a/scripts/extract_nomic_vectors.py +++ b/scripts/extract_nomic_vectors.py @@ -520,7 +520,7 @@ def main(): print() print("next steps:") print(f" 1. update Makefile.cbm: change UNIXCODER_BLOB_SRC path to vendored/nomic/") - print(f" 2. update #include in semantic.c: \"vendored/nomic/code_vectors.h\"") + print(f" 2. update #include in src/cbm_zig_c.h: \"nomic/code_vectors.h\"") print(f" 3. arch -arm64 make -j12 -f Makefile.cbm clean-c && arch -arm64 make -j12 -f Makefile.cbm") diff --git a/src/cbm_zig.zig b/src/cbm_zig.zig index 36eeb008..1e4a55ba 100644 --- a/src/cbm_zig.zig +++ b/src/cbm_zig.zig @@ -61,6 +61,9 @@ comptime { checkExports(@import("mcp/project_tools.zig")); checkExports(@import("mcp/search_tools.zig")); checkExports(@import("mcp/server.zig")); + checkExports(@import("semantic/ast_profile.zig")); + checkExports(@import("semantic/rotsq.zig")); + checkExports(@import("semantic/semantic.zig")); checkExports(@import("simhash/minhash.zig")); checkExports(@import("store/store.zig")); checkExports(@import("watcher/watcher.zig")); diff --git a/src/cbm_zig_c.h b/src/cbm_zig_c.h index 473c2025..0cb42818 100644 --- a/src/cbm_zig_c.h +++ b/src/cbm_zig_c.h @@ -92,9 +92,9 @@ #include "foundation/compat_fs_internal.h" #include "foundation/yaml.h" #include "mcp/mcp.h" +#include "nomic/code_vectors.h" #include "pipeline/pipeline.h" #include "pipeline/worker_pool.h" -#include "semantic/semantic.h" #include "store/store.h" #include "tree_sitter/api.h" #include "zova/cbm_zova.h" @@ -139,6 +139,8 @@ static inline FILE *cbm_zig_stderr(void) { #include "git/git_context.h" #include "graph_buffer/graph_buffer.h" #include "mcp/index_supervisor.h" +#include "semantic/ast_profile.h" +#include "semantic/semantic.h" #include "simhash/minhash.h" #include "watcher/watcher.h" diff --git a/src/semantic/ast_profile.c b/src/semantic/ast_profile.c deleted file mode 100644 index 283180d8..00000000 --- a/src/semantic/ast_profile.c +++ /dev/null @@ -1,417 +0,0 @@ -/* - * ast_profile.c — Extraction-time AST structural signals. - * - * Walks a function body AST (alongside MinHash) to compute: - * Signal 8: Control flow counts, nesting depth, expression types - * Signal 9: Approximate data flow (params in returns/conditions) - * Signal 11: Halstead-lite (unique/total operators/operands) - * - * Pure functions — thread-safe, no shared state. - */ -#include "semantic/ast_profile.h" -#include "foundation/constants.h" -#include "tree_sitter/api.h" - -#include -#include -#include -#include -#include - -/* ── Node type classification ────────────────────────────────────── */ - -enum { - WALK_STACK_CAP = 2048, - HALSTEAD_SET_SIZE = 512, - HALSTEAD_SET_MASK = 511, - PROFILE_FIELD_COUNT = 25, - DEPTH_SCALE = 10, - BASE_DECIMAL_AST = 10, - HALSTEAD_HASH_MUL = 31, -}; - -typedef struct { - TSNode node; - int depth; -} profile_frame_t; - -static bool is_control_if(const char *k) { - return strcmp(k, "if_statement") == 0 || strcmp(k, "if_expression") == 0 || - strcmp(k, "elif_clause") == 0; -} - -static bool is_control_for(const char *k) { - return strcmp(k, "for_statement") == 0 || strcmp(k, "for_range_loop") == 0 || - strcmp(k, "for_expression") == 0 || strcmp(k, "for_in_clause") == 0; -} - -static bool is_control_while(const char *k) { - return strcmp(k, "while_statement") == 0 || strcmp(k, "while_expression") == 0 || - strcmp(k, "do_statement") == 0; -} - -static bool is_control_switch(const char *k) { - return strcmp(k, "switch_statement") == 0 || strcmp(k, "switch_expression") == 0 || - strcmp(k, "match_expression") == 0 || strcmp(k, "type_switch_statement") == 0; -} - -static bool is_control_try(const char *k) { - return strcmp(k, "try_statement") == 0 || strcmp(k, "try_expression") == 0 || - strcmp(k, "catch_clause") == 0 || strcmp(k, "except_clause") == 0; -} - -static bool is_return(const char *k) { - return strcmp(k, "return_statement") == 0 || strcmp(k, "return_expression") == 0; -} - -static bool is_comparison(const char *k) { - return strcmp(k, "binary_expression") == 0 || strcmp(k, "comparison_operator") == 0 || - strcmp(k, "boolean_operator") == 0; -} - -static bool is_arithmetic(const char *k) { - return strcmp(k, "unary_expression") == 0 || strcmp(k, "update_expression") == 0; -} - -static bool is_assignment(const char *k) { - return strcmp(k, "assignment_expression") == 0 || strcmp(k, "assignment_statement") == 0 || - strcmp(k, "augmented_assignment") == 0 || strcmp(k, "short_var_declaration") == 0; -} - -static bool is_string_lit(const char *k) { - return strcmp(k, "string") == 0 || strcmp(k, "string_literal") == 0 || - strcmp(k, "interpreted_string_literal") == 0 || strcmp(k, "raw_string_literal") == 0; -} - -static bool is_number_lit(const char *k) { - return strcmp(k, "number") == 0 || strcmp(k, "integer") == 0 || strcmp(k, "float") == 0 || - strcmp(k, "integer_literal") == 0 || strcmp(k, "float_literal") == 0; -} - -static bool is_bool_lit(const char *k) { - return strcmp(k, "true") == 0 || strcmp(k, "false") == 0; -} - -static bool is_operator_node(const char *k) { - /* Named nodes that represent operations (not data). */ - return is_control_if(k) || is_control_for(k) || is_control_while(k) || is_control_switch(k) || - is_control_try(k) || is_return(k) || is_comparison(k) || is_arithmetic(k) || - is_assignment(k) || strcmp(k, "call_expression") == 0 || - strcmp(k, "member_expression") == 0 || strcmp(k, "subscript_expression") == 0; -} - -static bool is_identifier(const char *k) { - return strcmp(k, "identifier") == 0 || strcmp(k, "field_identifier") == 0 || - strcmp(k, "property_identifier") == 0 || strcmp(k, "type_identifier") == 0; -} - -/* Simple hash set for Halstead unique counting (open addressing). */ -static bool halstead_insert(uint32_t *set, const char *key) { - uint32_t h = 0; - for (const char *p = key; *p; p++) { - h = (h * HALSTEAD_HASH_MUL) + (uint32_t)*p; - } - uint32_t idx = h & HALSTEAD_SET_MASK; - for (int probe = 0; probe < HALSTEAD_SET_SIZE; probe++) { - uint32_t slot = (idx + (uint32_t)probe) & HALSTEAD_SET_MASK; - if (set[slot] == 0) { - set[slot] = h | SKIP_ONE; - return true; /* new */ - } - if (set[slot] == (h | SKIP_ONE)) { - return false; /* existing */ - } - } - return false; -} - -/* Check if an identifier matches any parameter name. */ -static bool is_param_name(const char *ident, const char *source, const char **param_names, - int param_count) { - if (!ident || !source || !param_names || param_count == 0) { - return false; - } - for (int i = 0; i < param_count; i++) { - if (param_names[i] && strcmp(ident, param_names[i]) == 0) { - return true; - } - } - return false; -} - -/* ── Main computation ────────────────────────────────────────────── */ - -/* Count control-flow statement kinds (if/for/while/switch/try/return). */ -static void accumulate_control_flow(const char *kind, cbm_ast_profile_t *out, bool *in_return) { - if (is_control_if(kind)) { - out->if_count++; - } - if (is_control_for(kind)) { - out->for_count++; - } - if (is_control_while(kind)) { - out->while_count++; - } - if (is_control_switch(kind)) { - out->switch_count++; - } - if (is_control_try(kind)) { - out->try_count++; - } - if (is_return(kind)) { - out->return_count++; - *in_return = true; - } -} - -/* Count expression- and literal-class counters for one node. */ -static void accumulate_expressions(const char *kind, cbm_ast_profile_t *out) { - if (is_comparison(kind)) { - out->comparison_ops++; - } - if (is_arithmetic(kind)) { - out->arithmetic_ops++; - } - if (strcmp(kind, "not_operator") == 0 || strcmp(kind, "boolean_operator") == 0) { - out->logical_ops++; - } - if (is_assignment(kind)) { - out->assignment_count++; - out->variable_reassigns++; - } - if (is_string_lit(kind)) { - out->string_literals++; - } - if (is_number_lit(kind)) { - out->number_literals++; - } - if (is_bool_lit(kind)) { - out->bool_literals++; - } -} - -/* Accumulate Halstead operator/operand counts. Leaf identifiers, literals, - * and booleans are "operands"; recognized statement/expression kinds are - * "operators". Uses an open-addressed hash set for unique counting. */ -static void accumulate_halstead(const char *kind, uint32_t child_count, uint32_t *op_set, - uint32_t *operand_set, cbm_ast_profile_t *out) { - if (is_operator_node(kind)) { - out->total_operators++; - if (halstead_insert(op_set, kind)) { - out->unique_operators++; - } - } - if (child_count == 0 && - (is_identifier(kind) || is_string_lit(kind) || is_number_lit(kind) || is_bool_lit(kind))) { - out->total_operands++; - if (halstead_insert(operand_set, kind)) { - out->unique_operands++; - } - out->body_tokens++; - } -} - -/* Data-flow counter update: bump params_in_returns / params_in_conditions - * when the current leaf identifier names a parameter and we're inside the - * corresponding syntactic scope. */ -static void accumulate_data_flow(TSNode node, const char *kind, uint32_t child_count, - const char *source, const char **param_names, int param_count, - bool in_return, bool in_condition, cbm_ast_profile_t *out) { - if (!(child_count == 0 && is_identifier(kind) && source)) { - return; - } - uint32_t start = ts_node_start_byte(node); - uint32_t end = ts_node_end_byte(node); - if (end <= start || (end - start) >= CBM_SZ_128) { - return; - } - char ident_buf[CBM_SZ_128]; - int ilen = (int)(end - start); - memcpy(ident_buf, source + start, (size_t)ilen); - ident_buf[ilen] = '\0'; - if (!is_param_name(ident_buf, source, param_names, param_count)) { - return; - } - if (in_return) { - out->params_in_returns++; - } - if (in_condition) { - out->params_in_conditions++; - } -} - -bool cbm_ast_profile_compute(TSNode func_body, const char *source, const char **param_names, - int param_count, cbm_ast_profile_t *out) { - if (ts_node_is_null(func_body)) { - return false; - } - - memset(out, 0, sizeof(*out)); - out->param_count = (uint16_t)param_count; - - uint32_t op_set[HALSTEAD_SET_SIZE]; - uint32_t operand_set[HALSTEAD_SET_SIZE]; - memset(op_set, 0, sizeof(op_set)); - memset(operand_set, 0, sizeof(operand_set)); - - int total_depth = 0; - int node_count = 0; - bool in_return = false; - bool in_condition = false; - - profile_frame_t stack[WALK_STACK_CAP]; - int top = 0; - stack[top++] = (profile_frame_t){func_body, 0}; - - while (top > 0) { - profile_frame_t frame = stack[--top]; - TSNode node = frame.node; - int depth = frame.depth; - uint32_t child_count = ts_node_child_count(node); - const char *kind = ts_node_type(node); - - if (!ts_node_is_named(node) && child_count == 0) { - /* Anonymous leaf (punctuation, keywords) — skip. */ - goto push_children; - } - - node_count++; - total_depth += depth; - - if ((uint16_t)depth > out->max_nesting_depth) { - out->max_nesting_depth = (uint16_t)depth; - } - - accumulate_control_flow(kind, out, &in_return); - accumulate_expressions(kind, out); - accumulate_halstead(kind, child_count, op_set, operand_set, out); - accumulate_data_flow(node, kind, child_count, source, param_names, param_count, in_return, - in_condition, out); - - /* Track context for data flow: are we inside a condition? */ - if (is_control_if(kind) || is_control_while(kind)) { - in_condition = true; - } - - push_children: - /* Reset context flags when leaving return/condition scope */ - if (is_return(kind)) { - in_return = false; - } - if (child_count > 0 && (is_control_if(kind) || is_control_while(kind))) { - in_condition = false; - } - - /* Push children in reverse order */ - for (int i = (int)child_count - SKIP_ONE; i >= 0 && top < WALK_STACK_CAP; i--) { - stack[top++] = (profile_frame_t){ts_node_child(node, (uint32_t)i), depth + SKIP_ONE}; - } - } - - /* Compute averages */ - if (node_count > 0) { - out->avg_nesting_depth_x10 = (uint16_t)((total_depth * DEPTH_SCALE) / node_count); - } - - return node_count > 0; -} - -/* ── Serialization ───────────────────────────────────────────────── */ - -void cbm_ast_profile_to_str(const cbm_ast_profile_t *p, char *buf, int bufsize) { - if (!p || !buf || bufsize < SKIP_ONE) { - if (buf && bufsize > 0) { - buf[0] = '\0'; - } - return; - } - snprintf(buf, (size_t)bufsize, - "%u,%u,%u,%u,%u,%u,%u,%u," - "%u,%u,%u,%u," - "%u,%u,%u," - "%u,%u,%u,%u," - "%u,%u,%u,%u," - "%u,%u", - p->if_count, p->for_count, p->while_count, p->switch_count, p->try_count, - p->return_count, p->max_nesting_depth, p->avg_nesting_depth_x10, p->comparison_ops, - p->arithmetic_ops, p->logical_ops, p->assignment_count, p->string_literals, - p->number_literals, p->bool_literals, p->param_count, p->params_in_returns, - p->params_in_conditions, p->variable_reassigns, p->unique_operators, - p->unique_operands, p->total_operators, p->total_operands, p->body_lines, - p->body_tokens); -} - -bool cbm_ast_profile_from_str(const char *str, cbm_ast_profile_t *out) { - if (!str || !out) { - return false; - } - memset(out, 0, sizeof(*out)); - /* Field layout must match cbm_ast_profile_to_str() exactly. */ - uint16_t *fields[PROFILE_FIELD_COUNT] = { - &out->if_count, &out->for_count, - &out->while_count, &out->switch_count, - &out->try_count, &out->return_count, - &out->max_nesting_depth, &out->avg_nesting_depth_x10, - &out->comparison_ops, &out->arithmetic_ops, - &out->logical_ops, &out->assignment_count, - &out->string_literals, &out->number_literals, - &out->bool_literals, &out->param_count, - &out->params_in_returns, &out->params_in_conditions, - &out->variable_reassigns, &out->unique_operators, - &out->unique_operands, &out->total_operators, - &out->total_operands, &out->body_lines, - &out->body_tokens, - }; - const char *p = str; - for (int i = 0; i < PROFILE_FIELD_COUNT; i++) { - char *end = NULL; - unsigned long parsed = strtoul(p, &end, BASE_DECIMAL_AST); - if (end == p) { - return false; - } - *fields[i] = (uint16_t)parsed; - p = end; - /* Skip the comma separator between fields. */ - if (i + SKIP_ONE < PROFILE_FIELD_COUNT) { - if (*p != ',') { - return false; - } - p++; - } - } - return true; -} - -void cbm_ast_profile_to_vector(const cbm_ast_profile_t *p, float *out) { - if (!p || !out) { - return; - } - /* Normalize each field to [0,1] range using reasonable maximums. */ - enum { MAX_COUNT = 100, MAX_DEPTH = 20, MAX_HALSTEAD = 200, MAX_TOKENS = 2000 }; - int i = 0; - out[i++] = (float)p->if_count / MAX_COUNT; - out[i++] = (float)p->for_count / MAX_COUNT; - out[i++] = (float)p->while_count / MAX_COUNT; - out[i++] = (float)p->switch_count / MAX_COUNT; - out[i++] = (float)p->try_count / MAX_COUNT; - out[i++] = (float)p->return_count / MAX_COUNT; - out[i++] = (float)p->max_nesting_depth / MAX_DEPTH; - out[i++] = (float)p->avg_nesting_depth_x10 / (MAX_DEPTH * DEPTH_SCALE); - out[i++] = (float)p->comparison_ops / MAX_COUNT; - out[i++] = (float)p->arithmetic_ops / MAX_COUNT; - out[i++] = (float)p->logical_ops / MAX_COUNT; - out[i++] = (float)p->assignment_count / MAX_COUNT; - out[i++] = (float)p->string_literals / MAX_COUNT; - out[i++] = (float)p->number_literals / MAX_COUNT; - out[i++] = (float)p->bool_literals / MAX_COUNT; - out[i++] = (float)p->param_count / MAX_DEPTH; - out[i++] = (float)p->params_in_returns / MAX_COUNT; - out[i++] = (float)p->params_in_conditions / MAX_COUNT; - out[i++] = (float)p->variable_reassigns / MAX_COUNT; - out[i++] = (float)p->unique_operators / MAX_HALSTEAD; - out[i++] = (float)p->unique_operands / MAX_HALSTEAD; - out[i++] = (float)p->total_operators / MAX_HALSTEAD; - out[i++] = (float)p->total_operands / MAX_HALSTEAD; - out[i++] = (float)p->body_lines / MAX_TOKENS; - out[i++] = (float)p->body_tokens / MAX_TOKENS; -} diff --git a/src/semantic/ast_profile.zig b/src/semantic/ast_profile.zig new file mode 100644 index 00000000..cf47e2f0 --- /dev/null +++ b/src/semantic/ast_profile.zig @@ -0,0 +1,393 @@ +//! ast_profile.zig — Extraction-time AST structural signals. +//! +//! Implements src/semantic/ast_profile.h. Walks a function body AST +//! (alongside MinHash) to compute: +//! Signal 8: Control flow counts, nesting depth, expression types +//! Signal 9: Approximate data flow (params in returns/conditions) +//! Signal 11: Halstead-lite (unique/total operators/operands) +//! +//! Pure functions — thread-safe, no shared state. + +const std = @import("std"); +const c = @import("c"); + +const Profile = c.cbm_ast_profile_t; +const Str = [*:0]const u8; + +// ── Node type classification ──────────────────────────────────────── + +const WALK_STACK_CAP = 2048; +const HALSTEAD_SET_SIZE = 512; +const HALSTEAD_SET_MASK = 511; +const PROFILE_FIELD_COUNT = 25; +const DEPTH_SCALE = 10; +const BASE_DECIMAL_AST = 10; +const HALSTEAD_HASH_MUL = 31; + +const ProfileFrame = struct { + node: c.TSNode, + depth: c_int, +}; + +fn isOneOf(kind: []const u8, comptime names: []const []const u8) bool { + inline for (names) |name| { + if (std.mem.eql(u8, kind, name)) return true; + } + return false; +} + +fn isControlIf(k: []const u8) bool { + return isOneOf(k, &.{ "if_statement", "if_expression", "elif_clause" }); +} + +fn isControlFor(k: []const u8) bool { + return isOneOf(k, &.{ "for_statement", "for_range_loop", "for_expression", "for_in_clause" }); +} + +fn isControlWhile(k: []const u8) bool { + return isOneOf(k, &.{ "while_statement", "while_expression", "do_statement" }); +} + +fn isControlSwitch(k: []const u8) bool { + return isOneOf(k, &.{ + "switch_statement", "switch_expression", "match_expression", "type_switch_statement", + }); +} + +fn isControlTry(k: []const u8) bool { + return isOneOf(k, &.{ "try_statement", "try_expression", "catch_clause", "except_clause" }); +} + +fn isReturn(k: []const u8) bool { + return isOneOf(k, &.{ "return_statement", "return_expression" }); +} + +fn isComparison(k: []const u8) bool { + return isOneOf(k, &.{ "binary_expression", "comparison_operator", "boolean_operator" }); +} + +fn isArithmetic(k: []const u8) bool { + return isOneOf(k, &.{ "unary_expression", "update_expression" }); +} + +fn isAssignment(k: []const u8) bool { + return isOneOf(k, &.{ + "assignment_expression", "assignment_statement", "augmented_assignment", + "short_var_declaration", + }); +} + +fn isStringLit(k: []const u8) bool { + return isOneOf(k, &.{ + "string", "string_literal", "interpreted_string_literal", "raw_string_literal", + }); +} + +fn isNumberLit(k: []const u8) bool { + return isOneOf(k, &.{ "number", "integer", "float", "integer_literal", "float_literal" }); +} + +fn isBoolLit(k: []const u8) bool { + return isOneOf(k, &.{ "true", "false" }); +} + +fn isOperatorNode(k: []const u8) bool { + // Named nodes that represent operations (not data). + return isControlIf(k) or isControlFor(k) or isControlWhile(k) or isControlSwitch(k) or + isControlTry(k) or isReturn(k) or isComparison(k) or isArithmetic(k) or + isAssignment(k) or + isOneOf(k, &.{ "call_expression", "member_expression", "subscript_expression" }); +} + +fn isIdentifier(k: []const u8) bool { + return isOneOf(k, &.{ + "identifier", "field_identifier", "property_identifier", "type_identifier", + }); +} + +/// Simple hash set for Halstead unique counting (open addressing). +fn halsteadInsert(set: *[HALSTEAD_SET_SIZE]u32, key: []const u8) bool { + var h: u32 = 0; + for (key) |ch| { + h = (h *% HALSTEAD_HASH_MUL) +% ch; + } + const idx = h & HALSTEAD_SET_MASK; + for (0..HALSTEAD_SET_SIZE) |probe| { + const slot = (idx +% @as(u32, @intCast(probe))) & HALSTEAD_SET_MASK; + if (set[slot] == 0) { + set[slot] = h | 1; + return true; // new + } + if (set[slot] == (h | 1)) { + return false; // existing + } + } + return false; +} + +/// Check if an identifier matches any parameter name. +fn isParamName(ident: Str, param_names: ?[*]?Str, param_count: c_int) bool { + const names = param_names orelse return false; + if (param_count <= 0) return false; + for (names[0..@intCast(param_count)]) |name| { + if (name != null and c.strcmp(ident, name) == 0) return true; + } + return false; +} + +// ── Main computation ──────────────────────────────────────────────── + +/// Count control-flow statement kinds (if/for/while/switch/try/return). +fn accumulateControlFlow(kind: []const u8, out: *Profile, in_return: *bool) void { + if (isControlIf(kind)) out.if_count +%= 1; + if (isControlFor(kind)) out.for_count +%= 1; + if (isControlWhile(kind)) out.while_count +%= 1; + if (isControlSwitch(kind)) out.switch_count +%= 1; + if (isControlTry(kind)) out.try_count +%= 1; + if (isReturn(kind)) { + out.return_count +%= 1; + in_return.* = true; + } +} + +/// Count expression- and literal-class counters for one node. +fn accumulateExpressions(kind: []const u8, out: *Profile) void { + if (isComparison(kind)) out.comparison_ops +%= 1; + if (isArithmetic(kind)) out.arithmetic_ops +%= 1; + if (isOneOf(kind, &.{ "not_operator", "boolean_operator" })) out.logical_ops +%= 1; + if (isAssignment(kind)) { + out.assignment_count +%= 1; + out.variable_reassigns +%= 1; + } + if (isStringLit(kind)) out.string_literals +%= 1; + if (isNumberLit(kind)) out.number_literals +%= 1; + if (isBoolLit(kind)) out.bool_literals +%= 1; +} + +/// Accumulate Halstead operator/operand counts. Leaf identifiers, literals, +/// and booleans are "operands"; recognized statement/expression kinds are +/// "operators". Uses an open-addressed hash set for unique counting. +fn accumulateHalstead( + kind: []const u8, + child_count: u32, + op_set: *[HALSTEAD_SET_SIZE]u32, + operand_set: *[HALSTEAD_SET_SIZE]u32, + out: *Profile, +) void { + if (isOperatorNode(kind)) { + out.total_operators +%= 1; + if (halsteadInsert(op_set, kind)) out.unique_operators +%= 1; + } + if (child_count == 0 and + (isIdentifier(kind) or isStringLit(kind) or isNumberLit(kind) or isBoolLit(kind))) + { + out.total_operands +%= 1; + if (halsteadInsert(operand_set, kind)) out.unique_operands +%= 1; + out.body_tokens +%= 1; + } +} + +/// Data-flow counter update: bump params_in_returns / params_in_conditions +/// when the current leaf identifier names a parameter and we're inside the +/// corresponding syntactic scope. +fn accumulateDataFlow( + node: c.TSNode, + kind: []const u8, + child_count: u32, + source_c: ?[*]const u8, + param_names: ?[*]?Str, + param_count: c_int, + in_return: bool, + in_condition: bool, + out: *Profile, +) void { + const source = source_c orelse return; + if (!(child_count == 0 and isIdentifier(kind))) return; + const start = c.ts_node_start_byte(node); + const end = c.ts_node_end_byte(node); + if (end <= start or (end - start) >= c.CBM_SZ_128) return; + var ident_buf: [c.CBM_SZ_128]u8 = undefined; + const ilen: usize = end - start; + @memcpy(ident_buf[0..ilen], source[start..end]); + ident_buf[ilen] = 0; + if (!isParamName(@ptrCast(&ident_buf), param_names, param_count)) return; + if (in_return) out.params_in_returns +%= 1; + if (in_condition) out.params_in_conditions +%= 1; +} + +pub export fn cbm_ast_profile_compute( + func_body: c.TSNode, + source: ?[*]const u8, + param_names: ?[*]?Str, + param_count: c_int, + out: *Profile, +) bool { + if (c.ts_node_is_null(func_body)) return false; + + out.* = std.mem.zeroes(Profile); + out.param_count = @truncate(@as(c_uint, @bitCast(param_count))); + + var op_set: [HALSTEAD_SET_SIZE]u32 = @splat(0); + var operand_set: [HALSTEAD_SET_SIZE]u32 = @splat(0); + + var total_depth: c_int = 0; + var node_count: c_int = 0; + var in_return = false; + var in_condition = false; + + var stack: [WALK_STACK_CAP]ProfileFrame = undefined; + var top: usize = 0; + stack[top] = .{ .node = func_body, .depth = 0 }; + top += 1; + + while (top > 0) { + top -= 1; + const frame = stack[top]; + const node = frame.node; + const depth = frame.depth; + const child_count = c.ts_node_child_count(node); + const kind = std.mem.span(@as(Str, @ptrCast(c.ts_node_type(node)))); + + // Anonymous leaves (punctuation, keywords) are skipped. + if (c.ts_node_is_named(node) or child_count != 0) { + node_count += 1; + total_depth +%= depth; + + const depth16: u16 = @truncate(@as(c_uint, @bitCast(depth))); + if (depth16 > out.max_nesting_depth) out.max_nesting_depth = depth16; + + accumulateControlFlow(kind, out, &in_return); + accumulateExpressions(kind, out); + accumulateHalstead(kind, child_count, &op_set, &operand_set, out); + accumulateDataFlow(node, kind, child_count, source, param_names, param_count, in_return, in_condition, out); + + // Track context for data flow: are we inside a condition? + if (isControlIf(kind) or isControlWhile(kind)) in_condition = true; + } + + // Reset context flags when leaving return/condition scope. + if (isReturn(kind)) in_return = false; + if (child_count > 0 and (isControlIf(kind) or isControlWhile(kind))) in_condition = false; + + // Push children in reverse order. + var i = child_count; + while (i > 0 and top < WALK_STACK_CAP) { + i -= 1; + stack[top] = .{ .node = c.ts_node_child(node, i), .depth = depth + 1 }; + top += 1; + } + } + + // Compute averages. + if (node_count > 0) { + const avg = @divTrunc(total_depth *% DEPTH_SCALE, node_count); + out.avg_nesting_depth_x10 = @truncate(@as(c_uint, @bitCast(avg))); + } + + return node_count > 0; +} + +// ── Serialization ─────────────────────────────────────────────────── + +/// Serialization order of the profile fields. Must match the argument order +/// in cbm_ast_profile_to_str(). +const field_order = [PROFILE_FIELD_COUNT][]const u8{ + "if_count", "for_count", "while_count", "switch_count", + "try_count", "return_count", "max_nesting_depth", "avg_nesting_depth_x10", + "comparison_ops", "arithmetic_ops", "logical_ops", "assignment_count", + "string_literals", "number_literals", "bool_literals", "param_count", + "params_in_returns", "params_in_conditions", "variable_reassigns", "unique_operators", + "unique_operands", "total_operators", "total_operands", "body_lines", + "body_tokens", +}; + +pub export fn cbm_ast_profile_to_str(p_c: ?*const Profile, buf_c: ?[*]u8, bufsize: c_int) void { + const buf = buf_c orelse return; + const p = p_c orelse { + if (bufsize > 0) buf[0] = 0; + return; + }; + if (bufsize < 1) return; + const u = struct { + fn u(v: u16) c_uint { + return v; + } + }.u; + _ = c.snprintf( + buf, + @intCast(bufsize), + "%u,%u,%u,%u,%u,%u,%u,%u," ++ + "%u,%u,%u,%u," ++ + "%u,%u,%u," ++ + "%u,%u,%u,%u," ++ + "%u,%u,%u,%u," ++ + "%u,%u", + u(p.if_count), + u(p.for_count), + u(p.while_count), + u(p.switch_count), + u(p.try_count), + u(p.return_count), + u(p.max_nesting_depth), + u(p.avg_nesting_depth_x10), + u(p.comparison_ops), + u(p.arithmetic_ops), + u(p.logical_ops), + u(p.assignment_count), + u(p.string_literals), + u(p.number_literals), + u(p.bool_literals), + u(p.param_count), + u(p.params_in_returns), + u(p.params_in_conditions), + u(p.variable_reassigns), + u(p.unique_operators), + u(p.unique_operands), + u(p.total_operators), + u(p.total_operands), + u(p.body_lines), + u(p.body_tokens), + ); +} + +pub export fn cbm_ast_profile_from_str(str: ?Str, out_c: ?*Profile) bool { + const out = out_c orelse return false; + var p: Str = str orelse return false; + out.* = std.mem.zeroes(Profile); + // Field layout must match cbm_ast_profile_to_str() exactly. + inline for (field_order, 0..) |name, i| { + var end: [*c]u8 = null; + const parsed = c.strtoul(p, &end, BASE_DECIMAL_AST); + if (@intFromPtr(end) == @intFromPtr(p)) return false; + @field(out, name) = @truncate(parsed); + p = @ptrCast(end); + // Skip the comma separator between fields. + if (i + 1 < PROFILE_FIELD_COUNT) { + if (p[0] != ',') return false; + p += 1; + } + } + return true; +} + +pub export fn cbm_ast_profile_to_vector(p_c: ?*const Profile, out_c: ?[*]f32) void { + const p = p_c orelse return; + const out = out_c orelse return; + // Normalize each field to [0,1] range using reasonable maximums. + const MAX_COUNT = 100; + const MAX_DEPTH = 20; + const MAX_HALSTEAD = 200; + const MAX_TOKENS = 2000; + const maxima = [PROFILE_FIELD_COUNT]f32{ + MAX_COUNT, MAX_COUNT, MAX_COUNT, MAX_COUNT, + MAX_COUNT, MAX_COUNT, MAX_DEPTH, MAX_DEPTH * DEPTH_SCALE, + MAX_COUNT, MAX_COUNT, MAX_COUNT, MAX_COUNT, + MAX_COUNT, MAX_COUNT, MAX_COUNT, MAX_DEPTH, + MAX_COUNT, MAX_COUNT, MAX_COUNT, MAX_HALSTEAD, + MAX_HALSTEAD, MAX_HALSTEAD, MAX_HALSTEAD, MAX_TOKENS, + MAX_TOKENS, + }; + inline for (field_order, maxima, 0..) |name, max, i| { + out[i] = @as(f32, @floatFromInt(@field(p, name))) / max; + } +} diff --git a/src/semantic/fp.zig b/src/semantic/fp.zig new file mode 100644 index 00000000..eeeb19f9 --- /dev/null +++ b/src/semantic/fp.zig @@ -0,0 +1,28 @@ +//! fp.zig — floating-point contraction that matches the C build. +//! +//! The C sources were compiled by clang with its default -ffp-contract=on: +//! where the target CPU has a fused multiply-add instruction, `a * b + c` +//! written in one expression becomes a single FMA with one rounding. Zig +//! never fuses on its own, so a port that writes `a * b + c` rounds twice and +//! produces scores that differ in the last bit. Those differences move +//! similarity scores across edge thresholds. +//! +//! mulAdd fuses exactly when clang would: on targets with FMA. Use it for +//! every expression the C code wrote as a product plus (or minus) a term in +//! one expression, including `x += a * b`. + +const std = @import("std"); +const builtin = @import("builtin"); + +/// True when the target has a fused multiply-add instruction, which is when +/// clang lowers llvm.fmuladd to FMA. +pub const has_fma = switch (builtin.cpu.arch) { + .x86_64, .x86 => std.Target.x86.featureSetHas(builtin.cpu.features, .fma), + .aarch64, .aarch64_be => true, + else => false, +}; + +/// `a * b + c` the way clang compiles that C expression. +pub inline fn mulAdd(comptime T: type, a: T, b: T, c: T) T { + return if (has_fma) @mulAdd(T, a, b, c) else a * b + c; +} diff --git a/src/semantic/rotsq.c b/src/semantic/rotsq.c deleted file mode 100644 index c067b5d9..00000000 --- a/src/semantic/rotsq.c +++ /dev/null @@ -1,129 +0,0 @@ -/* - * rotsq.c — see rotsq.h. From-paper implementation, no dependencies. - */ -#include "semantic/rotsq.h" - -#include - -#include "foundation/compat_thread.h" - -#define XXH_INLINE_ALL -#include "xxhash/xxhash.h" - -/* ── Deterministic ±1 diagonal (seeded, generated once) ─────────────── */ - -static float g_rsq_diag[CBM_RSQ_DIM]; -static cbm_once_t g_rsq_diag_once = CBM_ONCE_INIT; - -static void rsq_build_diag(void) { - for (int d = 0; d < CBM_RSQ_DIM; d++) { - uint64_t h = XXH3_64bits_withSeed(&d, sizeof(d), 0x5bd1e995u); - g_rsq_diag[d] = (h & 1u) ? 1.0F : -1.0F; - } -} - -static void rsq_init_diag(void) { - cbm_once(&g_rsq_diag_once, rsq_build_diag); -} - -/* ── Fast Walsh–Hadamard Transform (in place, unnormalized) ─────────── */ - -static void rsq_fwht(float *v) { - for (int len = 1; len < CBM_RSQ_DIM; len <<= 1) { - for (int i = 0; i < CBM_RSQ_DIM; i += len << 1) { - for (int j = i; j < i + len; j++) { - float a = v[j]; - float b = v[j + len]; - v[j] = a + b; - v[j + len] = a - b; - } - } - } -} - -/* ── Encode ─────────────────────────────────────────────────────────── */ - -void cbm_rsq_encode(const float *v, cbm_rsq_code_t *out) { - rsq_init_diag(); - - float rot[CBM_RSQ_DIM]; - for (int d = 0; d < CBM_RSQ_IN_DIM; d++) { - rot[d] = v[d] * g_rsq_diag[d]; - } - for (int d = CBM_RSQ_IN_DIM; d < CBM_RSQ_DIM; d++) { - rot[d] = 0.0F; - } - rsq_fwht(rot); - /* Normalize the transform so the rotation is orthonormal (H/√D): keeps - * the coordinates in a data-independent range and makes the estimated IP - * directly comparable to the pre-rotation IP. */ - const float inv_sqrt_d = 1.0F / 32.0F; /* 1/√1024 */ - float lo = rot[0] * inv_sqrt_d; - float hi = lo; - for (int d = 0; d < CBM_RSQ_DIM; d++) { - rot[d] *= inv_sqrt_d; - if (rot[d] < lo) { - lo = rot[d]; - } - if (rot[d] > hi) { - hi = rot[d]; - } - } - - /* Per-vector scalar quantization over [lo, hi] at CBM_RSQ_BITS. */ - float range = hi - lo; - float step = range > 0.0F ? range / (float)CBM_RSQ_LEVELS : 1.0F; - out->offset = lo; - out->scale = step; - - int32_t sum = 0; - memset(out->codes, 0, sizeof(out->codes)); - for (int d = 0; d < CBM_RSQ_DIM; d++) { - float q = (rot[d] - lo) / step; - int32_t c = (int32_t)(q + 0.5F); - if (c < 0) { - c = 0; - } - if (c > CBM_RSQ_LEVELS) { - c = CBM_RSQ_LEVELS; - } - sum += c; - if (d & 1) { - out->codes[d >> 1] |= (uint8_t)(c << 4); - } else { - out->codes[d >> 1] |= (uint8_t)c; - } - } - out->code_sum = sum; -} - -/* ── Estimated inner product from two codes ─────────────────────────── */ - -float cbm_rsq_ip(const cbm_rsq_code_t *a, const cbm_rsq_code_t *b) { - /* Integer dot of the 4-bit codes. */ - int64_t dot = 0; - for (int i = 0; i < CBM_RSQ_CODE_BYTES; i++) { - uint8_t ba = a->codes[i]; - uint8_t bb = b->codes[i]; - dot += (int64_t)(ba & 0x0F) * (bb & 0x0F); - dot += (int64_t)(ba >> 4) * (bb >> 4); - } - /* x_i ≈ oa + sa·ca_i, y_i ≈ ob + sb·cb_i (in the rotated space, where the - * IP equals the original IP because the rotation is orthonormal). */ - double d = (double)CBM_RSQ_DIM; - double ip = d * (double)a->offset * (double)b->offset + - (double)a->offset * (double)b->scale * (double)b->code_sum + - (double)b->offset * (double)a->scale * (double)a->code_sum + - (double)a->scale * (double)b->scale * (double)dot; - return (float)ip; -} - -/* ── Dequantize into the rotated basis ──────────────────────────────── */ - -void cbm_rsq_decode(const cbm_rsq_code_t *c, float *out) { - for (int i = 0; i < CBM_RSQ_CODE_BYTES; i++) { - uint8_t b = c->codes[i]; - out[i * 2] = c->offset + c->scale * (float)(b & 0x0F); - out[i * 2 + 1] = c->offset + c->scale * (float)(b >> 4); - } -} diff --git a/src/semantic/rotsq.zig b/src/semantic/rotsq.zig new file mode 100644 index 00000000..ebdefda0 --- /dev/null +++ b/src/semantic/rotsq.zig @@ -0,0 +1,121 @@ +//! rotsq.zig — see rotsq.h. From-paper implementation, no dependencies. + +const std = @import("std"); +const c = @import("c"); +const fp = @import("fp.zig"); + +const XxHash3 = std.hash.XxHash3; + +const IN_DIM: comptime_int = c.CBM_RSQ_IN_DIM; +const DIM: comptime_int = c.CBM_RSQ_DIM; +const LEVELS: comptime_int = c.CBM_RSQ_LEVELS; + +// ── Deterministic ±1 diagonal ────────────────────────────────────── + +/// The C code generated this table once at first use. It is a pure function +/// of the index, so it is computed at compile time here. +const rsq_diag: [DIM]f32 = blk: { + @setEvalBranchQuota(4_000_000); + var diag: [DIM]f32 = undefined; + for (&diag, 0..) |*v, d| { + const h = XxHash3.hash(0x5bd1e995, &std.mem.toBytes(@as(c_int, @intCast(d)))); + v.* = if (h & 1 != 0) 1.0 else -1.0; + } + break :blk diag; +}; + +// ── Fast Walsh–Hadamard Transform (in place, unnormalized) ───────── + +fn rsqFwht(v: *[DIM]f32) void { + var len: usize = 1; + while (len < DIM) : (len <<= 1) { + var i: usize = 0; + while (i < DIM) : (i += len << 1) { + for (i..i + len) |j| { + const a = v[j]; + const b = v[j + len]; + v[j] = a + b; + v[j + len] = a - b; + } + } + } +} + +// ── Encode ───────────────────────────────────────────────────────── + +pub export fn cbm_rsq_encode(v: [*]const f32, out: *c.cbm_rsq_code_t) void { + var rot: [DIM]f32 = undefined; + for (rot[0..IN_DIM], v[0..IN_DIM], rsq_diag[0..IN_DIM]) |*r, x, sign| { + r.* = x * sign; + } + @memset(rot[IN_DIM..], 0.0); + rsqFwht(&rot); + // Normalize the transform so the rotation is orthonormal (H/√D): keeps + // the coordinates in a data-independent range and makes the estimated IP + // directly comparable to the pre-rotation IP. + const inv_sqrt_d: f32 = 1.0 / 32.0; // 1/√1024 + var lo: f32 = rot[0] * inv_sqrt_d; + var hi: f32 = lo; + for (&rot) |*r| { + r.* *= inv_sqrt_d; + if (r.* < lo) lo = r.*; + if (r.* > hi) hi = r.*; + } + + // Per-vector scalar quantization over [lo, hi] at CBM_RSQ_BITS. + const range = hi - lo; + const step: f32 = if (range > 0.0) range / @as(f32, LEVELS) else 1.0; + out.offset = lo; + out.scale = step; + + var sum: i32 = 0; + @memset(&out.codes, 0); + for (rot, 0..) |r, d| { + const q = (r - lo) / step; + var code: i32 = @intFromFloat(q + 0.5); + if (code < 0) code = 0; + if (code > LEVELS) code = LEVELS; + sum += code; + const nibble: u8 = @intCast(code); + if (d & 1 != 0) { + out.codes[d >> 1] |= nibble << 4; + } else { + out.codes[d >> 1] |= nibble; + } + } + out.code_sum = sum; +} + +// ── Estimated inner product from two codes ───────────────────────── + +pub export fn cbm_rsq_ip(a: *const c.cbm_rsq_code_t, b: *const c.cbm_rsq_code_t) f32 { + // Integer dot of the 4-bit codes. + var dot: i64 = 0; + for (a.codes, b.codes) |ba, bb| { + dot += @as(i64, ba & 0x0F) * (bb & 0x0F); + dot += @as(i64, ba >> 4) * (bb >> 4); + } + // x_i ≈ oa + sa·ca_i, y_i ≈ ob + sb·cb_i (in the rotated space, where the + // IP equals the original IP because the rotation is orthonormal). + const d: f64 = DIM; + const oa: f64 = a.offset; + const ob: f64 = b.offset; + const sa: f64 = a.scale; + const sb: f64 = b.scale; + // C: d*oa*ob + oa*sb*Σcb + ob*sa*Σca + sa*sb*dot, summed left to right. + // The first sum fuses with its left product, the later ones with their + // right product. + var ip = fp.mulAdd(f64, d * oa, ob, oa * sb * @as(f64, @floatFromInt(b.code_sum))); + ip = fp.mulAdd(f64, ob * sa, @as(f64, @floatFromInt(a.code_sum)), ip); + ip = fp.mulAdd(f64, sa * sb, @as(f64, @floatFromInt(dot)), ip); + return @floatCast(ip); +} + +// ── Dequantize into the rotated basis ────────────────────────────── + +pub export fn cbm_rsq_decode(code: *const c.cbm_rsq_code_t, out: [*]f32) void { + for (code.codes, 0..) |b, i| { + out[i * 2] = fp.mulAdd(f32, code.scale, @floatFromInt(b & 0x0F), code.offset); + out[i * 2 + 1] = fp.mulAdd(f32, code.scale, @floatFromInt(b >> 4), code.offset); + } +} diff --git a/src/semantic/semantic.c b/src/semantic/semantic.c deleted file mode 100644 index 8023e70c..00000000 --- a/src/semantic/semantic.c +++ /dev/null @@ -1,1683 +0,0 @@ -/* - * semantic.c — Algorithmic code embeddings: TF-IDF, Random Indexing, - * API/Type/Decorator signatures, combined scoring, graph diffusion. - * - * All signals computed from graph buffer metadata — no source file reads. - * Uses xxHash for deterministic random vectors. Pure C, zero dependencies. - */ -#include "semantic/semantic.h" -#include "foundation/constants.h" -#include "foundation/hash_table.h" -#include "foundation/log.h" -#include "foundation/profile.h" -#include "foundation/platform.h" -#include "foundation/compat_thread.h" -#include "pipeline/worker_pool.h" -#include "simhash/minhash.h" -#include "nomic/code_vectors.h" - -#define XXH_INLINE_ALL -#include "xxhash/xxhash.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -/* ── Constants ───────────────────────────────────────────────────── */ - -enum { - TOKEN_BUF_LEN = 128, - CORPUS_INIT_CAP = 4096, - DOC_TOKENS_INIT = 64, - RI_SEED_BASE = 0x52494E44, /* "RIND" */ -}; - -/* Default signal weights for cbm_sem_combined_score. Must sum to ~1.0; - * proximity is a multiplier applied on top. */ -#define CBM_SEM_W_TFIDF 0.20F -#define CBM_SEM_W_RI 0.25F -#define CBM_SEM_W_MINHASH 0.10F -#define CBM_SEM_W_API 0.15F -#define CBM_SEM_W_TYPE 0.10F -#define CBM_SEM_W_DECORATOR 0.05F -#define CBM_SEM_W_STRUCT_PROFILE 0.10F -#define CBM_SEM_W_DATAFLOW 0.05F - -/* Threshold bounds for CBM_SEMANTIC_THRESHOLD env override. */ -#define CBM_SEM_THRESHOLD_MIN 0.0F -#define CBM_SEM_THRESHOLD_MAX 1.0F - -/* Epsilon for denominator guards in cosine math. */ -#define CBM_SEM_DENOM_EPS 1e-10F - -/* Int8 quantization bounds for vector storage (maps [-1.0, 1.0] to [-127, 127]). */ -#define CBM_SEM_INT8_MAX 127.0F -#define CBM_SEM_INT8_MIN (-127.0F) - -/* Unit vector bounds (normalized similarity / cosine value limits). */ -#define CBM_SEM_UNIT_POS 1.0F -#define CBM_SEM_UNIT_NEG (-1.0F) - -/* Proximity boost: same-file gets 1.0 + CBM_SEM_PROX_MAX_BOOST, distant gets 1.0. */ -#define CBM_SEM_PROX_MAX_BOOST 0.10F - -/* Reflective Random Indexing blend factors for pass2 (context mixing). */ -#define CBM_SEM_RRI_ALPHA 0.3F -#define CBM_SEM_RRI_BETA 0.7F - -/* IDF smoothing constant (log base): log2(1 + docs / doc_freq). */ -#define CBM_SEM_IDF_BASE 0.5F - -/* Strings of exactly two parts / dimensions used for co-occurrence window math. */ -#define CBM_SEM_COOCCUR_STRIDE 2 - -/* Worker tile/chunk sizes for parallel cooccurrence passes. */ -enum { - CBM_SEM_WORKER_STACK_CAP = 256, - CBM_SEM_TILE_SIZE = 40, - CBM_SEM_SEEN_INIT_CAP = 256, - CBM_SEM_RESOLVE_CHUNK = 64, - CBM_SEM_COOCCUR_CHUNK = 32, - CBM_SEM_RRI_TILE = 128, - CBM_SEM_INT8_I_LO = -128, - CBM_SEM_INT8_I_HI = 127, - CBM_SEM_ATOMIC_INC = 1, - CBM_SEM_RI_NONZERO_COUNT = 8, -}; - -/* Mutex-init state machine for the lazy-initialized pretrained token map. - * State 0 = not started, 1 = initializing, 2 = initialized. */ -enum { - MTX_STATE_UNINIT = 0, - MTX_STATE_INITIALIZING = 1, - MTX_STATE_READY = 2, -}; - -/* Pretrained map ready flag (signals a one-shot atomic transition 0 → 1). */ -enum { MAP_READY = 1 }; - -/* Numeric conversion radix for strtol (base 10 decimal). */ -enum { BASE_DECIMAL = 10 }; - -/* ── Configuration ───────────────────────────────────────────────── */ - -cbm_sem_config_t cbm_sem_get_config(void) { - cbm_sem_config_t cfg = { - .w_tfidf = CBM_SEM_W_TFIDF, - .w_ri = CBM_SEM_W_RI, - .w_minhash = CBM_SEM_W_MINHASH, - .w_api = CBM_SEM_W_API, - .w_type = CBM_SEM_W_TYPE, - .w_decorator = CBM_SEM_W_DECORATOR, - .w_struct_profile = CBM_SEM_W_STRUCT_PROFILE, - .w_dataflow = CBM_SEM_W_DATAFLOW, - .threshold = (float)CBM_SEM_EDGE_THRESHOLD, - .max_edges = CBM_SEM_MAX_EDGES, - }; - const char *thresh = getenv("CBM_SEMANTIC_THRESHOLD"); - if (thresh) { - /* strtod reports errors via endptr; reject non-numeric input silently. */ - char *end = NULL; - double parsed = strtod(thresh, &end); - if (end != thresh && parsed > CBM_SEM_THRESHOLD_MIN && parsed <= CBM_SEM_THRESHOLD_MAX) { - cfg.threshold = (float)parsed; - } - } - return cfg; -} - -bool cbm_sem_is_enabled(void) { - const char *val = getenv("CBM_SEMANTIC_ENABLED"); - return val && val[0] == '1'; -} - -/* ── Token extraction ────────────────────────────────────────────── */ - -/* True for characters that terminate a token regardless of case. */ -static bool is_token_delim(char c) { - return c == '.' || c == '/' || c == '_' || c == '-' || c == ' ' || c == '(' || c == ')' || - c == ',' || c == ':'; -} - -/* True for a camelCase transition: uppercase letter preceded by a lowercase. */ -static bool is_camel_break(const char *name, int i) { - if (i <= 0) { - return false; - } - char c = name[i]; - char p = name[i - SKIP_ONE]; - return c >= 'A' && c <= 'Z' && p >= 'a' && p <= 'z'; -} - -/* Flush the current buffer as a token into out[]. */ -static void flush_token(char *buf, int *blen, char **out, int *count, int max_out) { - if (*blen > 0 && *count < max_out) { - buf[*blen] = '\0'; - out[(*count)++] = strdup(buf); - } - *blen = 0; -} - -int cbm_sem_tokenize(const char *name, char **out, int max_out) { - if (!name || !out || max_out <= 0) { - return 0; - } - int count = 0; - char buf[TOKEN_BUF_LEN]; - int blen = 0; - - for (int i = 0; name[i] && count < max_out; i++) { - char c = name[i]; - bool split = is_token_delim(c); - bool camel = is_camel_break(name, i); - if (split || camel) { - flush_token(buf, &blen, out, &count, max_out); - if (split) { - continue; - } - } - if (blen < TOKEN_BUF_LEN - SKIP_ONE && isalnum((unsigned char)c)) { - buf[blen++] = (char)tolower((unsigned char)c); - } - } - flush_token(buf, &blen, out, &count, max_out); - - /* Abbreviation expansion: add expanded forms for common code abbreviations. - * "err" → also add "error", "ctx" → "context", etc. */ - /* Cross-language abbreviation table — covers Go, Python, JS/TS, Rust, - * Java, C/C++, Ruby, PHP, Kotlin, Swift, Scala, C#, and common patterns. */ - static const struct { - const char *abbrev; - const char *expanded; - } abbrevs[] = { - /* Error/exception handling */ - {"err", "error"}, - {"exc", "exception"}, - {"ex", "exception"}, - /* Context/config */ - {"ctx", "context"}, - {"cfg", "config"}, - {"conf", "configuration"}, - {"env", "environment"}, - {"opt", "option"}, - {"opts", "options"}, - /* Request/response (HTTP, RPC) */ - {"req", "request"}, - {"res", "response"}, - {"resp", "response"}, - {"rsp", "response"}, - {"hdr", "header"}, - {"hdrs", "headers"}, - /* Strings/formatting */ - {"str", "string"}, - {"fmt", "format"}, - {"msg", "message"}, - {"txt", "text"}, - {"lbl", "label"}, - {"desc", "description"}, - /* Data structures */ - {"buf", "buffer"}, - {"arr", "array"}, - {"vec", "vector"}, - {"lst", "list"}, - {"dict", "dictionary"}, - {"tbl", "table"}, - {"stk", "stack"}, - {"que", "queue"}, - /* Functions/callbacks */ - {"fn", "function"}, - {"func", "function"}, - {"cb", "callback"}, - {"proc", "procedure"}, - {"ctor", "constructor"}, - {"dtor", "destructor"}, - /* Database/storage */ - {"db", "database"}, - {"col", "column"}, - {"tbl", "table"}, - {"stmt", "statement"}, - {"txn", "transaction"}, - {"trx", "transaction"}, - {"repo", "repository"}, - /* Auth/security */ - {"auth", "authentication"}, - {"authz", "authorization"}, - {"perm", "permission"}, - {"cred", "credential"}, - {"tok", "token"}, - {"pwd", "password"}, - /* Values/types */ - {"val", "value"}, - {"num", "number"}, - {"int", "integer"}, - {"bool", "boolean"}, - {"flt", "float"}, - {"dbl", "double"}, - /* Indexing/iteration */ - {"idx", "index"}, - {"iter", "iterator"}, - {"elem", "element"}, - {"cnt", "count"}, - {"len", "length"}, - {"sz", "size"}, - {"pos", "position"}, - {"off", "offset"}, - {"cap", "capacity"}, - /* Lifecycle */ - {"init", "initialize"}, - {"deinit", "deinitialize"}, - {"alloc", "allocate"}, - {"dealloc", "deallocate"}, - {"del", "delete"}, - {"rm", "remove"}, - /* Implementation/interface */ - {"impl", "implementation"}, - {"iface", "interface"}, - {"abs", "abstract"}, - {"decl", "declaration"}, - /* Parameters/attributes */ - {"param", "parameter"}, - {"arg", "argument"}, - {"attr", "attribute"}, - {"prop", "property"}, - {"ret", "return"}, - /* Source/destination */ - {"src", "source"}, - {"dst", "destination"}, - {"tgt", "target"}, - {"orig", "original"}, - {"prev", "previous"}, - {"cur", "current"}, - {"tmp", "temporary"}, - {"temp", "temporary"}, - /* Networking/IO */ - {"conn", "connection"}, - {"sess", "session"}, - {"sock", "socket"}, - {"addr", "address"}, - {"url", "uniform"}, - {"srv", "server"}, - {"cli", "client"}, - {"svc", "service"}, - {"ep", "endpoint"}, - /* Management */ - {"mgr", "manager"}, - {"ctrl", "controller"}, - {"hdlr", "handler"}, - {"sched", "scheduler"}, - {"disp", "dispatcher"}, - {"reg", "registry"}, - /* Async/concurrent */ - {"chan", "channel"}, - {"sem", "semaphore"}, - {"mtx", "mutex"}, - {"wg", "waitgroup"}, - {"sig", "signal"}, - {"evt", "event"}, - {"sub", "subscriber"}, - {"pub", "publisher"}, - /* Testing */ - {"spec", "specification"}, - {"mock", "mock"}, - {"stub", "stub"}, - {"assert", "assertion"}, - /* Logging/monitoring */ - {"log", "logging"}, - {"lvl", "level"}, - {"dbg", "debug"}, - {"wrn", "warning"}, - {"inf", "info"}, - /* Time */ - {"ts", "timestamp"}, - {"dur", "duration"}, - {"ttl", "timetolive"}, - /* Miscellaneous */ - {"ver", "version"}, - {"ns", "namespace"}, - {"pkg", "package"}, - {"mod", "module"}, - {"lib", "library"}, - {"dep", "dependency"}, - {"ref", "reference"}, - {"ptr", "pointer"}, - {"obj", "object"}, - {"doc", "document"}, - {"cmd", "command"}, - {"ops", "operations"}, - {"util", "utility"}, - {"hlp", "helper"}, - {"ext", "extension"}, - {NULL, NULL}, - }; - int orig_count = count; - for (int t = 0; t < orig_count && count < max_out; t++) { - for (int a = 0; abbrevs[a].abbrev; a++) { - if (strcmp(out[t], abbrevs[a].abbrev) == 0) { - out[count++] = strdup(abbrevs[a].expanded); - break; - } - } - } - - return count; -} - -/* ── Dense vector operations ─────────────────────────────────────── */ - -float cbm_sem_cosine(const cbm_sem_vec_t *a, const cbm_sem_vec_t *b) { - if (!a || !b) { - return 0.0F; - } - float dot = 0.0F; - float mag_a = 0.0F; - float mag_b = 0.0F; - for (int i = 0; i < CBM_SEM_DIM; i++) { - dot += a->v[i] * b->v[i]; - mag_a += a->v[i] * a->v[i]; - mag_b += b->v[i] * b->v[i]; - } - float denom = sqrtf(mag_a) * sqrtf(mag_b); - if (denom < CBM_SEM_DENOM_EPS) { - return 0.0F; - } - return dot / denom; -} - -/* Pretrained token lookup table — built lazily on first use. */ -static CBMHashTable *g_pretrained_map = NULL; -static _Atomic int g_pretrained_ready = 0; -static cbm_mutex_t g_pretrained_mtx; -static _Atomic int g_pretrained_mtx_init = 0; - -/* Thread-safe lazy init of the pretrained token lookup map. - * Uses double-checked locking: fast path reads an atomic flag. */ -static void ensure_pretrained_map(void) { - if (atomic_load_explicit(&g_pretrained_ready, memory_order_acquire)) { - return; - } - /* First-time init of the mutex itself (also needs to be thread-safe) */ - int expected = MTX_STATE_UNINIT; - if (atomic_compare_exchange_strong_explicit(&g_pretrained_mtx_init, &expected, - MTX_STATE_INITIALIZING, memory_order_acq_rel, - memory_order_acquire)) { - cbm_mutex_init(&g_pretrained_mtx); - atomic_store_explicit(&g_pretrained_mtx_init, MTX_STATE_READY, memory_order_release); - } else { - /* Spin until another thread finishes initializing the mutex */ - while (atomic_load_explicit(&g_pretrained_mtx_init, memory_order_acquire) != - MTX_STATE_READY) { - /* brief spin */ - } - } - cbm_mutex_lock(&g_pretrained_mtx); - if (!atomic_load_explicit(&g_pretrained_ready, memory_order_acquire)) { - g_pretrained_map = cbm_ht_create(PRETRAINED_TOKEN_COUNT); - char idx_buf[CBM_SZ_16]; - for (int i = 0; i < PRETRAINED_TOKEN_COUNT; i++) { - const char *tok = PRETRAINED_TOKENS[i]; - if (tok && tok[0]) { - snprintf(idx_buf, sizeof(idx_buf), "%d", i); - cbm_ht_set(g_pretrained_map, strdup(tok), strdup(idx_buf)); - } - } - atomic_store_explicit(&g_pretrained_ready, MAP_READY, memory_order_release); - } - cbm_mutex_unlock(&g_pretrained_mtx); -} - -void cbm_sem_ensure_ready(void) { - ensure_pretrained_map(); -} - -void cbm_sem_random_index(const char *token, cbm_sem_vec_t *out) { - memset(out, 0, sizeof(*out)); - if (!token) { - return; - } - - /* Try pretrained nomic-embed-code vector first (768d, distilled from 7B). */ - ensure_pretrained_map(); - const char *idx_str = cbm_ht_get(g_pretrained_map, token); - if (idx_str) { - char *end = NULL; - long idx = strtol(idx_str, &end, BASE_DECIMAL); - if (end != idx_str && idx >= 0 && idx < PRETRAINED_TOKEN_COUNT) { - const int8_t *pvec = pretrained_vec_at((int)idx); - for (int d = 0; d < CBM_SEM_DIM && d < PRETRAINED_DIM; d++) { - out->v[d] = (float)pvec[d] / CBM_SEM_INT8_MAX; - } - return; - } - } - - /* Fallback: sparse random vector for tokens not in pretrained vocab. */ - uint64_t seed = XXH3_64bits(token, strlen(token)); - for (int i = 0; i < CBM_SEM_SPARSE_NNZE; i++) { - uint64_t h = XXH3_64bits_withSeed(&i, sizeof(i), seed + RI_SEED_BASE); - int pos = (int)(h % CBM_SEM_DIM); - float sign = (h & SKIP_ONE) ? CBM_SEM_UNIT_POS : -CBM_SEM_UNIT_POS; - out->v[pos] += sign; - } -} - -void cbm_sem_normalize(cbm_sem_vec_t *v) { - if (!v) { - return; - } - float mag = 0.0F; - for (int i = 0; i < CBM_SEM_DIM; i++) { - mag += v->v[i] * v->v[i]; - } - mag = sqrtf(mag); - if (mag < CBM_SEM_DENOM_EPS) { - return; - } - float inv = CBM_SEM_UNIT_POS / mag; - for (int i = 0; i < CBM_SEM_DIM; i++) { - v->v[i] *= inv; - } -} - -void cbm_sem_vec_add_scaled(cbm_sem_vec_t *dst, const cbm_sem_vec_t *src, float scale) { - if (!dst || !src) { - return; - } - for (int i = 0; i < CBM_SEM_DIM; i++) { - dst->v[i] += scale * src->v[i]; - } -} - -/* ── Corpus (IDF + Random Indexing enrichment) ───────────────────── */ - -typedef struct { - char *token; - int doc_freq; - cbm_sem_vec_t enriched_vec; /* context-enriched via co-occurrence */ -} corpus_entry_t; - -struct cbm_sem_corpus { - CBMHashTable *token_map; /* token → index into entries[] */ - corpus_entry_t *entries; - int entry_count; - int entry_cap; - int doc_count; - bool finalized; - - /* Per-document token lists for co-occurrence pass */ - int **doc_token_ids; - int *doc_token_counts; - int doc_cap; -}; - -static int corpus_get_or_add(cbm_sem_corpus_t *c, const char *token) { - char idx_buf[CBM_SZ_16]; - const char *existing = cbm_ht_get(c->token_map, token); - if (existing) { - char *end = NULL; - long parsed = strtol(existing, &end, BASE_DECIMAL); - return (end != existing) ? (int)parsed : CBM_NOT_FOUND; - } - if (c->entry_count >= c->entry_cap) { - int new_cap = c->entry_cap < CORPUS_INIT_CAP ? CORPUS_INIT_CAP : c->entry_cap * PAIR_LEN; - corpus_entry_t *grown = realloc(c->entries, (size_t)new_cap * sizeof(corpus_entry_t)); - if (!grown) { - return CBM_NOT_FOUND; - } - c->entries = grown; - c->entry_cap = new_cap; - } - int idx = c->entry_count++; - c->entries[idx].token = strdup(token); - c->entries[idx].doc_freq = 0; - memset(&c->entries[idx].enriched_vec, 0, sizeof(cbm_sem_vec_t)); - snprintf(idx_buf, sizeof(idx_buf), "%d", idx); - cbm_ht_set(c->token_map, strdup(token), strdup(idx_buf)); - return idx; -} - -cbm_sem_corpus_t *cbm_sem_corpus_new(void) { - cbm_sem_corpus_t *c = calloc(SKIP_ONE, sizeof(cbm_sem_corpus_t)); - if (c) { - c->token_map = cbm_ht_create(CORPUS_INIT_CAP); - } - return c; -} - -void cbm_sem_corpus_add_doc(cbm_sem_corpus_t *corpus, const char **tokens, int count) { - if (!corpus || !tokens || count <= 0) { - return; - } - /* Track document for co-occurrence pass */ - if (corpus->doc_count >= corpus->doc_cap) { - int new_cap = - corpus->doc_cap < DOC_TOKENS_INIT ? DOC_TOKENS_INIT : corpus->doc_cap * PAIR_LEN; - int **grown_ids = realloc(corpus->doc_token_ids, (size_t)new_cap * sizeof(int *)); - int *grown_counts = realloc(corpus->doc_token_counts, (size_t)new_cap * sizeof(int)); - if (!grown_ids || !grown_counts) { - free(grown_ids); - free(grown_counts); - return; - } - corpus->doc_token_ids = grown_ids; - corpus->doc_token_counts = grown_counts; - corpus->doc_cap = new_cap; - } - int doc_idx = corpus->doc_count++; - corpus->doc_token_ids[doc_idx] = malloc((size_t)count * sizeof(int)); - corpus->doc_token_counts[doc_idx] = count; - - /* Per-doc unique set for IDF */ - int *seen = calloc((size_t)corpus->entry_cap + (size_t)count + CORPUS_INIT_CAP, sizeof(int)); - int seen_count = 0; - - for (int i = 0; i < count; i++) { - int tid = corpus_get_or_add(corpus, tokens[i]); - corpus->doc_token_ids[doc_idx][i] = tid; - if (tid < 0) { - continue; - } - /* Check uniqueness for IDF (simple linear scan — tokens per doc is small) */ - bool is_new = true; - for (int j = 0; j < seen_count; j++) { - if (seen[j] == tid) { - is_new = false; - break; - } - } - if (is_new) { - seen[seen_count++] = tid; - corpus->entries[tid].doc_freq++; - } - } - free(seen); -} - -/* ── Parallel corpus batch build ──────────────────────────────────── */ -/* Strategy: - * Phase A (SEQUENTIAL): Scan all documents once to build the global - * token_map (inserts unique tokens, assigns global IDs). This is - * inherently sequential (hash table mutation), but much faster than - * the current per-doc add_doc because we avoid the per-doc malloc of - * the `seen` array and per-doc bookkeeping. - * Phase B (PARALLEL): Each worker processes a chunk of docs, translates - * tokens → global IDs via read-only token_map lookups, fills - * doc_token_ids[d], and accumulates doc_freq contributions via atomics. - */ - -typedef struct { - cbm_sem_corpus_t *corpus; - char **all_tokens; - const int *token_counts; - int max_tokens; - int doc_count; - _Atomic int *doc_freq_atomic; /* per-entry atomic counter (entry_count long) */ - _Atomic int next_idx; -} batch_resolve_ctx_t; - -/* Resolve one document: look up each token's global ID, fill the corpus - * doc_token_ids[d], and bump the per-token doc_freq counter atomically. The - * caller is responsible for ensuring `seen` has capacity for `count` ints - * before calling (the worker grows its per-thread scratch buffer). */ -static void batch_resolve_one_doc(batch_resolve_ctx_t *bc, int doc_index, int *seen) { - int count = bc->token_counts[doc_index]; - if (count <= 0) { - bc->corpus->doc_token_ids[doc_index] = NULL; - bc->corpus->doc_token_counts[doc_index] = 0; - return; - } - int *ids = malloc((size_t)count * sizeof(int)); - bc->corpus->doc_token_ids[doc_index] = ids; - bc->corpus->doc_token_counts[doc_index] = count; - - int seen_count = 0; - char **tokens = &bc->all_tokens[(ptrdiff_t)doc_index * bc->max_tokens]; - for (int i = 0; i < count; i++) { - const char *idx_str = cbm_ht_get(bc->corpus->token_map, tokens[i]); - int tid = CBM_NOT_FOUND; - if (idx_str) { - char *end = NULL; - long parsed = strtol(idx_str, &end, BASE_DECIMAL); - if (end != idx_str) { - tid = (int)parsed; - } - } - ids[i] = tid; - if (tid < 0) { - continue; - } - /* Unique-per-doc check for IDF */ - bool is_new = true; - for (int j = 0; j < seen_count; j++) { - if (seen[j] == tid) { - is_new = false; - break; - } - } - if (is_new) { - seen[seen_count++] = tid; - atomic_fetch_add_explicit(&bc->doc_freq_atomic[tid], CBM_SEM_ATOMIC_INC, - memory_order_relaxed); - } - } -} - -static void batch_resolve_worker(int worker_id, void *ctx_ptr) { - (void)worker_id; - batch_resolve_ctx_t *bc = ctx_ptr; - /* Per-worker scratch for unique-per-doc tracking */ - int local_seen_cap = CBM_SEM_SEEN_INIT_CAP; - int *seen = malloc((size_t)local_seen_cap * sizeof(int)); - if (!seen) { - return; - } - - while (true) { - int start = - atomic_fetch_add_explicit(&bc->next_idx, CBM_SEM_RESOLVE_CHUNK, memory_order_relaxed); - if (start >= bc->doc_count) { - break; - } - int end = start + CBM_SEM_RESOLVE_CHUNK; - if (end > bc->doc_count) { - end = bc->doc_count; - } - for (int d = start; d < end; d++) { - int count = bc->token_counts[d]; - if (count > local_seen_cap) { - int *grown = realloc(seen, (size_t)count * sizeof(int)); - if (!grown) { - continue; - } - seen = grown; - local_seen_cap = count; - } - batch_resolve_one_doc(bc, d, seen); - } - } - free(seen); -} - -void cbm_sem_corpus_add_docs_batch(cbm_sem_corpus_t *corpus, char **all_tokens, - const int *token_counts, int doc_count, int max_tokens_per_doc) { - if (!corpus || !all_tokens || !token_counts || doc_count <= 0) { - return; - } - - /* Phase A (SEQUENTIAL): Build token_map and allocate doc arrays. - * Hash table mutation can't be parallelized; strdup+insert is the cost. */ - if (corpus->doc_cap < corpus->doc_count + doc_count) { - int new_cap = corpus->doc_count + doc_count; - int **grown_ids = realloc(corpus->doc_token_ids, (size_t)new_cap * sizeof(int *)); - int *grown_counts = realloc(corpus->doc_token_counts, (size_t)new_cap * sizeof(int)); - if (!grown_ids || !grown_counts) { - free(grown_ids); - free(grown_counts); - return; - } - corpus->doc_token_ids = grown_ids; - corpus->doc_token_counts = grown_counts; - corpus->doc_cap = new_cap; - } - int base_doc = corpus->doc_count; - corpus->doc_count += doc_count; - - for (int d = 0; d < doc_count; d++) { - int count = token_counts[d]; - char **tokens = &all_tokens[(ptrdiff_t)d * max_tokens_per_doc]; - for (int i = 0; i < count; i++) { - /* Inserts token into token_map if new; we discard return here — - * Phase B will re-lookup in read-only mode to get the ID. */ - (void)corpus_get_or_add(corpus, tokens[i]); - } - } - - /* Phase B (PARALLEL): Resolve tokens → IDs and count doc_freq per entry. - * token_map is now read-only; each worker owns its doc range (no writes - * to shared state except atomic doc_freq counters). */ - _Atomic int *doc_freq_atomic = calloc((size_t)corpus->entry_count, sizeof(_Atomic int)); - if (!doc_freq_atomic) { - /* OOM fallback: sequential path. Roll back doc_count first since - * add_doc increments it itself. */ - corpus->doc_count = base_doc; - for (int d = 0; d < doc_count; d++) { - int count = token_counts[d]; - char **tokens = &all_tokens[(ptrdiff_t)d * max_tokens_per_doc]; - cbm_sem_corpus_add_doc(corpus, (const char **)tokens, count); - } - return; - } - - int worker_count = cbm_default_worker_count(false); - batch_resolve_ctx_t bc = { - .corpus = corpus, - .all_tokens = all_tokens, - .token_counts = token_counts, - .max_tokens = max_tokens_per_doc, - .doc_count = doc_count, - .doc_freq_atomic = doc_freq_atomic, - }; - atomic_init(&bc.next_idx, 0); - /* Temporarily re-base doc arrays so workers write to base_doc..base_doc+doc_count */ - corpus->doc_token_ids += base_doc; - corpus->doc_token_counts += base_doc; - cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false}; - cbm_parallel_for(worker_count, batch_resolve_worker, &bc, opts); - corpus->doc_token_ids -= base_doc; - corpus->doc_token_counts -= base_doc; - - /* Phase C (SEQUENTIAL reduce): atomic counters → entries[].doc_freq */ - for (int i = 0; i < corpus->entry_count; i++) { - corpus->entries[i].doc_freq += - atomic_load_explicit(&doc_freq_atomic[i], memory_order_relaxed); - } - free(doc_freq_atomic); -} - -/* ── Parallel corpus_finalize ─────────────────────────────────────── */ -/* Strategy: - * 1. Precompute base RI vectors into a shared array (eliminates ~333M - * redundant cbm_sem_random_index calls on kernel-scale corpora). - * 2. Co-occurrence passes: partition TARGET tokens across workers so each - * worker writes to a disjoint range of enriched_vec (zero contention). - * Each worker still scans all documents but only accumulates for targets - * in its range. Inner vector add is the parallelized work. - * 3. Normalize/blend loops are trivially parallel per-entry. - */ - -/* Reverse index: for each token id, a list of (doc_id, position_in_doc) pairs. - * Built once, reused for both cooccur passes. Eliminates the O(num_chunks × doc_count) - * redundant outer scan in the old algorithm. */ -typedef struct { - int32_t doc_id; - int32_t pos; -} cooccur_pos_t; - -typedef struct { - int *offsets; /* offsets[entry_count + 1], prefix sum of occurrences */ - cooccur_pos_t *flat; /* flat array of positions, total = offsets[entry_count] */ -} reverse_index_t; - -/* Tagged source vector: sparse (~30% of tokens, 8 inline nonzeros) or dense int8 - * reference into PRETRAINED_VECTOR_BLOB (no copy). Dense path converts int8→float - * on the fly in the hot loop. Tested with both int8 and float32 blob storage - * formats — cooccur passes are memory-bandwidth-bound, so the int8 format (4x - * less source traffic) is equivalent in wall time despite the conversion cost, - * while saving 90 MB of binary size. */ -typedef struct { - uint8_t is_sparse; /* 1 = sparse path, 0 = dense int8 reference */ - uint8_t nnz; /* number of nonzeros used in sparse path */ - uint16_t _pad; - uint16_t indices[CBM_SEM_SPARSE_NNZE]; /* 8 * 2 = 16 bytes */ - float values[CBM_SEM_SPARSE_NNZE]; /* 8 * 4 = 32 bytes */ - const int8_t *dense_int8; /* points into PRETRAINED_VECTOR_BLOB */ -} cbm_sem_src_entry_t; - -/* Inline helper: initialize a target vector from a sparse/dense source. */ -static inline void sem_target_init_from_src(cbm_sem_vec_t *dst, const cbm_sem_src_entry_t *src) { - memset(dst, 0, sizeof(*dst)); - if (src->is_sparse) { - for (int k = 0; k < src->nnz; k++) { - dst->v[src->indices[k]] = src->values[k]; - } - } else { - const int8_t *s = src->dense_int8; - const float inv127 = CBM_SEM_UNIT_POS / CBM_SEM_INT8_MAX; - for (int d = 0; d < CBM_SEM_DIM; d++) { - dst->v[d] = inv127 * (float)s[d]; - } - } -} - -/* Inline helper: add weighted source into target. - * Sparse path: ~8 operations, ~48 bytes source memory traffic. - * Dense path: 768 mul-adds with int8→float conversion, ~768 bytes traffic. */ -static inline void sem_vec_add_src_scaled(cbm_sem_vec_t *dst, const cbm_sem_src_entry_t *src, - float scale) { - if (src->is_sparse) { - for (int k = 0; k < src->nnz; k++) { - dst->v[src->indices[k]] += scale * src->values[k]; - } - } else { - const int8_t *s = src->dense_int8; - const float mul = scale * (CBM_SEM_UNIT_POS / CBM_SEM_INT8_MAX); - for (int d = 0; d < CBM_SEM_DIM; d++) { - dst->v[d] += mul * (float)s[d]; - } - } -} - -/* Pass 1 context: uses sparse/int8 tagged sources (most memory-efficient). */ -typedef struct { - corpus_entry_t *entries; - const cbm_sem_src_entry_t *src_entries; /* sparse or int8-dense per token */ - int **doc_token_ids; - const int *doc_token_counts; - const reverse_index_t *rev; - int doc_count; - int entry_count; - _Atomic int next_chunk; - int num_chunks; - int chunk_size; - - /* Cache-blocked tiling parameters */ - int tile_size; /* targets per L2-resident tile */ -} cooccur_sparse_ctx_t; - -/* Accumulate co-occurrence context for a single target token into `target`. - * Reads neighbors within ±CBM_SEM_WINDOW positions across all documents that - * reference this token id via the reverse index. */ -static void cooccur_sparse_one_target(cooccur_sparse_ctx_t *cc, int tid, cbm_sem_vec_t *target) { - int occ_start = cc->rev->offsets[tid]; - int occ_end = cc->rev->offsets[tid + SKIP_ONE]; - /* Frequent-token subsampling: stride over occurrences when a token is very - * common so the enriched vector's direction is preserved (it's normalized - * after) while bounding work to ~CBM_SEM_MAX_OCCUR samples. See semantic.h. */ - int occ_step = SKIP_ONE; - if (occ_end - occ_start > CBM_SEM_MAX_OCCUR) { - occ_step = (occ_end - occ_start) / CBM_SEM_MAX_OCCUR; - } - for (int p = occ_start; p < occ_end; p += occ_step) { - int d = cc->rev->flat[p].doc_id; - int i = cc->rev->flat[p].pos; - int *ids = cc->doc_token_ids[d]; - int len = cc->doc_token_counts[d]; - for (int w = -CBM_SEM_WINDOW; w <= CBM_SEM_WINDOW; w++) { - if (w == 0) { - continue; - } - int j = i + w; - if (j < 0 || j >= len) { - continue; - } - int nid = ids[j]; - if (nid < 0) { - continue; - } - float weight = CBM_SEM_UNIT_POS / (float)abs(w); - sem_vec_add_src_scaled(target, &cc->src_entries[nid], weight); - } - } -} - -static void cooccur_worker_sparse(int worker_id, void *ctx_ptr) { - (void)worker_id; - cooccur_sparse_ctx_t *cc = ctx_ptr; - while (true) { - int ci = - atomic_fetch_add_explicit(&cc->next_chunk, CBM_SEM_ATOMIC_INC, memory_order_relaxed); - if (ci >= cc->num_chunks) { - break; - } - int chunk_start = ci * cc->chunk_size; - int chunk_end = chunk_start + cc->chunk_size; - if (chunk_end > cc->entry_count) { - chunk_end = cc->entry_count; - } - - /* Cache-blocked target tiling: process tile_size targets at a time so - * their vectors stay resident in L2 cache during their accumulation. */ - for (int tile_start = chunk_start; tile_start < chunk_end; tile_start += cc->tile_size) { - int tile_end = tile_start + cc->tile_size; - if (tile_end > chunk_end) { - tile_end = chunk_end; - } - for (int tid = tile_start; tid < tile_end; tid++) { - sem_target_init_from_src(&cc->entries[tid].enriched_vec, &cc->src_entries[tid]); - cooccur_sparse_one_target(cc, tid, &cc->entries[tid].enriched_vec); - } - } - } -} - -/* Pass 2 (RRI) context: uses int8-quantized pass1 vectors as source. - * Pass1 outputs are dense float32 post-normalization, values in [-1,1]. - * We quantize to int8 once (×127) to cut source memory traffic 4x. */ -typedef struct { - corpus_entry_t *entries; - const int8_t *pass1_q; /* [entry_count × CBM_SEM_DIM] int8 quantized pass1 */ - int **doc_token_ids; - const int *doc_token_counts; - const reverse_index_t *rev; - int doc_count; - int entry_count; - _Atomic int next_chunk; - int num_chunks; - int chunk_size; - int tile_size; -} cooccur_int8_ctx_t; - -static inline void sem_vec_add_int8_scaled(cbm_sem_vec_t *dst, const int8_t *src, float scale) { - const float mul = scale * (CBM_SEM_UNIT_POS / CBM_SEM_INT8_MAX); - for (int d = 0; d < CBM_SEM_DIM; d++) { - dst->v[d] += mul * (float)src[d]; - } -} - -/* RRI pass 2 accumulator for a single target token, reading int8-quantized - * pass1 vectors as the source. */ -static void cooccur_int8_one_target(cooccur_int8_ctx_t *cc, int tid, cbm_sem_vec_t *target) { - int occ_start = cc->rev->offsets[tid]; - int occ_end = cc->rev->offsets[tid + SKIP_ONE]; - /* Frequent-token subsampling (see cooccur_sparse_one_target / semantic.h). */ - int occ_step = SKIP_ONE; - if (occ_end - occ_start > CBM_SEM_MAX_OCCUR) { - occ_step = (occ_end - occ_start) / CBM_SEM_MAX_OCCUR; - } - for (int p = occ_start; p < occ_end; p += occ_step) { - int d = cc->rev->flat[p].doc_id; - int i = cc->rev->flat[p].pos; - int *ids = cc->doc_token_ids[d]; - int len = cc->doc_token_counts[d]; - for (int w = -CBM_SEM_WINDOW; w <= CBM_SEM_WINDOW; w++) { - if (w == 0) { - continue; - } - int j = i + w; - if (j < 0 || j >= len) { - continue; - } - int nid = ids[j]; - if (nid < 0) { - continue; - } - float weight = CBM_SEM_UNIT_POS / (float)abs(w); - sem_vec_add_int8_scaled(target, &cc->pass1_q[(size_t)nid * CBM_SEM_DIM], weight); - } - } -} - -static void cooccur_worker_int8(int worker_id, void *ctx_ptr) { - (void)worker_id; - cooccur_int8_ctx_t *cc = ctx_ptr; - while (true) { - int ci = - atomic_fetch_add_explicit(&cc->next_chunk, CBM_SEM_ATOMIC_INC, memory_order_relaxed); - if (ci >= cc->num_chunks) { - break; - } - int chunk_start = ci * cc->chunk_size; - int chunk_end = chunk_start + cc->chunk_size; - if (chunk_end > cc->entry_count) { - chunk_end = cc->entry_count; - } - - for (int tile_start = chunk_start; tile_start < chunk_end; tile_start += cc->tile_size) { - int tile_end = tile_start + cc->tile_size; - if (tile_end > chunk_end) { - tile_end = chunk_end; - } - for (int tid = tile_start; tid < tile_end; tid++) { - /* RRI pass 2 starts from zero (no self-init) */ - memset(&cc->entries[tid].enriched_vec, 0, sizeof(cbm_sem_vec_t)); - cooccur_int8_one_target(cc, tid, &cc->entries[tid].enriched_vec); - } - } - } -} - -typedef struct { - corpus_entry_t *entries; - int entry_count; - _Atomic int next_idx; -} norm_ctx_t; - -static void normalize_worker(int worker_id, void *ctx_ptr) { - (void)worker_id; - norm_ctx_t *nc = ctx_ptr; - while (true) { - int start = atomic_fetch_add_explicit(&nc->next_idx, CBM_SEM_WORKER_STACK_CAP, - memory_order_relaxed); - if (start >= nc->entry_count) { - break; - } - int end = start + CBM_SEM_WORKER_STACK_CAP; - if (end > nc->entry_count) { - end = nc->entry_count; - } - for (int i = start; i < end; i++) { - cbm_sem_normalize(&nc->entries[i].enriched_vec); - } - } -} - -typedef struct { - corpus_entry_t *entries; - const cbm_sem_vec_t *pass1; - int entry_count; - _Atomic int next_idx; -} blend_ctx_t; - -static void blend_worker(int worker_id, void *ctx_ptr) { - (void)worker_id; - blend_ctx_t *bc = ctx_ptr; - while (true) { - int start = atomic_fetch_add_explicit(&bc->next_idx, CBM_SEM_WORKER_STACK_CAP, - memory_order_relaxed); - if (start >= bc->entry_count) { - break; - } - int end = start + CBM_SEM_WORKER_STACK_CAP; - if (end > bc->entry_count) { - end = bc->entry_count; - } - for (int i = start; i < end; i++) { - cbm_sem_normalize(&bc->entries[i].enriched_vec); - for (int d = 0; d < CBM_SEM_DIM; d++) { - bc->entries[i].enriched_vec.v[d] = - (CBM_SEM_RRI_BETA * bc->pass1[i].v[d]) + - (CBM_SEM_RRI_ALPHA * bc->entries[i].enriched_vec.v[d]); - } - } - } -} - -typedef struct { - corpus_entry_t *entries; - cbm_sem_src_entry_t *src_entries; - int entry_count; - _Atomic int next_idx; -} src_build_ctx_t; - -/* Build one src_entry for a token: dense float32 reference if in nomic vocab, - * sparse inline representation otherwise. Collisions in the sparse hash are - * merged and zeros filtered so the final representation is exactly the same - * mathematical vector that the old dense path produced. */ -static void build_src_entry(const char *token, cbm_sem_src_entry_t *out) { - memset(out, 0, sizeof(*out)); - if (!token) { - out->is_sparse = SKIP_ONE; - out->nnz = 0; - return; - } - /* Dense path: direct int8 pointer into pretrained blob (zero-copy). */ - const char *idx_str = cbm_ht_get(g_pretrained_map, token); - if (idx_str) { - char *end = NULL; - long idx = strtol(idx_str, &end, BASE_DECIMAL); - if (end != idx_str && idx >= 0 && idx < PRETRAINED_TOKEN_COUNT) { - out->is_sparse = 0; - out->dense_int8 = pretrained_vec_at((int)idx); - return; - } - } - /* Sparse path: compute 8 hash positions with collision merging. */ - out->is_sparse = SKIP_ONE; - uint16_t tmp_idx[CBM_SEM_SPARSE_NNZE]; - float tmp_val[CBM_SEM_SPARSE_NNZE]; - int count = 0; - uint64_t seed = XXH3_64bits(token, strlen(token)); - for (int i = 0; i < CBM_SEM_SPARSE_NNZE; i++) { - uint64_t h = XXH3_64bits_withSeed(&i, sizeof(i), seed + RI_SEED_BASE); - int pos = (int)(h % CBM_SEM_DIM); - float sign = (h & SKIP_ONE) ? CBM_SEM_UNIT_POS : -CBM_SEM_UNIT_POS; - /* Merge collisions */ - int found = CBM_NOT_FOUND; - for (int j = 0; j < count; j++) { - if (tmp_idx[j] == (uint16_t)pos) { - found = j; - break; - } - } - if (found >= 0) { - tmp_val[found] += sign; - } else { - tmp_idx[count] = (uint16_t)pos; - tmp_val[count] = sign; - count++; - } - } - /* Filter zeros */ - int nnz = 0; - for (int j = 0; j < count; j++) { - if (tmp_val[j] != 0.0F) { - out->indices[nnz] = tmp_idx[j]; - out->values[nnz] = tmp_val[j]; - nnz++; - } - } - out->nnz = (uint8_t)nnz; -} - -static void src_build_worker(int worker_id, void *ctx_ptr) { - (void)worker_id; - src_build_ctx_t *sc = ctx_ptr; - while (true) { - int start = atomic_fetch_add_explicit(&sc->next_idx, CBM_SEM_WORKER_STACK_CAP, - memory_order_relaxed); - if (start >= sc->entry_count) { - break; - } - int end = start + CBM_SEM_WORKER_STACK_CAP; - if (end > sc->entry_count) { - end = sc->entry_count; - } - for (int i = start; i < end; i++) { - build_src_entry(sc->entries[i].token, &sc->src_entries[i]); - } - } -} - -/* Quantize pass1 float vectors to int8 (× 127) for use as pass2 source. - * Input vectors are unit-normalized, so values are in [-1, 1] → int8 preserves - * ~99% precision. 4x less memory traffic for pass2. */ -typedef struct { - const corpus_entry_t *entries; - int8_t *pass1_q; - int entry_count; - _Atomic int next_idx; -} pass1_quant_ctx_t; - -static void pass1_quantize_worker(int worker_id, void *ctx_ptr) { - (void)worker_id; - pass1_quant_ctx_t *qc = ctx_ptr; - while (true) { - int start = - atomic_fetch_add_explicit(&qc->next_idx, CBM_SEM_RRI_TILE, memory_order_relaxed); - if (start >= qc->entry_count) { - break; - } - int end = start + CBM_SEM_RRI_TILE; - if (end > qc->entry_count) { - end = qc->entry_count; - } - for (int i = start; i < end; i++) { - const cbm_sem_vec_t *src = &qc->entries[i].enriched_vec; - int8_t *dst = &qc->pass1_q[(size_t)i * CBM_SEM_DIM]; - for (int d = 0; d < CBM_SEM_DIM; d++) { - float v = src->v[d] * CBM_SEM_INT8_MAX; - if (v > CBM_SEM_INT8_MAX) { - v = CBM_SEM_INT8_MAX; - } - if (v < -CBM_SEM_INT8_MAX) { - v = -CBM_SEM_INT8_MAX; - } - dst[d] = (int8_t)(v >= 0 ? v + CBM_SEM_IDF_BASE : v - CBM_SEM_IDF_BASE); - } - } - } -} - -/* Build reverse index: token_id → list of (doc_id, position) pairs. - * SEQUENTIAL (fast: just pointer arithmetic + flat array fill). */ -static reverse_index_t *build_reverse_index(cbm_sem_corpus_t *corpus) { - reverse_index_t *rev = calloc(SKIP_ONE, sizeof(reverse_index_t)); - if (!rev) { - return NULL; - } - /* Phase A: count occurrences per token */ - int *counts = calloc((size_t)corpus->entry_count + SKIP_ONE, sizeof(int)); - if (!counts) { - free(rev); - return NULL; - } - long total = 0; - for (int d = 0; d < corpus->doc_count; d++) { - int *ids = corpus->doc_token_ids[d]; - int len = corpus->doc_token_counts[d]; - for (int i = 0; i < len; i++) { - int tid = ids[i]; - if (tid >= 0 && tid < corpus->entry_count) { - counts[tid]++; - total++; - } - } - } - /* Phase B: exclusive prefix sum → offsets[] */ - rev->offsets = malloc(((size_t)corpus->entry_count + SKIP_ONE) * sizeof(int)); - if (!rev->offsets) { - free(counts); - free(rev); - return NULL; - } - int running = 0; - for (int t = 0; t < corpus->entry_count; t++) { - rev->offsets[t] = running; - running += counts[t]; - counts[t] = 0; /* reuse as per-token fill cursor */ - } - rev->offsets[corpus->entry_count] = running; - /* Phase C: fill flat array. Ensure allocation size > 0 even for empty - * corpora (avoids malloc(0) which is implementation-defined). */ - size_t flat_bytes = (total > 0 ? (size_t)total : SKIP_ONE) * sizeof(cooccur_pos_t); - rev->flat = malloc(flat_bytes); - if (!rev->flat) { - free(rev->offsets); - free(counts); - free(rev); - return NULL; - } - for (int d = 0; d < corpus->doc_count; d++) { - int *ids = corpus->doc_token_ids[d]; - int len = corpus->doc_token_counts[d]; - for (int i = 0; i < len; i++) { - int tid = ids[i]; - if (tid >= 0 && tid < corpus->entry_count) { - int slot = rev->offsets[tid] + counts[tid]++; - rev->flat[slot].doc_id = (int32_t)d; - rev->flat[slot].pos = (int32_t)i; - } - } - } - free(counts); - return rev; -} - -static void free_reverse_index(reverse_index_t *rev) { - if (!rev) { - return; - } - free(rev->offsets); - free(rev->flat); - free(rev); -} - -/* Bundle of parameters shared by the finalize sub-phases. */ -typedef struct { - cbm_sem_corpus_t *corpus; - reverse_index_t *rev; - cbm_sem_src_entry_t *src_entries; - int worker_count; - int num_chunks; - int chunk_size; - int tile_size; - cbm_parallel_for_opts_t opts; -} finalize_params_t; - -/* Sub-phase 1: build tagged source vectors (sparse or dense-int8) in parallel. */ -static void finalize_build_sources(finalize_params_t *p) { - src_build_ctx_t sc = { - .entries = p->corpus->entries, - .src_entries = p->src_entries, - .entry_count = p->corpus->entry_count, - }; - atomic_init(&sc.next_idx, 0); - cbm_parallel_for(p->worker_count, src_build_worker, &sc, p->opts); -} - -/* Sub-phases 2+3: co-occurrence pass 1 + normalize. */ -static void finalize_pass1(finalize_params_t *p) { - cooccur_sparse_ctx_t cc = { - .entries = p->corpus->entries, - .src_entries = p->src_entries, - .doc_token_ids = p->corpus->doc_token_ids, - .doc_token_counts = p->corpus->doc_token_counts, - .rev = p->rev, - .doc_count = p->corpus->doc_count, - .entry_count = p->corpus->entry_count, - .num_chunks = p->num_chunks, - .chunk_size = p->chunk_size, - .tile_size = p->tile_size, - }; - atomic_init(&cc.next_chunk, 0); - cbm_parallel_for(p->worker_count, cooccur_worker_sparse, &cc, p->opts); - - norm_ctx_t nc = {.entries = p->corpus->entries, .entry_count = p->corpus->entry_count}; - atomic_init(&nc.next_idx, 0); - cbm_parallel_for(p->worker_count, normalize_worker, &nc, p->opts); -} - -/* Sub-phases 4+5: quantize pass1 to int8, run RRI pass 2, blend + normalize. */ -static void finalize_pass2(finalize_params_t *p) { - int8_t *pass1_q = malloc((size_t)p->corpus->entry_count * CBM_SEM_DIM * sizeof(int8_t)); - if (pass1_q) { - pass1_quant_ctx_t qc = { - .entries = p->corpus->entries, - .pass1_q = pass1_q, - .entry_count = p->corpus->entry_count, - }; - atomic_init(&qc.next_idx, 0); - cbm_parallel_for(p->worker_count, pass1_quantize_worker, &qc, p->opts); - } - - cbm_sem_vec_t *pass1 = malloc((size_t)p->corpus->entry_count * sizeof(cbm_sem_vec_t)); - if (pass1) { - for (int i = 0; i < p->corpus->entry_count; i++) { - pass1[i] = p->corpus->entries[i].enriched_vec; - } - } - - if (pass1_q) { - cooccur_int8_ctx_t cc = { - .entries = p->corpus->entries, - .pass1_q = pass1_q, - .doc_token_ids = p->corpus->doc_token_ids, - .doc_token_counts = p->corpus->doc_token_counts, - .rev = p->rev, - .doc_count = p->corpus->doc_count, - .entry_count = p->corpus->entry_count, - .num_chunks = p->num_chunks, - .chunk_size = p->chunk_size, - .tile_size = p->tile_size, - }; - atomic_init(&cc.next_chunk, 0); - cbm_parallel_for(p->worker_count, cooccur_worker_int8, &cc, p->opts); - } - - if (pass1) { - blend_ctx_t bc = { - .entries = p->corpus->entries, - .pass1 = pass1, - .entry_count = p->corpus->entry_count, - }; - atomic_init(&bc.next_idx, 0); - cbm_parallel_for(p->worker_count, blend_worker, &bc, p->opts); - free(pass1); - } - free(pass1_q); - - norm_ctx_t nc = {.entries = p->corpus->entries, .entry_count = p->corpus->entry_count}; - atomic_init(&nc.next_idx, 0); - cbm_parallel_for(p->worker_count, normalize_worker, &nc, p->opts); -} - -void cbm_sem_corpus_finalize(cbm_sem_corpus_t *corpus) { - if (!corpus || corpus->finalized) { - return; - } - - /* Eager init before parallel dispatch to avoid lazy-init races */ - ensure_pretrained_map(); - - int worker_count = cbm_default_worker_count(false); - cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false}; - - /* Finer chunks = better load balancing for skewed token distributions. */ - int num_chunks = worker_count * CBM_SEM_COOCCUR_CHUNK; - if (num_chunks > corpus->entry_count) { - num_chunks = corpus->entry_count; - } - if (num_chunks < SKIP_ONE) { - num_chunks = SKIP_ONE; - } - int chunk_size = (corpus->entry_count + num_chunks - SKIP_ONE) / num_chunks; - - reverse_index_t *rev = build_reverse_index(corpus); - if (!rev) { - corpus->finalized = true; - return; - } - cbm_sem_src_entry_t *src_entries = - calloc((size_t)corpus->entry_count, sizeof(cbm_sem_src_entry_t)); - if (!src_entries) { - free_reverse_index(rev); - corpus->finalized = true; - return; - } - - finalize_params_t params = { - .corpus = corpus, - .rev = rev, - .src_entries = src_entries, - .worker_count = worker_count, - .num_chunks = num_chunks, - .chunk_size = chunk_size, - .tile_size = CBM_SEM_TILE_SIZE, - .opts = opts, - }; - finalize_build_sources(¶ms); - finalize_pass1(¶ms); - finalize_pass2(¶ms); - - free(src_entries); - free_reverse_index(rev); - corpus->finalized = true; -} - -/* Parse a decimal index string via strtol, returning CBM_NOT_FOUND on parse - * failure. Centralizes the atoi-replacement pattern for token_map lookups. */ -static int parse_token_index(const char *idx_str) { - if (!idx_str) { - return CBM_NOT_FOUND; - } - char *end = NULL; - long parsed = strtol(idx_str, &end, BASE_DECIMAL); - return (end != idx_str) ? (int)parsed : CBM_NOT_FOUND; -} - -float cbm_sem_corpus_idf(const cbm_sem_corpus_t *corpus, const char *token) { - if (!corpus || !token || corpus->doc_count == 0) { - return 0.0F; - } - int idx = parse_token_index(cbm_ht_get(corpus->token_map, token)); - if (idx < 0 || idx >= corpus->entry_count) { - return 0.0F; - } - int df = corpus->entries[idx].doc_freq; - if (df <= 0) { - return 0.0F; - } - return logf((float)corpus->doc_count / (float)df); -} - -const cbm_sem_vec_t *cbm_sem_corpus_ri_vec(const cbm_sem_corpus_t *corpus, const char *token) { - if (!corpus || !token) { - return NULL; - } - int idx = parse_token_index(cbm_ht_get(corpus->token_map, token)); - if (idx < 0 || idx >= corpus->entry_count) { - return NULL; - } - return &corpus->entries[idx].enriched_vec; -} - -int cbm_sem_corpus_doc_count(const cbm_sem_corpus_t *corpus) { - return corpus ? corpus->doc_count : 0; -} - -int cbm_sem_corpus_token_count(const cbm_sem_corpus_t *corpus) { - return corpus ? corpus->entry_count : 0; -} - -const char *cbm_sem_corpus_token_at(const cbm_sem_corpus_t *corpus, int index, - const cbm_sem_vec_t **out_vec, float *out_idf) { - if (!corpus || index < 0 || index >= corpus->entry_count) { - return NULL; - } - if (out_vec) { - *out_vec = &corpus->entries[index].enriched_vec; - } - if (out_idf && corpus->doc_count > 0) { - int df = corpus->entries[index].doc_freq; - *out_idf = df > 0 ? logf((float)corpus->doc_count / (float)df) : 0.0F; - } - return corpus->entries[index].token; -} - -static void free_ht_kv(const char *key, void *value, void *userdata) { - (void)userdata; - free((void *)key); - free(value); -} - -void cbm_sem_corpus_free(cbm_sem_corpus_t *corpus) { - if (!corpus) { - return; - } - for (int i = 0; i < corpus->entry_count; i++) { - free(corpus->entries[i].token); - } - free(corpus->entries); - for (int d = 0; d < corpus->doc_count; d++) { - free(corpus->doc_token_ids[d]); - } - free(corpus->doc_token_ids); - free(corpus->doc_token_counts); - if (corpus->token_map) { - cbm_ht_foreach(corpus->token_map, free_ht_kv, NULL); - cbm_ht_free(corpus->token_map); - } - free(corpus); -} - -/* ── Combined scoring ────────────────────────────────────────────── */ - -float cbm_sem_proximity(const char *path_a, const char *path_b) { - if (!path_a || !path_b) { - return CBM_SEM_UNIT_POS; - } - /* Count shared directory components */ - int shared = 0; - int total_a = 0; - int total_b = 0; - const char *a = path_a; - const char *b = path_b; - while (*a && *b && *a == *b) { - if (*a == '/') { - shared++; - } - a++; - b++; - } - for (const char *p = path_a; *p; p++) { - if (*p == '/') { - total_a++; - } - } - for (const char *p = path_b; *p; p++) { - if (*p == '/') { - total_b++; - } - } - int max_total = total_a > total_b ? total_a : total_b; - if (max_total == 0) { - return CBM_SEM_UNIT_POS; - } - float ratio = (float)shared / (float)max_total; - /* Same file = 1.10, same dir = 1.05, distant = 1.00 */ - return CBM_SEM_UNIT_POS + (ratio * CBM_SEM_PROX_MAX_BOOST); -} - -/* Cosine similarity for small float arrays (AST profile). */ -static float small_cosine(const float *a, const float *b, int dims) { - float dot = 0.0F; - float ma = 0.0F; - float mb = 0.0F; - for (int i = 0; i < dims; i++) { - dot += a[i] * b[i]; - ma += a[i] * a[i]; - mb += b[i] * b[i]; - } - float denom = sqrtf(ma) * sqrtf(mb); - return denom < CBM_SEM_DENOM_EPS ? 0.0F : dot / denom; -} - -/* Sparse cosine over two pre-sorted (index, weight) vectors. Returns 0 when - * either side is empty or the magnitude product is below the epsilon guard. */ -static float sparse_tfidf_cosine(const cbm_sem_func_t *a, const cbm_sem_func_t *b) { - if (a->tfidf_len <= 0 || b->tfidf_len <= 0) { - return 0.0F; - } - float dot = 0.0F; - float ma = 0.0F; - float mb = 0.0F; - int ia = 0; - int ib = 0; - while (ia < a->tfidf_len && ib < b->tfidf_len) { - if (a->tfidf_indices[ia] == b->tfidf_indices[ib]) { - dot += a->tfidf_weights[ia] * b->tfidf_weights[ib]; - ia++; - ib++; - } else if (a->tfidf_indices[ia] < b->tfidf_indices[ib]) { - ia++; - } else { - ib++; - } - } - for (int i = 0; i < a->tfidf_len; i++) { - ma += a->tfidf_weights[i] * a->tfidf_weights[i]; - } - for (int i = 0; i < b->tfidf_len; i++) { - mb += b->tfidf_weights[i] * b->tfidf_weights[i]; - } - float denom = sqrtf(ma) * sqrtf(mb); - return denom > CBM_SEM_DENOM_EPS ? (dot / denom) : 0.0F; -} - -float cbm_sem_combined_score(const cbm_sem_func_t *a, const cbm_sem_func_t *b, - const cbm_sem_config_t *cfg) { - if (!a || !b || !cfg) { - return 0.0F; - } - - /* Short-circuit: if MinHash Jaccard is already above the SIMILAR_TO threshold, - * the pass_similarity pipeline already emitted a SIMILAR_TO edge for this pair. - * Returning 0 here avoids flooding top-k with cross-service copy-paste boilerplate - * (logging_middleware, shared push/pull handlers) that SIMILAR_TO already covers, - * freeing the edge budget for true semantic leaps (vocabulary-bridged relations). */ - if (a->has_minhash && b->has_minhash) { - double early_j = cbm_minhash_jaccard((const cbm_minhash_t *)a->minhash, - (const cbm_minhash_t *)b->minhash); - if (early_j >= CBM_MINHASH_JACCARD_THRESHOLD) { - return 0.0F; - } - } - - float score = cfg->w_tfidf * sparse_tfidf_cosine(a, b); - - /* Signal 2: Random Indexing */ - score += cfg->w_ri * cbm_rsq_ip(&a->ri_code, &b->ri_code); - - /* Signal 3: MinHash Jaccard */ - if (a->has_minhash && b->has_minhash) { - double j = cbm_minhash_jaccard((const cbm_minhash_t *)a->minhash, - (const cbm_minhash_t *)b->minhash); - score += cfg->w_minhash * (float)j; - } - - /* Signal 4: API Signatures */ - score += cfg->w_api * cbm_rsq_ip(&a->api_code, &b->api_code); - - /* Signal 5: Type Signatures */ - score += cfg->w_type * cbm_rsq_ip(&a->type_code, &b->type_code); - - /* Signal 7: Decorator Pattern */ - score += cfg->w_decorator * cbm_rsq_ip(&a->deco_code, &b->deco_code); - - /* Signal 8+9+11: Structural profile + data flow + Halstead */ - float sp_score = small_cosine(a->struct_profile, b->struct_profile, CBM_SEM_AST_PROFILE_DIMS); - score += cfg->w_struct_profile * sp_score; - - /* Signal 6: Module proximity (multiplier, not additive). - * Proximity returns [1.0, 1.10] — a same-file/same-dir boost for ranking. - * Clamp the final product to [0, 1] so the output stays within valid - * cosine-similarity range. Without the clamp, a 0.95 base × 1.10 proximity - * would emit a 1.045 score which violates the semantic of "similarity" and - * breaks downstream consumers that expect a normalized value. */ - score *= cbm_sem_proximity(a->file_path, b->file_path); - if (score > CBM_SEM_UNIT_POS) { - score = CBM_SEM_UNIT_POS; - } - if (score < 0.0F) { - score = 0.0F; - } - - return score; -} - -/* ── Graph diffusion ─────────────────────────────────────────────── */ - -void cbm_sem_diffuse(cbm_sem_vec_t *combined, const cbm_sem_vec_t *neighbors, int neighbor_count, - float alpha) { - if (!combined || !neighbors || neighbor_count <= 0) { - return; - } - /* Blend: combined = (1-α) × combined + α × mean(neighbors) */ - cbm_sem_vec_t mean; - memset(&mean, 0, sizeof(mean)); - for (int n = 0; n < neighbor_count; n++) { - for (int i = 0; i < CBM_SEM_DIM; i++) { - mean.v[i] += neighbors[n].v[i]; - } - } - float inv_n = CBM_SEM_UNIT_POS / (float)neighbor_count; - float one_minus_alpha = CBM_SEM_UNIT_POS - alpha; - for (int i = 0; i < CBM_SEM_DIM; i++) { - combined->v[i] = (one_minus_alpha * combined->v[i]) + (alpha * mean.v[i] * inv_n); - } - cbm_sem_normalize(combined); -} diff --git a/src/semantic/semantic.zig b/src/semantic/semantic.zig new file mode 100644 index 00000000..5520cb31 --- /dev/null +++ b/src/semantic/semantic.zig @@ -0,0 +1,1599 @@ +//! semantic.zig — Algorithmic code embeddings: TF-IDF, Random Indexing, +//! API/Type/Decorator signatures, combined scoring, graph diffusion. +//! +//! Implements src/semantic/semantic.h. All signals computed from graph buffer +//! metadata — no source file reads. Uses xxHash for deterministic random +//! vectors; std.hash.XxHash3 produces the same values as the vendored xxHash. +//! +//! Float expressions go through fp.mulAdd wherever the C code wrote a +//! product plus a term in one expression: see fp.zig. + +const std = @import("std"); +const c = @import("c"); +const fp = @import("fp.zig"); +const interop = @import("../foundation/interop.zig"); + +const XxHash3 = std.hash.XxHash3; +const AtomicInt = std.atomic.Value(c_int); + +const Vec = c.cbm_sem_vec_t; +const Func = c.cbm_sem_func_t; +const Str = [*:0]const u8; +const CStr = [*c]const u8; + +// ── Constants ─────────────────────────────────────────────────────── + +const DIM: comptime_int = c.CBM_SEM_DIM; +const SPARSE_NNZE: comptime_int = c.CBM_SEM_SPARSE_NNZE; +const WINDOW: comptime_int = c.CBM_SEM_WINDOW; +const MAX_OCCUR: comptime_int = c.CBM_SEM_MAX_OCCUR; +const AST_PROFILE_DIMS: comptime_int = c.CBM_SEM_AST_PROFILE_DIMS; +const PRETRAINED_TOKEN_COUNT: comptime_int = c.PRETRAINED_TOKEN_COUNT; +const PRETRAINED_DIM: comptime_int = c.PRETRAINED_DIM; +const NOT_FOUND: c_int = c.CBM_NOT_FOUND; + +const TOKEN_BUF_LEN = 128; +const CORPUS_INIT_CAP = 4096; +const DOC_TOKENS_INIT = 64; +const RI_SEED_BASE = 0x52494E44; // "RIND" + +/// Default signal weights for cbm_sem_combined_score. Must sum to ~1.0; +/// proximity is a multiplier applied on top. +const W_TFIDF: f32 = 0.20; +const W_RI: f32 = 0.25; +const W_MINHASH: f32 = 0.10; +const W_API: f32 = 0.15; +const W_TYPE: f32 = 0.10; +const W_DECORATOR: f32 = 0.05; +const W_STRUCT_PROFILE: f32 = 0.10; +const W_DATAFLOW: f32 = 0.05; + +/// Threshold bounds for CBM_SEMANTIC_THRESHOLD env override. +const THRESHOLD_MIN: f32 = 0.0; +const THRESHOLD_MAX: f32 = 1.0; + +/// Epsilon for denominator guards in cosine math. +const DENOM_EPS: f32 = 1e-10; + +/// Int8 quantization bound for vector storage (maps [-1.0, 1.0] to [-127, 127]). +const INT8_MAX: f32 = 127.0; + +/// Unit vector bound (normalized similarity / cosine value limit). +const UNIT_POS: f32 = 1.0; + +/// Proximity boost: same-file gets 1.0 + PROX_MAX_BOOST, distant gets 1.0. +const PROX_MAX_BOOST: f32 = 0.10; + +/// Reflective Random Indexing blend factors for pass2 (context mixing). +const RRI_ALPHA: f32 = 0.3; +const RRI_BETA: f32 = 0.7; + +/// Rounding offset for the int8 quantization. +const IDF_BASE: f32 = 0.5; + +/// Worker tile/chunk sizes for parallel cooccurrence passes. +const WORKER_STACK_CAP = 256; +const TILE_SIZE = 40; +const SEEN_INIT_CAP = 256; +const RESOLVE_CHUNK = 64; +const COOCCUR_CHUNK = 32; +const RRI_TILE = 128; + +/// Mutex-init state machine for the lazy-initialized pretrained token map. +const MTX_STATE_UNINIT = 0; +const MTX_STATE_INITIALIZING = 1; +const MTX_STATE_READY = 2; + +/// Pretrained map ready flag (signals a one-shot atomic transition 0 → 1). +const MAP_READY = 1; + +/// Numeric conversion radix for strtol (base 10 decimal). +const BASE_DECIMAL = 10; + +/// A C int used as an array index. +inline fn ix(i: c_int) usize { + return @intCast(i); +} + +/// &p[i] for a C pointer. Plain indexing gives a `*allowzero T`, which does +/// not coerce to `*T`. +inline fn at(p: anytype, i: usize) @TypeOf(&p[0..1][0]) { + return &p[i .. i + 1][0]; +} + +// ── Configuration ─────────────────────────────────────────────────── + +pub export fn cbm_sem_get_config() c.cbm_sem_config_t { + var cfg: c.cbm_sem_config_t = .{ + .w_tfidf = W_TFIDF, + .w_ri = W_RI, + .w_minhash = W_MINHASH, + .w_api = W_API, + .w_type = W_TYPE, + .w_decorator = W_DECORATOR, + .w_struct_profile = W_STRUCT_PROFILE, + .w_dataflow = W_DATAFLOW, + .threshold = @floatCast(c.CBM_SEM_EDGE_THRESHOLD), + .max_edges = c.CBM_SEM_MAX_EDGES, + }; + const thresh: CStr = c.getenv("CBM_SEMANTIC_THRESHOLD"); + if (thresh != null) { + // strtod reports errors via endptr; reject non-numeric input silently. + var end: [*c]u8 = null; + const parsed = c.strtod(thresh, &end); + if (end != thresh and parsed > THRESHOLD_MIN and parsed <= THRESHOLD_MAX) { + cfg.threshold = @floatCast(parsed); + } + } + return cfg; +} + +pub export fn cbm_sem_is_enabled() bool { + const val: CStr = c.getenv("CBM_SEMANTIC_ENABLED"); + return val != null and val[0] == '1'; +} + +// ── Token extraction ──────────────────────────────────────────────── + +/// True for characters that terminate a token regardless of case. +fn isTokenDelim(ch: u8) bool { + return switch (ch) { + '.', '/', '_', '-', ' ', '(', ')', ',', ':' => true, + else => false, + }; +} + +/// True for a camelCase transition: uppercase letter preceded by a lowercase. +fn isCamelBreak(name: Str, i: usize) bool { + if (i == 0) return false; + const ch = name[i]; + const p = name[i - 1]; + return ch >= 'A' and ch <= 'Z' and p >= 'a' and p <= 'z'; +} + +/// Flush the current buffer as a token into out[]. +fn flushToken(buf: *[TOKEN_BUF_LEN]u8, blen: *usize, out: [*][*c]u8, count: *c_int, max_out: c_int) void { + if (blen.* > 0 and count.* < max_out) { + buf[blen.*] = 0; + out[ix(count.*)] = c.strdup(buf); + count.* += 1; + } + blen.* = 0; +} + +/// Cross-language abbreviation table — covers Go, Python, JS/TS, Rust, +/// Java, C/C++, Ruby, PHP, Kotlin, Swift, Scala, C#, and common patterns. +const abbrevs = [_][2][:0]const u8{ + // Error/exception handling + .{ "err", "error" }, + .{ "exc", "exception" }, + .{ "ex", "exception" }, + // Context/config + .{ "ctx", "context" }, + .{ "cfg", "config" }, + .{ "conf", "configuration" }, + .{ "env", "environment" }, + .{ "opt", "option" }, + .{ "opts", "options" }, + // Request/response (HTTP, RPC) + .{ "req", "request" }, + .{ "res", "response" }, + .{ "resp", "response" }, + .{ "rsp", "response" }, + .{ "hdr", "header" }, + .{ "hdrs", "headers" }, + // Strings/formatting + .{ "str", "string" }, + .{ "fmt", "format" }, + .{ "msg", "message" }, + .{ "txt", "text" }, + .{ "lbl", "label" }, + .{ "desc", "description" }, + // Data structures + .{ "buf", "buffer" }, + .{ "arr", "array" }, + .{ "vec", "vector" }, + .{ "lst", "list" }, + .{ "dict", "dictionary" }, + .{ "tbl", "table" }, + .{ "stk", "stack" }, + .{ "que", "queue" }, + // Functions/callbacks + .{ "fn", "function" }, + .{ "func", "function" }, + .{ "cb", "callback" }, + .{ "proc", "procedure" }, + .{ "ctor", "constructor" }, + .{ "dtor", "destructor" }, + // Database/storage + .{ "db", "database" }, + .{ "col", "column" }, + .{ "tbl", "table" }, + .{ "stmt", "statement" }, + .{ "txn", "transaction" }, + .{ "trx", "transaction" }, + .{ "repo", "repository" }, + // Auth/security + .{ "auth", "authentication" }, + .{ "authz", "authorization" }, + .{ "perm", "permission" }, + .{ "cred", "credential" }, + .{ "tok", "token" }, + .{ "pwd", "password" }, + // Values/types + .{ "val", "value" }, + .{ "num", "number" }, + .{ "int", "integer" }, + .{ "bool", "boolean" }, + .{ "flt", "float" }, + .{ "dbl", "double" }, + // Indexing/iteration + .{ "idx", "index" }, + .{ "iter", "iterator" }, + .{ "elem", "element" }, + .{ "cnt", "count" }, + .{ "len", "length" }, + .{ "sz", "size" }, + .{ "pos", "position" }, + .{ "off", "offset" }, + .{ "cap", "capacity" }, + // Lifecycle + .{ "init", "initialize" }, + .{ "deinit", "deinitialize" }, + .{ "alloc", "allocate" }, + .{ "dealloc", "deallocate" }, + .{ "del", "delete" }, + .{ "rm", "remove" }, + // Implementation/interface + .{ "impl", "implementation" }, + .{ "iface", "interface" }, + .{ "abs", "abstract" }, + .{ "decl", "declaration" }, + // Parameters/attributes + .{ "param", "parameter" }, + .{ "arg", "argument" }, + .{ "attr", "attribute" }, + .{ "prop", "property" }, + .{ "ret", "return" }, + // Source/destination + .{ "src", "source" }, + .{ "dst", "destination" }, + .{ "tgt", "target" }, + .{ "orig", "original" }, + .{ "prev", "previous" }, + .{ "cur", "current" }, + .{ "tmp", "temporary" }, + .{ "temp", "temporary" }, + // Networking/IO + .{ "conn", "connection" }, + .{ "sess", "session" }, + .{ "sock", "socket" }, + .{ "addr", "address" }, + .{ "url", "uniform" }, + .{ "srv", "server" }, + .{ "cli", "client" }, + .{ "svc", "service" }, + .{ "ep", "endpoint" }, + // Management + .{ "mgr", "manager" }, + .{ "ctrl", "controller" }, + .{ "hdlr", "handler" }, + .{ "sched", "scheduler" }, + .{ "disp", "dispatcher" }, + .{ "reg", "registry" }, + // Async/concurrent + .{ "chan", "channel" }, + .{ "sem", "semaphore" }, + .{ "mtx", "mutex" }, + .{ "wg", "waitgroup" }, + .{ "sig", "signal" }, + .{ "evt", "event" }, + .{ "sub", "subscriber" }, + .{ "pub", "publisher" }, + // Testing + .{ "spec", "specification" }, + .{ "mock", "mock" }, + .{ "stub", "stub" }, + .{ "assert", "assertion" }, + // Logging/monitoring + .{ "log", "logging" }, + .{ "lvl", "level" }, + .{ "dbg", "debug" }, + .{ "wrn", "warning" }, + .{ "inf", "info" }, + // Time + .{ "ts", "timestamp" }, + .{ "dur", "duration" }, + .{ "ttl", "timetolive" }, + // Miscellaneous + .{ "ver", "version" }, + .{ "ns", "namespace" }, + .{ "pkg", "package" }, + .{ "mod", "module" }, + .{ "lib", "library" }, + .{ "dep", "dependency" }, + .{ "ref", "reference" }, + .{ "ptr", "pointer" }, + .{ "obj", "object" }, + .{ "doc", "document" }, + .{ "cmd", "command" }, + .{ "ops", "operations" }, + .{ "util", "utility" }, + .{ "hlp", "helper" }, + .{ "ext", "extension" }, +}; + +pub export fn cbm_sem_tokenize(name_c: ?Str, out_c: ?[*][*c]u8, max_out: c_int) c_int { + const name = name_c orelse return 0; + const out = out_c orelse return 0; + if (max_out <= 0) return 0; + var count: c_int = 0; + var buf: [TOKEN_BUF_LEN]u8 = undefined; + var blen: usize = 0; + + var i: usize = 0; + while (name[i] != 0 and count < max_out) : (i += 1) { + const ch = name[i]; + const split = isTokenDelim(ch); + const camel = isCamelBreak(name, i); + if (split or camel) { + flushToken(&buf, &blen, out, &count, max_out); + if (split) continue; + } + if (blen < TOKEN_BUF_LEN - 1 and std.ascii.isAlphanumeric(ch)) { + buf[blen] = std.ascii.toLower(ch); + blen += 1; + } + } + flushToken(&buf, &blen, out, &count, max_out); + + // Abbreviation expansion: add expanded forms for common code abbreviations. + // "err" → also add "error", "ctx" → "context", etc. + const orig_count = count; + var t: c_int = 0; + while (t < orig_count and count < max_out) : (t += 1) { + const tok = std.mem.span(out[ix(t)]); + for (abbrevs) |ab| { + if (std.mem.eql(u8, tok, ab[0])) { + out[ix(count)] = c.strdup(ab[1]); + count += 1; + break; + } + } + } + + return count; +} + +// ── Dense vector operations ───────────────────────────────────────── + +pub export fn cbm_sem_cosine(a_c: ?*const Vec, b_c: ?*const Vec) f32 { + const a = a_c orelse return 0.0; + const b = b_c orelse return 0.0; + var dot: f32 = 0.0; + var mag_a: f32 = 0.0; + var mag_b: f32 = 0.0; + for (a.v, b.v) |x, y| { + dot = fp.mulAdd(f32, x, y, dot); + mag_a = fp.mulAdd(f32, x, x, mag_a); + mag_b = fp.mulAdd(f32, y, y, mag_b); + } + const denom = @sqrt(mag_a) * @sqrt(mag_b); + if (denom < DENOM_EPS) return 0.0; + return dot / denom; +} + +/// Pretrained token lookup table — built lazily on first use. +var g_pretrained_map: ?*c.CBMHashTable = null; +var g_pretrained_ready: AtomicInt = .init(0); +var g_pretrained_mtx: c.cbm_mutex_t = undefined; +var g_pretrained_mtx_init: AtomicInt = .init(MTX_STATE_UNINIT); + +/// Thread-safe lazy init of the pretrained token lookup map. +/// Uses double-checked locking: fast path reads an atomic flag. +fn ensurePretrainedMap() void { + if (g_pretrained_ready.load(.acquire) != 0) return; + // First-time init of the mutex itself (also needs to be thread-safe) + if (g_pretrained_mtx_init.cmpxchgStrong( + MTX_STATE_UNINIT, + MTX_STATE_INITIALIZING, + .acq_rel, + .acquire, + ) == null) { + c.cbm_mutex_init(&g_pretrained_mtx); + g_pretrained_mtx_init.store(MTX_STATE_READY, .release); + } else { + // Spin until another thread finishes initializing the mutex + while (g_pretrained_mtx_init.load(.acquire) != MTX_STATE_READY) { + // brief spin + } + } + c.cbm_mutex_lock(&g_pretrained_mtx); + if (g_pretrained_ready.load(.acquire) == 0) { + g_pretrained_map = c.cbm_ht_create(PRETRAINED_TOKEN_COUNT); + var idx_buf: [c.CBM_SZ_16]u8 = undefined; + for (0..PRETRAINED_TOKEN_COUNT) |i| { + const tok: CStr = c.PRETRAINED_TOKENS[i]; + if (tok != null and tok[0] != 0) { + _ = c.snprintf(&idx_buf, idx_buf.len, "%d", @as(c_int, @intCast(i))); + _ = c.cbm_ht_set(g_pretrained_map, c.strdup(tok), c.strdup(&idx_buf)); + } + } + g_pretrained_ready.store(MAP_READY, .release); + } + c.cbm_mutex_unlock(&g_pretrained_mtx); +} + +pub export fn cbm_sem_ensure_ready() void { + ensurePretrainedMap(); +} + +/// The pretrained int8 vector for a token, or null when the token is not in +/// the pretrained vocabulary. The map must be initialized. +fn pretrainedVec(token: CStr) ?[*]const i8 { + const idx_str: CStr = @ptrCast(c.cbm_ht_get(g_pretrained_map, token)); + if (idx_str == null) return null; + var end: [*c]u8 = null; + const idx = c.strtol(idx_str, &end, BASE_DECIMAL); + if (end != idx_str and idx >= 0 and idx < PRETRAINED_TOKEN_COUNT) { + return @ptrCast(c.pretrained_vec_at(@intCast(idx))); + } + return null; +} + +pub export fn cbm_sem_random_index(token_c: ?Str, out: *Vec) void { + @memset(&out.v, 0.0); + const token = token_c orelse return; + + // Try pretrained nomic-embed-code vector first (768d, distilled from 7B). + ensurePretrainedMap(); + if (pretrainedVec(token)) |pvec| { + for (out.v[0..@min(DIM, PRETRAINED_DIM)], pvec[0..@min(DIM, PRETRAINED_DIM)]) |*o, p| { + o.* = @as(f32, @floatFromInt(p)) / INT8_MAX; + } + return; + } + + // Fallback: sparse random vector for tokens not in pretrained vocab. + const seed = XxHash3.hash(0, std.mem.span(token)); + for (0..SPARSE_NNZE) |i| { + const h = XxHash3.hash(seed +% RI_SEED_BASE, &std.mem.toBytes(@as(c_int, @intCast(i)))); + const pos: usize = @intCast(h % DIM); + const sign: f32 = if (h & 1 != 0) UNIT_POS else -UNIT_POS; + out.v[pos] += sign; + } +} + +pub export fn cbm_sem_normalize(v_c: ?*Vec) void { + const v = v_c orelse return; + var mag: f32 = 0.0; + for (v.v) |x| { + mag = fp.mulAdd(f32, x, x, mag); + } + mag = @sqrt(mag); + if (mag < DENOM_EPS) return; + const inv = UNIT_POS / mag; + for (&v.v) |*x| { + x.* *= inv; + } +} + +pub export fn cbm_sem_vec_add_scaled(dst_c: ?*Vec, src_c: ?*const Vec, scale: f32) void { + const dst = dst_c orelse return; + const src = src_c orelse return; + for (&dst.v, src.v) |*d, s| { + d.* = fp.mulAdd(f32, scale, s, d.*); + } +} + +// ── Corpus (IDF + Random Indexing enrichment) ─────────────────────── + +const CorpusEntry = extern struct { + token: [*c]u8, + doc_freq: c_int, + enriched_vec: Vec, // context-enriched via co-occurrence +}; + +const Corpus = extern struct { + token_map: ?*c.CBMHashTable, // token → index into entries[] + entries: [*c]CorpusEntry, + entry_count: c_int, + entry_cap: c_int, + doc_count: c_int, + finalized: bool, + + // Per-document token lists for co-occurrence pass + doc_token_ids: [*c][*c]c_int, + doc_token_counts: [*c]c_int, + doc_cap: c_int, +}; + +/// Parse a decimal index string via strtol, returning CBM_NOT_FOUND on parse +/// failure. Centralizes the atoi-replacement pattern for token_map lookups. +fn parseTokenIndex(idx_str: CStr) c_int { + if (idx_str == null) return NOT_FOUND; + var end: [*c]u8 = null; + const parsed = c.strtol(idx_str, &end, BASE_DECIMAL); + return if (end != idx_str) @truncate(parsed) else NOT_FOUND; +} + +fn corpusGetOrAdd(cp: *Corpus, token: CStr) c_int { + var idx_buf: [c.CBM_SZ_16]u8 = undefined; + const existing: CStr = @ptrCast(c.cbm_ht_get(cp.token_map, token)); + if (existing != null) { + return parseTokenIndex(existing); + } + if (cp.entry_count >= cp.entry_cap) { + const new_cap: c_int = if (cp.entry_cap < CORPUS_INIT_CAP) CORPUS_INIT_CAP else cp.entry_cap * 2; + const grown = interop.reallocN(CorpusEntry, cp.entries, ix(new_cap)); + if (grown == null) return NOT_FOUND; + cp.entries = grown; + cp.entry_cap = new_cap; + } + const idx = cp.entry_count; + cp.entry_count += 1; + const entry = at(cp.entries, ix(idx)); + entry.token = c.strdup(token); + entry.doc_freq = 0; + @memset(&entry.enriched_vec.v, 0.0); + _ = c.snprintf(&idx_buf, idx_buf.len, "%d", idx); + _ = c.cbm_ht_set(cp.token_map, c.strdup(token), c.strdup(&idx_buf)); + return idx; +} + +pub export fn cbm_sem_corpus_new() ?*Corpus { + const cp: ?*Corpus = interop.callocN(Corpus, 1); + if (cp) |corpus| { + corpus.token_map = c.cbm_ht_create(CORPUS_INIT_CAP); + } + return cp; +} + +pub export fn cbm_sem_corpus_add_doc(corpus_c: ?*Corpus, tokens_c: ?[*]CStr, count: c_int) void { + const corpus = corpus_c orelse return; + const tokens = tokens_c orelse return; + if (count <= 0) return; + // Track document for co-occurrence pass + if (corpus.doc_count >= corpus.doc_cap) { + const new_cap: c_int = if (corpus.doc_cap < DOC_TOKENS_INIT) DOC_TOKENS_INIT else corpus.doc_cap * 2; + const grown_ids = interop.reallocN([*c]c_int, corpus.doc_token_ids, ix(new_cap)); + const grown_counts = interop.reallocN(c_int, corpus.doc_token_counts, ix(new_cap)); + if (grown_ids == null or grown_counts == null) { + c.free(@ptrCast(grown_ids)); + c.free(@ptrCast(grown_counts)); + return; + } + corpus.doc_token_ids = grown_ids; + corpus.doc_token_counts = grown_counts; + corpus.doc_cap = new_cap; + } + const doc_idx = ix(corpus.doc_count); + corpus.doc_count += 1; + corpus.doc_token_ids[doc_idx] = interop.mallocN(c_int, ix(count)); + corpus.doc_token_counts[doc_idx] = count; + + // Per-doc unique set for IDF + const seen = interop.callocN(c_int, ix(corpus.entry_cap) + ix(count) + CORPUS_INIT_CAP); + var seen_count: usize = 0; + + for (0..ix(count)) |i| { + const tid = corpusGetOrAdd(corpus, tokens[i]); + corpus.doc_token_ids[doc_idx][i] = tid; + if (tid < 0) continue; + // Check uniqueness for IDF (simple linear scan — tokens per doc is small) + var is_new = true; + for (0..seen_count) |j| { + if (seen[j] == tid) { + is_new = false; + break; + } + } + if (is_new) { + seen[seen_count] = tid; + seen_count += 1; + corpus.entries[ix(tid)].doc_freq += 1; + } + } + c.free(@ptrCast(seen)); +} + +// ── Parallel corpus batch build ───────────────────────────────────── +// Strategy: +// Phase A (SEQUENTIAL): Scan all documents once to build the global +// token_map (inserts unique tokens, assigns global IDs). This is +// inherently sequential (hash table mutation), but much faster than +// the current per-doc add_doc because we avoid the per-doc malloc of +// the `seen` array and per-doc bookkeeping. +// Phase B (PARALLEL): Each worker processes a chunk of docs, translates +// tokens → global IDs via read-only token_map lookups, fills +// doc_token_ids[d], and accumulates doc_freq contributions via atomics. + +const BatchResolveCtx = struct { + corpus: *Corpus, + all_tokens: [*][*c]u8, + token_counts: [*]const c_int, + max_tokens: c_int, + doc_count: c_int, + doc_freq_atomic: [*]AtomicInt, // per-entry atomic counter (entry_count long) + next_idx: AtomicInt, +}; + +/// Resolve one document: look up each token's global ID, fill the corpus +/// doc_token_ids[d], and bump the per-token doc_freq counter atomically. The +/// caller is responsible for ensuring `seen` has capacity for `count` ints +/// before calling (the worker grows its per-thread scratch buffer). +fn batchResolveOneDoc(bc: *BatchResolveCtx, doc_index: usize, seen: [*c]c_int) void { + const count = bc.token_counts[doc_index]; + if (count <= 0) { + bc.corpus.doc_token_ids[doc_index] = null; + bc.corpus.doc_token_counts[doc_index] = 0; + return; + } + const ids = interop.mallocN(c_int, ix(count)); + bc.corpus.doc_token_ids[doc_index] = ids; + bc.corpus.doc_token_counts[doc_index] = count; + + var seen_count: usize = 0; + const tokens = bc.all_tokens + doc_index * ix(bc.max_tokens); + for (0..ix(count)) |i| { + const tid = parseTokenIndex(@ptrCast(c.cbm_ht_get(bc.corpus.token_map, tokens[i]))); + ids[i] = tid; + if (tid < 0) continue; + // Unique-per-doc check for IDF + var is_new = true; + for (0..seen_count) |j| { + if (seen[j] == tid) { + is_new = false; + break; + } + } + if (is_new) { + seen[seen_count] = tid; + seen_count += 1; + _ = bc.doc_freq_atomic[ix(tid)].fetchAdd(1, .monotonic); + } + } +} + +fn batchResolveWorker(worker_id: c_int, ctx_ptr: ?*anyopaque) callconv(.c) void { + _ = worker_id; + const bc: *BatchResolveCtx = @ptrCast(@alignCast(ctx_ptr)); + // Per-worker scratch for unique-per-doc tracking + var local_seen_cap: c_int = SEEN_INIT_CAP; + var seen = interop.mallocN(c_int, ix(local_seen_cap)); + if (seen == null) return; + + while (true) { + const start = bc.next_idx.fetchAdd(RESOLVE_CHUNK, .monotonic); + if (start >= bc.doc_count) break; + var end = start + RESOLVE_CHUNK; + if (end > bc.doc_count) end = bc.doc_count; + for (ix(start)..ix(end)) |d| { + const count = bc.token_counts[d]; + if (count > local_seen_cap) { + const grown = interop.reallocN(c_int, seen, ix(count)); + if (grown == null) continue; + seen = grown; + local_seen_cap = count; + } + batchResolveOneDoc(bc, d, seen); + } + } + c.free(@ptrCast(seen)); +} + +pub export fn cbm_sem_corpus_add_docs_batch( + corpus_c: ?*Corpus, + all_tokens_c: ?[*][*c]u8, + token_counts_c: ?[*]const c_int, + doc_count: c_int, + max_tokens_per_doc: c_int, +) void { + const corpus = corpus_c orelse return; + const all_tokens = all_tokens_c orelse return; + const token_counts = token_counts_c orelse return; + if (doc_count <= 0) return; + + // Phase A (SEQUENTIAL): Build token_map and allocate doc arrays. + // Hash table mutation can't be parallelized; strdup+insert is the cost. + if (corpus.doc_cap < corpus.doc_count + doc_count) { + const new_cap = corpus.doc_count + doc_count; + const grown_ids = interop.reallocN([*c]c_int, corpus.doc_token_ids, ix(new_cap)); + const grown_counts = interop.reallocN(c_int, corpus.doc_token_counts, ix(new_cap)); + if (grown_ids == null or grown_counts == null) { + c.free(@ptrCast(grown_ids)); + c.free(@ptrCast(grown_counts)); + return; + } + corpus.doc_token_ids = grown_ids; + corpus.doc_token_counts = grown_counts; + corpus.doc_cap = new_cap; + } + const base_doc = corpus.doc_count; + corpus.doc_count += doc_count; + + for (0..ix(doc_count)) |d| { + const count = token_counts[d]; + const tokens = all_tokens + d * ix(max_tokens_per_doc); + var i: c_int = 0; + while (i < count) : (i += 1) { + // Inserts token into token_map if new; we discard return here — + // Phase B will re-lookup in read-only mode to get the ID. + _ = corpusGetOrAdd(corpus, tokens[ix(i)]); + } + } + + // Phase B (PARALLEL): Resolve tokens → IDs and count doc_freq per entry. + // token_map is now read-only; each worker owns its doc range (no writes + // to shared state except atomic doc_freq counters). + const doc_freq_atomic: ?[*]AtomicInt = interop.callocN(AtomicInt, ix(corpus.entry_count)); + if (doc_freq_atomic == null) { + // OOM fallback: sequential path. Roll back doc_count first since + // add_doc increments it itself. + corpus.doc_count = base_doc; + for (0..ix(doc_count)) |d| { + const count = token_counts[d]; + const tokens = all_tokens + d * ix(max_tokens_per_doc); + cbm_sem_corpus_add_doc(corpus, @ptrCast(tokens), count); + } + return; + } + + const worker_count = c.cbm_default_worker_count(false); + var bc: BatchResolveCtx = .{ + .corpus = corpus, + .all_tokens = all_tokens, + .token_counts = token_counts, + .max_tokens = max_tokens_per_doc, + .doc_count = doc_count, + .doc_freq_atomic = doc_freq_atomic.?, + .next_idx = .init(0), + }; + // Temporarily re-base doc arrays so workers write to base_doc..base_doc+doc_count + corpus.doc_token_ids += ix(base_doc); + corpus.doc_token_counts += ix(base_doc); + const opts: c.cbm_parallel_for_opts_t = .{ .max_workers = worker_count, .force_pthreads = false }; + c.cbm_parallel_for(worker_count, batchResolveWorker, &bc, opts); + corpus.doc_token_ids -= ix(base_doc); + corpus.doc_token_counts -= ix(base_doc); + + // Phase C (SEQUENTIAL reduce): atomic counters → entries[].doc_freq + for (0..ix(corpus.entry_count)) |i| { + corpus.entries[i].doc_freq += doc_freq_atomic.?[i].load(.monotonic); + } + c.free(@ptrCast(doc_freq_atomic)); +} + +// ── Parallel corpus_finalize ──────────────────────────────────────── +// Strategy: +// 1. Precompute base RI vectors into a shared array (eliminates ~333M +// redundant cbm_sem_random_index calls on kernel-scale corpora). +// 2. Co-occurrence passes: partition TARGET tokens across workers so each +// worker writes to a disjoint range of enriched_vec (zero contention). +// Each worker still scans all documents but only accumulates for targets +// in its range. Inner vector add is the parallelized work. +// 3. Normalize/blend loops are trivially parallel per-entry. + +/// Reverse index: for each token id, a list of (doc_id, position_in_doc) pairs. +/// Built once, reused for both cooccur passes. Eliminates the O(num_chunks × doc_count) +/// redundant outer scan in the old algorithm. +const CooccurPos = extern struct { + doc_id: i32, + pos: i32, +}; + +const ReverseIndex = extern struct { + offsets: [*c]c_int, // offsets[entry_count + 1], prefix sum of occurrences + flat: [*c]CooccurPos, // flat array of positions, total = offsets[entry_count] +}; + +/// Tagged source vector: sparse (~30% of tokens, 8 inline nonzeros) or dense int8 +/// reference into PRETRAINED_VECTOR_BLOB (no copy). Dense path converts int8→float +/// on the fly in the hot loop. Tested with both int8 and float32 blob storage +/// formats — cooccur passes are memory-bandwidth-bound, so the int8 format (4x +/// less source traffic) is equivalent in wall time despite the conversion cost, +/// while saving 90 MB of binary size. +const SrcEntry = extern struct { + is_sparse: u8, // 1 = sparse path, 0 = dense int8 reference + nnz: u8, // number of nonzeros used in sparse path + _pad: u16, + indices: [SPARSE_NNZE]u16, // 8 * 2 = 16 bytes + values: [SPARSE_NNZE]f32, // 8 * 4 = 32 bytes + dense_int8: ?[*]const i8, // points into PRETRAINED_VECTOR_BLOB +}; + +/// Initialize a target vector from a sparse/dense source. +inline fn semTargetInitFromSrc(dst: *Vec, src: *const SrcEntry) void { + @memset(&dst.v, 0.0); + if (src.is_sparse != 0) { + for (src.indices[0..src.nnz], src.values[0..src.nnz]) |idx, val| { + dst.v[idx] = val; + } + } else { + const s = src.dense_int8.?; + const inv127 = UNIT_POS / INT8_MAX; + for (&dst.v, s[0..DIM]) |*d, x| { + d.* = inv127 * @as(f32, @floatFromInt(x)); + } + } +} + +/// Add weighted source into target. +/// Sparse path: ~8 operations, ~48 bytes source memory traffic. +/// Dense path: 768 mul-adds with int8→float conversion, ~768 bytes traffic. +inline fn semVecAddSrcScaled(dst: *Vec, src: *const SrcEntry, scale: f32) void { + if (src.is_sparse != 0) { + for (src.indices[0..src.nnz], src.values[0..src.nnz]) |idx, val| { + dst.v[idx] = fp.mulAdd(f32, scale, val, dst.v[idx]); + } + } else { + const s = src.dense_int8.?; + const mul = scale * (UNIT_POS / INT8_MAX); + for (&dst.v, s[0..DIM]) |*d, x| { + d.* = fp.mulAdd(f32, mul, @floatFromInt(x), d.*); + } + } +} + +/// Pass 1 context: uses sparse/int8 tagged sources (most memory-efficient). +const CooccurSparseCtx = struct { + entries: [*c]CorpusEntry, + src_entries: [*c]const SrcEntry, // sparse or int8-dense per token + doc_token_ids: [*c][*c]c_int, + doc_token_counts: [*c]const c_int, + rev: *const ReverseIndex, + doc_count: c_int, + entry_count: c_int, + next_chunk: AtomicInt, + num_chunks: c_int, + chunk_size: c_int, + + // Cache-blocked tiling parameters + tile_size: c_int, // targets per L2-resident tile +}; + +/// Accumulate co-occurrence context for a single target token into `target`. +/// Reads neighbors within ±CBM_SEM_WINDOW positions across all documents that +/// reference this token id via the reverse index. +fn cooccurSparseOneTarget(cc: *CooccurSparseCtx, tid: c_int, target: *Vec) void { + const occ_start = cc.rev.offsets[ix(tid)]; + const occ_end = cc.rev.offsets[ix(tid) + 1]; + // Frequent-token subsampling: stride over occurrences when a token is very + // common so the enriched vector's direction is preserved (it's normalized + // after) while bounding work to ~CBM_SEM_MAX_OCCUR samples. See semantic.h. + var occ_step: c_int = 1; + if (occ_end - occ_start > MAX_OCCUR) { + occ_step = @divTrunc(occ_end - occ_start, MAX_OCCUR); + } + var p = occ_start; + while (p < occ_end) : (p += occ_step) { + const d = cc.rev.flat[ix(p)].doc_id; + const i = cc.rev.flat[ix(p)].pos; + const ids = cc.doc_token_ids[ix(d)]; + const len = cc.doc_token_counts[ix(d)]; + var w: c_int = -WINDOW; + while (w <= WINDOW) : (w += 1) { + if (w == 0) continue; + const j = i + w; + if (j < 0 or j >= len) continue; + const nid = ids[ix(j)]; + if (nid < 0) continue; + const weight = UNIT_POS / @as(f32, @floatFromInt(@abs(w))); + semVecAddSrcScaled(target, at(cc.src_entries, ix(nid)), weight); + } + } +} + +fn cooccurWorkerSparse(worker_id: c_int, ctx_ptr: ?*anyopaque) callconv(.c) void { + _ = worker_id; + const cc: *CooccurSparseCtx = @ptrCast(@alignCast(ctx_ptr)); + while (true) { + const ci = cc.next_chunk.fetchAdd(1, .monotonic); + if (ci >= cc.num_chunks) break; + const chunk_start = ci * cc.chunk_size; + var chunk_end = chunk_start + cc.chunk_size; + if (chunk_end > cc.entry_count) chunk_end = cc.entry_count; + + // Cache-blocked target tiling: process tile_size targets at a time so + // their vectors stay resident in L2 cache during their accumulation. + var tile_start = chunk_start; + while (tile_start < chunk_end) : (tile_start += cc.tile_size) { + var tile_end = tile_start + cc.tile_size; + if (tile_end > chunk_end) tile_end = chunk_end; + var tid = tile_start; + while (tid < tile_end) : (tid += 1) { + const target = &at(cc.entries, ix(tid)).enriched_vec; + semTargetInitFromSrc(target, at(cc.src_entries, ix(tid))); + cooccurSparseOneTarget(cc, tid, target); + } + } + } +} + +/// Pass 2 (RRI) context: uses int8-quantized pass1 vectors as source. +/// Pass1 outputs are dense float32 post-normalization, values in [-1,1]. +/// We quantize to int8 once (×127) to cut source memory traffic 4x. +const CooccurInt8Ctx = struct { + entries: [*c]CorpusEntry, + pass1_q: [*]const i8, // [entry_count × CBM_SEM_DIM] int8 quantized pass1 + doc_token_ids: [*c][*c]c_int, + doc_token_counts: [*c]const c_int, + rev: *const ReverseIndex, + doc_count: c_int, + entry_count: c_int, + next_chunk: AtomicInt, + num_chunks: c_int, + chunk_size: c_int, + tile_size: c_int, +}; + +inline fn semVecAddInt8Scaled(dst: *Vec, src: *const [DIM]i8, scale: f32) void { + const mul = scale * (UNIT_POS / INT8_MAX); + for (&dst.v, src) |*d, x| { + d.* = fp.mulAdd(f32, mul, @floatFromInt(x), d.*); + } +} + +/// RRI pass 2 accumulator for a single target token, reading int8-quantized +/// pass1 vectors as the source. +fn cooccurInt8OneTarget(cc: *CooccurInt8Ctx, tid: c_int, target: *Vec) void { + const occ_start = cc.rev.offsets[ix(tid)]; + const occ_end = cc.rev.offsets[ix(tid) + 1]; + // Frequent-token subsampling (see cooccurSparseOneTarget / semantic.h). + var occ_step: c_int = 1; + if (occ_end - occ_start > MAX_OCCUR) { + occ_step = @divTrunc(occ_end - occ_start, MAX_OCCUR); + } + var p = occ_start; + while (p < occ_end) : (p += occ_step) { + const d = cc.rev.flat[ix(p)].doc_id; + const i = cc.rev.flat[ix(p)].pos; + const ids = cc.doc_token_ids[ix(d)]; + const len = cc.doc_token_counts[ix(d)]; + var w: c_int = -WINDOW; + while (w <= WINDOW) : (w += 1) { + if (w == 0) continue; + const j = i + w; + if (j < 0 or j >= len) continue; + const nid = ids[ix(j)]; + if (nid < 0) continue; + const weight = UNIT_POS / @as(f32, @floatFromInt(@abs(w))); + semVecAddInt8Scaled(target, cc.pass1_q[ix(nid) * DIM ..][0..DIM], weight); + } + } +} + +fn cooccurWorkerInt8(worker_id: c_int, ctx_ptr: ?*anyopaque) callconv(.c) void { + _ = worker_id; + const cc: *CooccurInt8Ctx = @ptrCast(@alignCast(ctx_ptr)); + while (true) { + const ci = cc.next_chunk.fetchAdd(1, .monotonic); + if (ci >= cc.num_chunks) break; + const chunk_start = ci * cc.chunk_size; + var chunk_end = chunk_start + cc.chunk_size; + if (chunk_end > cc.entry_count) chunk_end = cc.entry_count; + + var tile_start = chunk_start; + while (tile_start < chunk_end) : (tile_start += cc.tile_size) { + var tile_end = tile_start + cc.tile_size; + if (tile_end > chunk_end) tile_end = chunk_end; + var tid = tile_start; + while (tid < tile_end) : (tid += 1) { + // RRI pass 2 starts from zero (no self-init) + const target = &at(cc.entries, ix(tid)).enriched_vec; + @memset(&target.v, 0.0); + cooccurInt8OneTarget(cc, tid, target); + } + } + } +} + +const NormCtx = struct { + entries: [*c]CorpusEntry, + entry_count: c_int, + next_idx: AtomicInt, +}; + +fn normalizeWorker(worker_id: c_int, ctx_ptr: ?*anyopaque) callconv(.c) void { + _ = worker_id; + const nc: *NormCtx = @ptrCast(@alignCast(ctx_ptr)); + while (true) { + const start = nc.next_idx.fetchAdd(WORKER_STACK_CAP, .monotonic); + if (start >= nc.entry_count) break; + var end = start + WORKER_STACK_CAP; + if (end > nc.entry_count) end = nc.entry_count; + for (ix(start)..ix(end)) |i| { + cbm_sem_normalize(&at(nc.entries, i).enriched_vec); + } + } +} + +const BlendCtx = struct { + entries: [*c]CorpusEntry, + pass1: [*]const Vec, + entry_count: c_int, + next_idx: AtomicInt, +}; + +fn blendWorker(worker_id: c_int, ctx_ptr: ?*anyopaque) callconv(.c) void { + _ = worker_id; + const bc: *BlendCtx = @ptrCast(@alignCast(ctx_ptr)); + while (true) { + const start = bc.next_idx.fetchAdd(WORKER_STACK_CAP, .monotonic); + if (start >= bc.entry_count) break; + var end = start + WORKER_STACK_CAP; + if (end > bc.entry_count) end = bc.entry_count; + for (ix(start)..ix(end)) |i| { + const vec = &at(bc.entries, i).enriched_vec; + cbm_sem_normalize(vec); + for (&vec.v, bc.pass1[i].v) |*e, p1| { + e.* = fp.mulAdd(f32, RRI_BETA, p1, RRI_ALPHA * e.*); + } + } + } +} + +const SrcBuildCtx = struct { + entries: [*c]CorpusEntry, + src_entries: [*c]SrcEntry, + entry_count: c_int, + next_idx: AtomicInt, +}; + +/// Build one src_entry for a token: dense int8 reference if in nomic vocab, +/// sparse inline representation otherwise. Collisions in the sparse hash are +/// merged and zeros filtered so the final representation is exactly the same +/// mathematical vector that the old dense path produced. +fn buildSrcEntry(token: CStr, out: *SrcEntry) void { + out.* = std.mem.zeroes(SrcEntry); + if (token == null) { + out.is_sparse = 1; + out.nnz = 0; + return; + } + // Dense path: direct int8 pointer into pretrained blob (zero-copy). + if (pretrainedVec(token)) |pvec| { + out.is_sparse = 0; + out.dense_int8 = pvec; + return; + } + // Sparse path: compute 8 hash positions with collision merging. + out.is_sparse = 1; + var tmp_idx: [SPARSE_NNZE]u16 = undefined; + var tmp_val: [SPARSE_NNZE]f32 = undefined; + var count: usize = 0; + const seed = XxHash3.hash(0, std.mem.span(token)); + for (0..SPARSE_NNZE) |i| { + const h = XxHash3.hash(seed +% RI_SEED_BASE, &std.mem.toBytes(@as(c_int, @intCast(i)))); + const pos: u16 = @intCast(h % DIM); + const sign: f32 = if (h & 1 != 0) UNIT_POS else -UNIT_POS; + // Merge collisions + var found: ?usize = null; + for (tmp_idx[0..count], 0..) |idx, j| { + if (idx == pos) { + found = j; + break; + } + } + if (found) |j| { + tmp_val[j] += sign; + } else { + tmp_idx[count] = pos; + tmp_val[count] = sign; + count += 1; + } + } + // Filter zeros + var nnz: u8 = 0; + for (tmp_idx[0..count], tmp_val[0..count]) |idx, val| { + if (val != 0.0) { + out.indices[nnz] = idx; + out.values[nnz] = val; + nnz += 1; + } + } + out.nnz = nnz; +} + +fn srcBuildWorker(worker_id: c_int, ctx_ptr: ?*anyopaque) callconv(.c) void { + _ = worker_id; + const sc: *SrcBuildCtx = @ptrCast(@alignCast(ctx_ptr)); + while (true) { + const start = sc.next_idx.fetchAdd(WORKER_STACK_CAP, .monotonic); + if (start >= sc.entry_count) break; + var end = start + WORKER_STACK_CAP; + if (end > sc.entry_count) end = sc.entry_count; + for (ix(start)..ix(end)) |i| { + buildSrcEntry(sc.entries[i].token, at(sc.src_entries, i)); + } + } +} + +/// Quantize pass1 float vectors to int8 (× 127) for use as pass2 source. +/// Input vectors are unit-normalized, so values are in [-1, 1] → int8 preserves +/// ~99% precision. 4x less memory traffic for pass2. +const Pass1QuantCtx = struct { + entries: [*c]const CorpusEntry, + pass1_q: [*]i8, + entry_count: c_int, + next_idx: AtomicInt, +}; + +fn pass1QuantizeWorker(worker_id: c_int, ctx_ptr: ?*anyopaque) callconv(.c) void { + _ = worker_id; + const qc: *Pass1QuantCtx = @ptrCast(@alignCast(ctx_ptr)); + while (true) { + const start = qc.next_idx.fetchAdd(RRI_TILE, .monotonic); + if (start >= qc.entry_count) break; + var end = start + RRI_TILE; + if (end > qc.entry_count) end = qc.entry_count; + for (ix(start)..ix(end)) |i| { + const src = &at(qc.entries, i).enriched_vec; + const dst = qc.pass1_q[i * DIM ..][0..DIM]; + for (dst, src.v) |*q, x| { + var v = x * INT8_MAX; + if (v > INT8_MAX) v = INT8_MAX; + if (v < -INT8_MAX) v = -INT8_MAX; + q.* = @intFromFloat(if (v >= 0) v + IDF_BASE else v - IDF_BASE); + } + } + } +} + +/// Build reverse index: token_id → list of (doc_id, position) pairs. +/// SEQUENTIAL (fast: just pointer arithmetic + flat array fill). +fn buildReverseIndex(corpus: *Corpus) ?*ReverseIndex { + const rev: *ReverseIndex = @as(?*ReverseIndex, interop.callocN(ReverseIndex, 1)) orelse return null; + const entry_count = ix(corpus.entry_count); + const doc_count = ix(corpus.doc_count); + // Phase A: count occurrences per token + const counts = interop.callocN(c_int, entry_count + 1); + if (counts == null) { + c.free(rev); + return null; + } + var total: c_long = 0; + for (0..doc_count) |d| { + const ids = corpus.doc_token_ids[d]; + const len = corpus.doc_token_counts[d]; + var i: c_int = 0; + while (i < len) : (i += 1) { + const tid = ids[ix(i)]; + if (tid >= 0 and tid < corpus.entry_count) { + counts[ix(tid)] += 1; + total += 1; + } + } + } + // Phase B: exclusive prefix sum → offsets[] + rev.offsets = interop.mallocN(c_int, entry_count + 1); + if (rev.offsets == null) { + c.free(@ptrCast(counts)); + c.free(rev); + return null; + } + var running: c_int = 0; + for (0..entry_count) |t| { + rev.offsets[t] = running; + running += counts[t]; + counts[t] = 0; // reuse as per-token fill cursor + } + rev.offsets[entry_count] = running; + // Phase C: fill flat array. Ensure allocation size > 0 even for empty + // corpora (avoids malloc(0) which is implementation-defined). + rev.flat = interop.mallocN(CooccurPos, if (total > 0) @intCast(total) else 1); + if (rev.flat == null) { + c.free(@ptrCast(rev.offsets)); + c.free(@ptrCast(counts)); + c.free(rev); + return null; + } + for (0..doc_count) |d| { + const ids = corpus.doc_token_ids[d]; + const len = corpus.doc_token_counts[d]; + var i: c_int = 0; + while (i < len) : (i += 1) { + const tid = ids[ix(i)]; + if (tid >= 0 and tid < corpus.entry_count) { + const slot = ix(rev.offsets[ix(tid)] + counts[ix(tid)]); + counts[ix(tid)] += 1; + rev.flat[slot].doc_id = @intCast(d); + rev.flat[slot].pos = i; + } + } + } + c.free(@ptrCast(counts)); + return rev; +} + +fn freeReverseIndex(rev_c: ?*ReverseIndex) void { + const rev = rev_c orelse return; + c.free(@ptrCast(rev.offsets)); + c.free(@ptrCast(rev.flat)); + c.free(rev); +} + +/// Bundle of parameters shared by the finalize sub-phases. +const FinalizeParams = struct { + corpus: *Corpus, + rev: *ReverseIndex, + src_entries: [*c]SrcEntry, + worker_count: c_int, + num_chunks: c_int, + chunk_size: c_int, + tile_size: c_int, + opts: c.cbm_parallel_for_opts_t, +}; + +/// Sub-phase 1: build tagged source vectors (sparse or dense-int8) in parallel. +fn finalizeBuildSources(p: *FinalizeParams) void { + var sc: SrcBuildCtx = .{ + .entries = p.corpus.entries, + .src_entries = p.src_entries, + .entry_count = p.corpus.entry_count, + .next_idx = .init(0), + }; + c.cbm_parallel_for(p.worker_count, srcBuildWorker, &sc, p.opts); +} + +/// Sub-phases 2+3: co-occurrence pass 1 + normalize. +fn finalizePass1(p: *FinalizeParams) void { + var cc: CooccurSparseCtx = .{ + .entries = p.corpus.entries, + .src_entries = p.src_entries, + .doc_token_ids = p.corpus.doc_token_ids, + .doc_token_counts = p.corpus.doc_token_counts, + .rev = p.rev, + .doc_count = p.corpus.doc_count, + .entry_count = p.corpus.entry_count, + .next_chunk = .init(0), + .num_chunks = p.num_chunks, + .chunk_size = p.chunk_size, + .tile_size = p.tile_size, + }; + c.cbm_parallel_for(p.worker_count, cooccurWorkerSparse, &cc, p.opts); + + var nc: NormCtx = .{ + .entries = p.corpus.entries, + .entry_count = p.corpus.entry_count, + .next_idx = .init(0), + }; + c.cbm_parallel_for(p.worker_count, normalizeWorker, &nc, p.opts); +} + +/// Sub-phases 4+5: quantize pass1 to int8, run RRI pass 2, blend + normalize. +fn finalizePass2(p: *FinalizeParams) void { + const entry_count = ix(p.corpus.entry_count); + const pass1_q: ?[*]i8 = interop.mallocN(i8, entry_count * DIM); + if (pass1_q) |q| { + var qc: Pass1QuantCtx = .{ + .entries = p.corpus.entries, + .pass1_q = q, + .entry_count = p.corpus.entry_count, + .next_idx = .init(0), + }; + c.cbm_parallel_for(p.worker_count, pass1QuantizeWorker, &qc, p.opts); + } + + const pass1: ?[*]Vec = interop.mallocN(Vec, entry_count); + if (pass1) |p1| { + for (0..entry_count) |i| { + p1[i] = p.corpus.entries[i].enriched_vec; + } + } + + if (pass1_q) |q| { + var cc: CooccurInt8Ctx = .{ + .entries = p.corpus.entries, + .pass1_q = q, + .doc_token_ids = p.corpus.doc_token_ids, + .doc_token_counts = p.corpus.doc_token_counts, + .rev = p.rev, + .doc_count = p.corpus.doc_count, + .entry_count = p.corpus.entry_count, + .next_chunk = .init(0), + .num_chunks = p.num_chunks, + .chunk_size = p.chunk_size, + .tile_size = p.tile_size, + }; + c.cbm_parallel_for(p.worker_count, cooccurWorkerInt8, &cc, p.opts); + } + + if (pass1) |p1| { + var bc: BlendCtx = .{ + .entries = p.corpus.entries, + .pass1 = p1, + .entry_count = p.corpus.entry_count, + .next_idx = .init(0), + }; + c.cbm_parallel_for(p.worker_count, blendWorker, &bc, p.opts); + c.free(@ptrCast(p1)); + } + c.free(@ptrCast(pass1_q)); + + var nc: NormCtx = .{ + .entries = p.corpus.entries, + .entry_count = p.corpus.entry_count, + .next_idx = .init(0), + }; + c.cbm_parallel_for(p.worker_count, normalizeWorker, &nc, p.opts); +} + +pub export fn cbm_sem_corpus_finalize(corpus_c: ?*Corpus) void { + const corpus = corpus_c orelse return; + if (corpus.finalized) return; + + // Eager init before parallel dispatch to avoid lazy-init races + ensurePretrainedMap(); + + const worker_count = c.cbm_default_worker_count(false); + const opts: c.cbm_parallel_for_opts_t = .{ .max_workers = worker_count, .force_pthreads = false }; + + // Finer chunks = better load balancing for skewed token distributions. + var num_chunks = worker_count * COOCCUR_CHUNK; + if (num_chunks > corpus.entry_count) num_chunks = corpus.entry_count; + if (num_chunks < 1) num_chunks = 1; + const chunk_size = @divTrunc(corpus.entry_count + num_chunks - 1, num_chunks); + + const rev = buildReverseIndex(corpus) orelse { + corpus.finalized = true; + return; + }; + const src_entries = interop.callocN(SrcEntry, ix(corpus.entry_count)); + if (src_entries == null) { + freeReverseIndex(rev); + corpus.finalized = true; + return; + } + + var params: FinalizeParams = .{ + .corpus = corpus, + .rev = rev, + .src_entries = src_entries, + .worker_count = worker_count, + .num_chunks = num_chunks, + .chunk_size = chunk_size, + .tile_size = TILE_SIZE, + .opts = opts, + }; + finalizeBuildSources(¶ms); + finalizePass1(¶ms); + finalizePass2(¶ms); + + c.free(@ptrCast(src_entries)); + freeReverseIndex(rev); + corpus.finalized = true; +} + +/// log(doc_count / doc_freq) through libm, as the C code computed it. +fn idfOf(doc_count: c_int, df: c_int) f32 { + return c.logf(@as(f32, @floatFromInt(doc_count)) / @as(f32, @floatFromInt(df))); +} + +pub export fn cbm_sem_corpus_idf(corpus_c: ?*const Corpus, token: ?Str) f32 { + const corpus = corpus_c orelse return 0.0; + if (token == null or corpus.doc_count == 0) return 0.0; + const idx = parseTokenIndex(@ptrCast(c.cbm_ht_get(corpus.token_map, token))); + if (idx < 0 or idx >= corpus.entry_count) return 0.0; + const df = corpus.entries[ix(idx)].doc_freq; + if (df <= 0) return 0.0; + return idfOf(corpus.doc_count, df); +} + +pub export fn cbm_sem_corpus_ri_vec(corpus_c: ?*const Corpus, token: ?Str) ?*const Vec { + const corpus = corpus_c orelse return null; + if (token == null) return null; + const idx = parseTokenIndex(@ptrCast(c.cbm_ht_get(corpus.token_map, token))); + if (idx < 0 or idx >= corpus.entry_count) return null; + return &at(corpus.entries, ix(idx)).enriched_vec; +} + +pub export fn cbm_sem_corpus_doc_count(corpus: ?*const Corpus) c_int { + return if (corpus) |cp| cp.doc_count else 0; +} + +pub export fn cbm_sem_corpus_token_count(corpus: ?*const Corpus) c_int { + return if (corpus) |cp| cp.entry_count else 0; +} + +pub export fn cbm_sem_corpus_token_at( + corpus_c: ?*const Corpus, + index: c_int, + out_vec: ?*?*const Vec, + out_idf: ?*f32, +) CStr { + const corpus = corpus_c orelse return null; + if (index < 0 or index >= corpus.entry_count) return null; + const entry = at(corpus.entries, ix(index)); + if (out_vec) |ov| { + ov.* = &entry.enriched_vec; + } + if (out_idf) |oi| { + if (corpus.doc_count > 0) { + const df = entry.doc_freq; + oi.* = if (df > 0) idfOf(corpus.doc_count, df) else 0.0; + } + } + return entry.token; +} + +fn freeHtKv(key: CStr, value: ?*anyopaque, userdata: ?*anyopaque) callconv(.c) void { + _ = userdata; + c.free(@ptrCast(@constCast(key))); + c.free(value); +} + +pub export fn cbm_sem_corpus_free(corpus_c: ?*Corpus) void { + const corpus = corpus_c orelse return; + for (0..ix(corpus.entry_count)) |i| { + c.free(corpus.entries[i].token); + } + c.free(@ptrCast(corpus.entries)); + for (0..ix(corpus.doc_count)) |d| { + c.free(@ptrCast(corpus.doc_token_ids[d])); + } + c.free(@ptrCast(corpus.doc_token_ids)); + c.free(@ptrCast(corpus.doc_token_counts)); + if (corpus.token_map) |map| { + c.cbm_ht_foreach(map, freeHtKv, null); + c.cbm_ht_free(map); + } + c.free(corpus); +} + +// ── Combined scoring ──────────────────────────────────────────────── + +pub export fn cbm_sem_proximity(path_a: ?Str, path_b: ?Str) f32 { + const pa = path_a orelse return UNIT_POS; + const pb = path_b orelse return UNIT_POS; + // Count shared directory components + var shared: c_int = 0; + var total_a: c_int = 0; + var total_b: c_int = 0; + var i: usize = 0; + while (pa[i] != 0 and pb[i] != 0 and pa[i] == pb[i]) : (i += 1) { + if (pa[i] == '/') shared += 1; + } + for (std.mem.span(pa)) |ch| { + if (ch == '/') total_a += 1; + } + for (std.mem.span(pb)) |ch| { + if (ch == '/') total_b += 1; + } + const max_total = @max(total_a, total_b); + if (max_total == 0) return UNIT_POS; + const ratio = @as(f32, @floatFromInt(shared)) / @as(f32, @floatFromInt(max_total)); + // Same file = 1.10, same dir = 1.05, distant = 1.00 + return fp.mulAdd(f32, ratio, PROX_MAX_BOOST, UNIT_POS); +} + +/// Cosine similarity for small float arrays (AST profile). +fn smallCosine(a: []const f32, b: []const f32) f32 { + var dot: f32 = 0.0; + var ma: f32 = 0.0; + var mb: f32 = 0.0; + for (a, b) |x, y| { + dot = fp.mulAdd(f32, x, y, dot); + ma = fp.mulAdd(f32, x, x, ma); + mb = fp.mulAdd(f32, y, y, mb); + } + const denom = @sqrt(ma) * @sqrt(mb); + return if (denom < DENOM_EPS) 0.0 else dot / denom; +} + +/// Sparse cosine over two pre-sorted (index, weight) vectors. Returns 0 when +/// either side is empty or the magnitude product is below the epsilon guard. +fn sparseTfidfCosine(a: *const Func, b: *const Func) f32 { + if (a.tfidf_len <= 0 or b.tfidf_len <= 0) return 0.0; + const a_len = ix(a.tfidf_len); + const b_len = ix(b.tfidf_len); + var dot: f32 = 0.0; + var ma: f32 = 0.0; + var mb: f32 = 0.0; + var ia: usize = 0; + var ib: usize = 0; + while (ia < a_len and ib < b_len) { + if (a.tfidf_indices[ia] == b.tfidf_indices[ib]) { + dot = fp.mulAdd(f32, a.tfidf_weights[ia], b.tfidf_weights[ib], dot); + ia += 1; + ib += 1; + } else if (a.tfidf_indices[ia] < b.tfidf_indices[ib]) { + ia += 1; + } else { + ib += 1; + } + } + for (a.tfidf_weights[0..a_len]) |w| { + ma = fp.mulAdd(f32, w, w, ma); + } + for (b.tfidf_weights[0..b_len]) |w| { + mb = fp.mulAdd(f32, w, w, mb); + } + const denom = @sqrt(ma) * @sqrt(mb); + return if (denom > DENOM_EPS) dot / denom else 0.0; +} + +pub export fn cbm_sem_combined_score( + a_c: ?*const Func, + b_c: ?*const Func, + cfg_c: ?*const c.cbm_sem_config_t, +) f32 { + const a = a_c orelse return 0.0; + const b = b_c orelse return 0.0; + const cfg = cfg_c orelse return 0.0; + const a_minhash: *const c.cbm_minhash_t = @ptrCast(&a.minhash); + const b_minhash: *const c.cbm_minhash_t = @ptrCast(&b.minhash); + + // Short-circuit: if MinHash Jaccard is already above the SIMILAR_TO threshold, + // the pass_similarity pipeline already emitted a SIMILAR_TO edge for this pair. + // Returning 0 here avoids flooding top-k with cross-service copy-paste boilerplate + // (logging_middleware, shared push/pull handlers) that SIMILAR_TO already covers, + // freeing the edge budget for true semantic leaps (vocabulary-bridged relations). + if (a.has_minhash and b.has_minhash) { + const early_j = c.cbm_minhash_jaccard(a_minhash, b_minhash); + if (early_j >= c.CBM_MINHASH_JACCARD_THRESHOLD) return 0.0; + } + + var score = cfg.w_tfidf * sparseTfidfCosine(a, b); + + // Signal 2: Random Indexing + score = fp.mulAdd(f32, cfg.w_ri, c.cbm_rsq_ip(&a.ri_code, &b.ri_code), score); + + // Signal 3: MinHash Jaccard + if (a.has_minhash and b.has_minhash) { + const j = c.cbm_minhash_jaccard(a_minhash, b_minhash); + score = fp.mulAdd(f32, cfg.w_minhash, @floatCast(j), score); + } + + // Signal 4: API Signatures + score = fp.mulAdd(f32, cfg.w_api, c.cbm_rsq_ip(&a.api_code, &b.api_code), score); + + // Signal 5: Type Signatures + score = fp.mulAdd(f32, cfg.w_type, c.cbm_rsq_ip(&a.type_code, &b.type_code), score); + + // Signal 7: Decorator Pattern + score = fp.mulAdd(f32, cfg.w_decorator, c.cbm_rsq_ip(&a.deco_code, &b.deco_code), score); + + // Signal 8+9+11: Structural profile + data flow + Halstead + const sp_score = smallCosine(&a.struct_profile, &b.struct_profile); + score = fp.mulAdd(f32, cfg.w_struct_profile, sp_score, score); + + // Signal 6: Module proximity (multiplier, not additive). + // Proximity returns [1.0, 1.10] — a same-file/same-dir boost for ranking. + // Clamp the final product to [0, 1] so the output stays within valid + // cosine-similarity range. Without the clamp, a 0.95 base × 1.10 proximity + // would emit a 1.045 score which violates the semantic of "similarity" and + // breaks downstream consumers that expect a normalized value. + score *= cbm_sem_proximity(a.file_path, b.file_path); + if (score > UNIT_POS) score = UNIT_POS; + if (score < 0.0) score = 0.0; + + return score; +} + +// ── Graph diffusion ───────────────────────────────────────────────── + +pub export fn cbm_sem_diffuse( + combined_c: ?*Vec, + neighbors_c: ?[*]const Vec, + neighbor_count: c_int, + alpha: f32, +) void { + const combined = combined_c orelse return; + const neighbors = neighbors_c orelse return; + if (neighbor_count <= 0) return; + // Blend: combined = (1-α) × combined + α × mean(neighbors) + var mean: Vec = undefined; + @memset(&mean.v, 0.0); + for (neighbors[0..ix(neighbor_count)]) |*neighbor| { + for (&mean.v, neighbor.v) |*m, x| { + m.* += x; + } + } + const inv_n = UNIT_POS / @as(f32, @floatFromInt(neighbor_count)); + const one_minus_alpha = UNIT_POS - alpha; + for (&combined.v, mean.v) |*x, m| { + x.* = fp.mulAdd(f32, one_minus_alpha, x.*, alpha * m * inv_n); + } + cbm_sem_normalize(combined); +}