diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 5834ac89c..9466b7fa8 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -12906,6 +12906,154 @@ static char *snippet_budget_floor(bool json_format, int max_output_tokens) { return json; } +/* Doc -> code candidates (doc_link_candidates): on a Section, the functions, + * whole files and the home folder it is possibly about; on a function or + * method, the sections that possibly describe it; on a file (get_file_outline), + * the sections that possibly describe the file or a folder holding it -- + * highest p first. Candidates with a judged probability for the agent to + * verify, never links. A row whose other end no longer resolves (a delta + * update renamed or removed it) is skipped. */ +enum { MCP_DOC_CANDIDATES_MAX = 5, MCP_DOC_CANDIDATES_SECTION_MAX = 10 }; +static const char MCP_DOC_CANDIDATES_NOTE[] = + "doc<->code candidates by shared terms or the doc's folder, not links: p is the judged " + "probability the section is about the target; verify"; + +/* The evidence fields of a candidate row as shown: kind, position, and (on + * the section side) the shared terms and name tokens. */ +static void doc_candidate_evidence(yyjson_mut_doc *doc, yyjson_mut_val *o, const char *evidence, + bool counts) { + yyjson_doc *ev = evidence ? yyjson_read(evidence, strlen(evidence), 0) : NULL; + yyjson_val *root = ev ? yyjson_doc_get_root(ev) : NULL; + static const char *const STRS[] = {"kind", "position", NULL}; + for (int k = 0; root && STRS[k]; k++) { + yyjson_val *v = yyjson_obj_get(root, STRS[k]); + if (v && yyjson_is_str(v)) { + yyjson_mut_obj_add_strcpy(doc, o, STRS[k], yyjson_get_str(v)); + } + } + static const char *const INTS[] = {"shared_terms", "name_tokens", NULL}; + for (int k = 0; counts && root && INTS[k]; k++) { + yyjson_val *v = yyjson_obj_get(root, INTS[k]); + if (v && yyjson_is_int(v)) { + yyjson_mut_obj_add_int(doc, o, INTS[k], (int)yyjson_get_int(v)); + } + } + yyjson_doc_free(ev); +} + +/* rows -> an array of shown candidates; the other end of each row is its + * target (by_section) or its section. Returns how many resolved. */ +static int doc_candidates_array(yyjson_mut_doc *doc, yyjson_mut_val *arr, cbm_store_t *store, + const char *project, const cbm_doc_candidate_t *rows, int n, + bool by_section) { + int added = 0; + for (int i = 0; i < n; i++) { + cbm_node_t other = {0}; + if (cbm_store_find_node_by_qn(store, project, + by_section ? rows[i].target_qn : rows[i].section_qn, + &other) != CBM_STORE_OK) { + continue; + } + yyjson_mut_val *o = yyjson_mut_obj(doc); + yyjson_mut_obj_add_strcpy(doc, o, by_section ? "qualified_name" : "section", + other.qualified_name ? other.qualified_name : ""); + if (by_section) { + yyjson_mut_obj_add_strcpy(doc, o, "label", other.label ? other.label : ""); + } + yyjson_mut_obj_add_strcpy(doc, o, "file_path", other.file_path ? other.file_path : ""); + yyjson_mut_obj_add_int(doc, o, "start_line", other.start_line); + if (by_section) { + yyjson_mut_obj_add_int(doc, o, "end_line", other.end_line); + } + yyjson_mut_obj_add_real(doc, o, "p", (double)(int)(rows[i].p * 1000.0 + 0.5) / 1000.0); + doc_candidate_evidence(doc, o, rows[i].evidence, by_section); + yyjson_mut_arr_append(arr, o); + added++; + free_node_contents(&other); + } + return added; +} + +static void add_doc_candidates(yyjson_mut_doc *doc, yyjson_mut_val *root, cbm_store_t *store, + const cbm_node_t *node) { + if (!store || !node->label || !node->qualified_name || !node->project) { + return; + } + bool section = strcmp(node->label, "Section") == 0; + if (!section && strcmp(node->label, "Function") != 0 && strcmp(node->label, "Method") != 0) { + return; + } + cbm_doc_candidate_t *rows = NULL; + int n = 0; + if (cbm_store_doc_candidates_get(store, node->project, section ? node->qualified_name : NULL, + section ? NULL : node->qualified_name, + section ? MCP_DOC_CANDIDATES_SECTION_MAX + : MCP_DOC_CANDIDATES_MAX, + &rows, &n) != CBM_STORE_OK) { + return; + } + yyjson_mut_val *arr = yyjson_mut_arr(doc); + int added = doc_candidates_array(doc, arr, store, node->project, rows, n, section); + cbm_store_doc_candidates_free(rows, n); + if (added > 0) { + yyjson_mut_obj_add_val(doc, root, section ? "possibly_about" : "possibly_described_in", + arr); + yyjson_mut_obj_add_str(doc, root, "candidates_note", MCP_DOC_CANDIDATES_NOTE); + } +} + +/* get_file_outline: the sections possibly about the file as a whole or about + * a folder holding it (NULL: none). */ +static yyjson_mut_val *file_doc_candidates(yyjson_mut_doc *doc, cbm_store_t *store, + const char *project, const char *path) { + cbm_doc_candidate_t *rows = NULL; + int n = 0; + if (cbm_store_doc_candidates_for_path(store, project, path, MCP_DOC_CANDIDATES_MAX, &rows, + &n) != CBM_STORE_OK) { + return NULL; + } + yyjson_mut_val *arr = yyjson_mut_arr(doc); + int added = doc_candidates_array(doc, arr, store, project, rows, n, false); + cbm_store_doc_candidates_free(rows, n); + return added > 0 ? arr : NULL; +} + +static const char *mut_str(yyjson_mut_val *o, const char *key) { + yyjson_mut_val *v = yyjson_mut_obj_get(o, key); + return v && yyjson_mut_is_str(v) ? yyjson_mut_get_str(v) : ""; +} + +/* The tree form of file_doc_candidates. */ +static void file_doc_candidates_tree(cbm_sb_t *sb, cbm_store_t *store, const char *project, + const char *path) { + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + yyjson_mut_val *arr = doc ? file_doc_candidates(doc, store, project, path) : NULL; + if (arr) { + static const char *const columns[] = {"section", "file_path", "line", "p", "kind"}; + cbm_tree_table_header(sb, "possibly_described_in", (int)yyjson_mut_arr_size(arr), columns, + (int)(sizeof(columns) / sizeof(columns[0]))); + size_t i = 0; + size_t max = 0; + yyjson_mut_val *o = NULL; + yyjson_mut_arr_foreach(arr, i, max, o) { + char line[CBM_SZ_32]; + char p[CBM_SZ_32]; + snprintf(line, sizeof(line), "%d", + (int)yyjson_mut_get_int(yyjson_mut_obj_get(o, "start_line"))); + snprintf(p, sizeof(p), "%.2f", yyjson_mut_get_real(yyjson_mut_obj_get(o, "p"))); + cbm_tree_row_begin(sb); + cbm_tree_cell_str(sb, mut_str(o, "section"), true); + cbm_tree_cell_str(sb, mut_str(o, "file_path"), false); + cbm_tree_cell_str(sb, line, false); + cbm_tree_cell_str(sb, p, false); + cbm_tree_cell_str(sb, mut_str(o, "kind"), false); + cbm_tree_row_end(sb); + } + cbm_tree_scalar_str(sb, "candidates_note", MCP_DOC_CANDIDATES_NOTE); + } + yyjson_mut_doc_free(doc); +} + static char *build_snippet_response(cbm_mcp_server_t *srv, cbm_node_t *node, const char *match_method, bool include_neighbors, cbm_node_t *alternatives, int alt_count, const char *args) { @@ -13122,6 +13270,8 @@ static char *build_snippet_response(cbm_mcp_server_t *srv, cbm_node_t *node, yyjson_mut_obj_add_val(doc, root_obj, "alternatives", arr); } + add_doc_candidates(doc, root_obj, store, node); + bool bounded_default = max_output_tokens > 0; size_t snippet_byte_budget = bounded_default ? (size_t)max_output_tokens * (size_t)MCP_OUTPUT_BYTES_PER_TOKEN_ESTIMATE @@ -13387,6 +13537,9 @@ static char *handle_get_file_outline(cbm_mcp_server_t *srv, const char *args) { cbm_tree_scalar_int(&sb, "limit", limit); cbm_tree_scalar_int(&sb, "returned", row_count); cbm_tree_scalar_bool(&sb, "has_more", (int64_t)offset + row_count < total); + if (offset == 0) { + file_doc_candidates_tree(&sb, store, project, normalized_path); + } payload = cbm_sb_finish(&sb); } else { yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); @@ -13422,6 +13575,12 @@ static char *handle_get_file_outline(cbm_mcp_server_t *srv, const char *args) { yyjson_mut_obj_add_int(doc, object, "limit", limit); yyjson_mut_obj_add_int(doc, object, "returned", row_count); yyjson_mut_obj_add_bool(doc, object, "has_more", (int64_t)offset + row_count < total); + yyjson_mut_val *described = + offset == 0 ? file_doc_candidates(doc, store, project, normalized_path) : NULL; + if (described) { + yyjson_mut_obj_add_val(doc, object, "possibly_described_in", described); + yyjson_mut_obj_add_str(doc, object, "candidates_note", MCP_DOC_CANDIDATES_NOTE); + } payload = yy_doc_to_str(doc); yyjson_mut_doc_free(doc); } diff --git a/src/pipeline/pass_semantic_edges.c b/src/pipeline/pass_semantic_edges.c index 14597289e..d406306e3 100644 --- a/src/pipeline/pass_semantic_edges.c +++ b/src/pipeline/pass_semantic_edges.c @@ -30,6 +30,8 @@ #include "foundation/platform.h" #include "foundation/profile.h" +#include +#include #include #include #include @@ -71,6 +73,7 @@ typedef struct { int64_t target_id; float score; bool same_file; + bool below; /* under the threshold: recorded for CBM_SEM_PAIR_SIGNALS only, never admitted */ /* Canonical admission keys (determinism): func index of the discovering * side, its candidate rank, and the partner's func index. The sequential * admission pass replays pairs in (i, c) order so which pairs win the @@ -93,7 +96,7 @@ static void deferred_buf_init(deferred_edge_buf_t *buf) { } static void deferred_buf_push(deferred_edge_buf_t *buf, int64_t src, int64_t tgt, float score, - bool same_file, int i, int j, int c) { + bool same_file, bool below, int i, int j, int c) { if (buf->count >= buf->cap) { int nc = buf->cap < CBM_SZ_256 ? CBM_SZ_256 : buf->cap * GROW; deferred_edge_t *grown = @@ -108,6 +111,7 @@ static void deferred_buf_push(deferred_edge_buf_t *buf, int64_t src, int64_t tgt .target_id = tgt, .score = score, .same_file = same_file, + .below = below, .i = i, .j = j, .c = c}; @@ -325,29 +329,9 @@ static const char *file_ext(const char *path) { return dot ? dot : ""; } -/* Extract a JSON string value by key (simple strstr-based, no full parse). */ +/* Extract a JSON string value by key, escapes honoured (cbm_sem_json_str). */ static const char *json_str_value(const char *json, const char *key, char *buf, int bufsize) { - if (!json || !key) { - return NULL; - } - char search[CBM_SZ_64]; - snprintf(search, sizeof(search), "\"%s\":\"", key); - const char *start = strstr(json, search); - if (!start) { - return NULL; - } - start += strlen(search); - const char *end = strchr(start, '"'); - if (!end) { - return NULL; - } - int len = (int)(end - start); - if (len >= bufsize) { - len = bufsize - SKIP_ONE; - } - memcpy(buf, start, (size_t)len); - buf[len] = '\0'; - return buf; + return cbm_sem_json_str(json, key, buf, bufsize); } /* Extract a JSON array of strings by key. Returns count. */ @@ -724,19 +708,10 @@ static void vec_build_worker(int worker_id, void *ctx_ptr) { token_index[t] = cbm_sem_corpus_token_index(vc->corpus, tokens[t]); } - /* TF-IDF weights */ + /* TF-IDF terms, keyed by corpus token (cbm_sem_tfidf_terms). */ int *indices = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)tc * sizeof(int)); float *weights = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)tc * sizeof(float)); - int tfidf_len = 0; - for (int t = 0; t < tc; t++) { - float idf = token_index ? cbm_sem_corpus_idf_at(vc->corpus, token_index[t]) - : cbm_sem_corpus_idf(vc->corpus, tokens[t]); - if (idf > 0.0F) { - indices[tfidf_len] = t; - weights[tfidf_len] = idf; - tfidf_len++; - } - } + int tfidf_len = cbm_sem_tfidf_terms(vc->corpus, tokens, token_index, tc, indices, weights); vc->funcs[f].tfidf_indices = indices; vc->funcs[f].tfidf_weights = weights; vc->funcs[f].tfidf_len = tfidf_len; @@ -833,6 +808,7 @@ typedef struct { uint64_t *signatures; int *edge_counts; /* budget applied sequentially in phase6b (determinism) */ cbm_sem_config_t cfg; + float record_floor; /* kept from here up; under the threshold: signals dump only */ int func_count; /* LSH buckets (read-only during scoring) */ @@ -927,13 +903,13 @@ static void score_try_emit(score_ctx_t *sc, int i, int j, int c, deferred_edge_b return; } float score = cbm_sem_combined_score(&sc->funcs[i], &sc->funcs[j], &sc->cfg); - if (score < sc->cfg.threshold) { + if (score < sc->record_floor) { return; } bool same_file = sc->funcs[i].file_path && sc->funcs[j].file_path && strcmp(sc->funcs[i].file_path, sc->funcs[j].file_path) == 0; - deferred_buf_push(my_buf, sc->funcs[i].node_id, sc->funcs[j].node_id, score, same_file, i, j, - c); + deferred_buf_push(my_buf, sc->funcs[i].node_id, sc->funcs[j].node_id, score, same_file, + score < sc->cfg.threshold, i, j, c); } static void score_worker(int worker_id, void *ctx_ptr) { @@ -1138,8 +1114,31 @@ static int cmp_deferred_edge_canonical(const void *pa, const void *pb) { * and apply the per-node max_edges budget HERE — single-threaded — so the * admitted edge set is a pure function of the (canonically sorted) inputs, * independent of worker count and scheduling. */ +/* CBM_SEM_PAIR_SIGNALS=: one line per recorded pair (from the record + * floor up, CBM_SEM_PAIR_SIGNALS_FLOOR, default the threshold), in canonical + * order: both qualified names, the score, every signal value and whether the + * pair was admitted. The measuring harness for the semantic engine: judged + * samples are drawn from it, stratified by score. Off unless set. */ +static void pair_signals_line(FILE *f, const cbm_gbuf_t *gbuf, const cbm_sem_func_t *funcs, + const deferred_edge_t *de, bool admitted) { + cbm_sem_signals_t s; + cbm_sem_signal_values(&funcs[de->i], &funcs[de->j], &s); + const cbm_gbuf_node_t *a = cbm_gbuf_find_by_id(gbuf, de->source_id); + const cbm_gbuf_node_t *b = cbm_gbuf_find_by_id(gbuf, de->target_id); + (void)fprintf( + f, + "%s\t%s\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.4f\t%d\t%d" + "\t%s:%d-%d\t%s:%d-%d\n", + a && a->qualified_name ? a->qualified_name : "?", + b && b->qualified_name ? b->qualified_name : "?", (double)de->score, (double)s.tfidf, + (double)s.ri, (double)s.minhash, (double)s.api, (double)s.type, (double)s.decorator, + (double)s.struct_profile, (double)s.proximity, de->same_file ? 1 : 0, admitted ? 1 : 0, + a && a->file_path ? a->file_path : "?", a ? a->start_line : 0, a ? a->end_line : 0, + b && b->file_path ? b->file_path : "?", b ? b->start_line : 0, b ? b->end_line : 0); +} + static int phase6b_merge_edges(cbm_gbuf_t *gbuf, deferred_edge_buf_t *worker_bufs, int worker_count, - int *edge_counts, int max_edges) { + int *edge_counts, int max_edges, const cbm_sem_func_t *funcs) { int total_pairs = 0; for (int w = 0; w < worker_count; w++) { total_pairs += worker_bufs[w].count; @@ -1175,33 +1174,77 @@ static int phase6b_merge_edges(cbm_gbuf_t *gbuf, deferred_edge_buf_t *worker_buf * chase the near-threshold SEMANTICALLY_RELATED flicker: find the pair that * flips, then compare its score between runs. */ FILE *pair_dump = NULL; + FILE *signals = NULL; { char dump_path[CBM_SZ_1K]; if (cbm_safe_getenv("CBM_SEM_PAIR_DUMP", dump_path, sizeof(dump_path), NULL)) { pair_dump = cbm_fopen(dump_path, "w"); } + if (funcs && cbm_safe_getenv("CBM_SEM_PAIR_SIGNALS", dump_path, sizeof(dump_path), NULL)) { + signals = cbm_fopen(dump_path, "w"); + } + } + + /* Admission: best-first by score under the per-function budget + * (cbm_sem_admit_best_first); the dumps and the edge inserts below keep + * the canonical order. */ + float *scores = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)n * sizeof(float)); + int *fa = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)n * sizeof(int)); + int *fb = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)n * sizeof(int)); + bool *eligible = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)n * sizeof(bool)); + bool *admitted = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)n * sizeof(bool)); + bool ranked = scores && fa && fb && eligible && admitted; + for (int e = 0; ranked && e < n; e++) { + scores[e] = pairs[e].score; + fa[e] = pairs[e].i; + fb[e] = pairs[e].j; + eligible[e] = !pairs[e].below; + } + ranked = ranked && cbm_sem_admit_best_first(scores, fa, fb, eligible, n, max_edges, edge_counts, + admitted); + if (!ranked) { + cbm_log_warn("semantic_edges.admission", "reason", "out_of_memory", "pairs", itoa_log(n)); } int total_edges = 0; - for (int e = 0; e < n; e++) { + for (int e = 0; ranked && e < n; e++) { deferred_edge_t *de = &pairs[e]; + if (de->below) { + if (signals) { + pair_signals_line(signals, gbuf, funcs, de, false); + } + continue; + } if (pair_dump) { fprintf(pair_dump, "%d %d %.9g\n", de->i, de->j, (double)de->score); } - if (edge_counts[de->i] >= max_edges || edge_counts[de->j] >= max_edges) { + if (signals) { + pair_signals_line(signals, gbuf, funcs, de, admitted[e]); + } + if (!admitted[e]) { continue; } + /* p from the score as stored, so one shown score carries one p. */ + char score_text[CBM_SZ_32]; + snprintf(score_text, sizeof(score_text), "%.3f", (double)de->score); char props[PROPS_BUF]; - snprintf(props, sizeof(props), "{\"score\":%.3f,\"same_file\":%s}", de->score, - de->same_file ? "true" : "false"); + snprintf(props, sizeof(props), "{\"score\":%s,\"same_file\":%s,\"p\":%.2f}", score_text, + de->same_file ? "true" : "false", + (double)cbm_sem_calibrated_p(strtof(score_text, NULL))); cbm_gbuf_insert_edge(gbuf, de->source_id, de->target_id, "SEMANTICALLY_RELATED", props); - edge_counts[de->i]++; - edge_counts[de->j]++; total_edges++; } + cbm_free(CBM_MEM_CLASS_SEMANTIC, scores); + cbm_free(CBM_MEM_CLASS_SEMANTIC, fa); + cbm_free(CBM_MEM_CLASS_SEMANTIC, fb); + cbm_free(CBM_MEM_CLASS_SEMANTIC, eligible); + cbm_free(CBM_MEM_CLASS_SEMANTIC, admitted); if (pair_dump) { (void)fclose(pair_dump); } + if (signals) { + (void)fclose(signals); + } cbm_free(CBM_MEM_CLASS_SEMANTIC, pairs); return total_edges; } @@ -1542,13 +1585,14 @@ static void sem_corpus_dump(const cbm_sem_corpus_t *corpus, const char *suffix) /* Phase 6a: score candidate pairs in parallel and collect deferred edges. */ static void phase6a_score_candidates(cbm_sem_func_t *funcs, uint64_t *signatures, int *edge_counts, sem_bucket_t **band_buckets, cbm_sem_config_t cfg, - deferred_edge_buf_t *worker_bufs, int func_count, - int worker_count) { + float record_floor, deferred_edge_buf_t *worker_bufs, + int func_count, int worker_count) { score_ctx_t sc = { .funcs = funcs, .signatures = signatures, .edge_counts = edge_counts, .cfg = cfg, + .record_floor = record_floor, .func_count = func_count, .band_buckets = (void *)band_buckets, .worker_bufs = worker_bufs, @@ -1619,14 +1663,30 @@ static int run_scoring_phase(cbm_gbuf_t *gbuf, cbm_sem_func_t *funcs, uint64_t * deferred_buf_init(&worker_bufs[w]); } + /* The measuring harness may record pairs below the threshold (never + * admitted): CBM_SEM_PAIR_SIGNALS_FLOOR, only with CBM_SEM_PAIR_SIGNALS. */ + float record_floor = cfg.threshold; + { + char path[CBM_SZ_1K]; /* the dump path: a buffer too short reads as unset */ + char buf[CBM_SZ_64]; + if (cbm_safe_getenv("CBM_SEM_PAIR_SIGNALS", path, sizeof(path), NULL) && + cbm_safe_getenv("CBM_SEM_PAIR_SIGNALS_FLOOR", buf, sizeof(buf), NULL)) { + float floor_value = strtof(buf, NULL); + if (floor_value > 0.0F && floor_value < cfg.threshold) { + record_floor = floor_value; + } + } + } + CBM_PROF_START(t_phase6a); sem_state_dump(funcs, func_count); - phase6a_score_candidates(funcs, signatures, edge_counts, band_buckets, cfg, worker_bufs, - func_count, worker_count); + phase6a_score_candidates(funcs, signatures, edge_counts, band_buckets, cfg, record_floor, + worker_bufs, func_count, worker_count); CBM_PROF_END_N("semantic_edges", "6a_score_parallel", t_phase6a, func_count); CBM_PROF_START(t_phase6b); - int total = phase6b_merge_edges(gbuf, worker_bufs, worker_count, edge_counts, cfg.max_edges); + int total = + phase6b_merge_edges(gbuf, worker_bufs, worker_count, edge_counts, cfg.max_edges, funcs); CBM_PROF_END_N("semantic_edges", "6b_edge_merge_seq", t_phase6b, total); cbm_free(CBM_MEM_CLASS_SEMANTIC, worker_bufs); @@ -1812,6 +1872,1233 @@ static cbm_sem_corpus_t *run_token_phases_batched(cbm_gbuf_t *gbuf, cbm_sem_func /* ── Pass entry point ────────────────────────────────────────────── */ +/* ── Phase 6c: doc sections -> functions (candidate generator) ───── */ + +/* Doc -> code candidates: for every Section node that can be about code, the + * functions whose TF-IDF terms best match the section's heading + body, over + * the FUNCTIONS' corpus (sections never change a function-pair score). + * Gate: no candidates for sections whose heading names a release (v1.2), a + * changelog, licence, contributing, links, contents, install or support + * kind of section, whose body has fewer than SEM_DOC_MIN_WORDS words, or + * whose body is mostly links. + * Targets: never a test function (by path or name); candidates come from + * the section's key terms (held by at most one in SEM_DOC_KEY_DF_DIV + * functions, or SEM_DOC_KEY_DF_MIN) and are scored on every shared term. + * Stored (doc_link_candidates, never graph edges): a section's best + * CBM_SEM_DOC_TOP_K with tfidf >= CBM_SEM_DOC_MIN_SCORE, each with its judged + * probability (cbm_sem_doc_calibrated_p) -- candidates for an agent to + * verify, not links. + * CBM_SEM_DOC_SIGNALS= also dumps every section's best SEM_DOC_TOP_K: + * section and function qualified names, tfidf, rank, shared terms, both + * locations, and the name evidence (the function's name tokens present in the + * section, and the highest idf among them) -- the population the judged + * calibration samples are drawn from. */ +enum { + SEM_DOC_TOP_K = 10, + SEM_DOC_KEY_DF_MIN = 50, + SEM_DOC_KEY_DF_DIV = 20, + SEM_DOC_MAX_TOKENS = 1024, + SEM_DOC_TEXT_MAX = 8192, + SEM_DOC_NAME_TOKENS = 16, + SEM_DOC_MIN_WORDS = 20, + SEM_DOC_LINK_PCT_MAX = 20, /* links per 100 body words */ + SEM_DOC_WORD_MIN_LEN = 3, +}; + +typedef struct { + float score; + int func; + int shared; +} doc_cand_t; + +typedef struct { + int func; + float tfidf; + int rank; + int shared; + int name_shared; /* the function's NAME tokens present in the section */ + float name_idf; /* the highest idf among them (0: none) */ +} doc_hit_t; + +typedef struct { + int file; + float tfidf; + int rank; + int shared; +} doc_file_hit_t; + +typedef struct { + doc_hit_t hits[SEM_DOC_TOP_K]; + int n; + doc_hit_t local[CBM_SEM_DOC_LOCAL_K]; /* inside the home, below the top CBM_SEM_DOC_TOP_K */ + int nlocal; + doc_file_hit_t files[CBM_SEM_DOC_FILE_K]; + int nfiles; +} doc_result_t; + +static int cmp_doc_cand(const void *pa, const void *pb) { + const doc_cand_t *a = pa; + const doc_cand_t *b = pb; + if (a->score != b->score) { + return a->score > b->score ? -1 : 1; + } + return (a->func > b->func) - (a->func < b->func); +} + +static int cmp_section_node(const void *pa, const void *pb) { + const cbm_gbuf_node_t *a = *(const cbm_gbuf_node_t *const *)pa; + const cbm_gbuf_node_t *b = *(const cbm_gbuf_node_t *const *)pb; + int c = strcmp(a->qualified_name ? a->qualified_name : "", + b->qualified_name ? b->qualified_name : ""); + if (c) { + return c; + } + return (a->start_line > b->start_line) - (a->start_line < b->start_line); +} + +/* Heading kinds that are about the project, not about code: a release + * number, or one of these words (a trailing '*' matches any ending). */ +static const char *const DOC_SKIP_HEADING[] = {"changelog", + "change log", + "release note*", + "what's new", + "license", + "licence", + "contribut*", + "acknowledg*", + "credits", + "sponsor*", + "author*", + "code of conduct", + "security policy", + "links", + "references", + "resources", + "further reading", + "table of contents", + "contents", + "toc", + "installation", + "install", + "getting started", + "support", + "community", + "faq", + "roadmap", + "todo", + NULL}; + +static bool doc_word_char(char c) { + return isalnum((unsigned char)c) || c == '\''; +} + +/* Does lowercase heading lc contain word (or word* as a prefix) at a word start? */ +static bool doc_heading_has(const char *lc, const char *word) { + size_t wl = strlen(word); + bool prefix = wl > 0 && word[wl - SKIP_ONE] == '*'; + size_t ml = prefix ? wl - SKIP_ONE : wl; + for (const char *p = lc; *p; p++) { + if (strncmp(p, word, ml) != 0) { + continue; + } + if (p > lc && doc_word_char(p[-SKIP_ONE])) { + continue; + } + if (!prefix && doc_word_char(p[ml])) { + continue; + } + return true; + } + return false; +} + +/* A release-number heading: "v1.2", "1.2.3", "[0.12.15]", "Gin v1.11.0". */ +static bool doc_heading_is_release(const char *lc) { + for (const char *p = lc; *p; p++) { + if (p > lc && doc_word_char(p[-SKIP_ONE])) { + continue; + } + const char *q = *p == 'v' ? p + SKIP_ONE : p; + if (!isdigit((unsigned char)*q)) { + continue; + } + while (isdigit((unsigned char)*q)) { + q++; + } + if (*q == '.' && isdigit((unsigned char)q[SKIP_ONE])) { + return true; + } + } + return false; +} + +/* Can this section be about code? (heading kind, body length, link share) */ +static bool doc_section_gated(const cbm_gbuf_node_t *sec, char *body) { + char lc[CBM_SZ_256]; + size_t n = 0; + for (const char *p = sec->name ? sec->name : ""; *p && n + SKIP_ONE < sizeof(lc); p++) { + lc[n++] = (char)tolower((unsigned char)*p); + } + lc[n] = '\0'; + if (doc_heading_is_release(lc)) { + return false; + } + for (int k = 0; DOC_SKIP_HEADING[k]; k++) { + if (doc_heading_has(lc, DOC_SKIP_HEADING[k])) { + return false; + } + } + if (!json_str_value(sec->properties_json, "docstring", body, SEM_DOC_TEXT_MAX)) { + return false; + } + int words = 0; + int links = 0; + int run = 0; + for (const char *p = body;; p++) { + if (isalpha((unsigned char)*p)) { + run++; + continue; + } + words += run >= SEM_DOC_WORD_MIN_LEN ? SKIP_ONE : 0; + run = 0; + if (!*p) { + break; + } + if (strncmp(p, "http://", sizeof("http://") - SKIP_ONE) == 0 || + strncmp(p, "https://", sizeof("https://") - SKIP_ONE) == 0 || + strncmp(p, "](", sizeof("](") - SKIP_ONE) == 0) { + links++; + } + } + return words >= SEM_DOC_MIN_WORDS && links * 100 < SEM_DOC_LINK_PCT_MAX * words; +} + +/* A test file: in a test directory, or a test file name. */ +static bool doc_is_test_path(const char *path) { + static const char *const DIRS[] = {"test/", "tests/", "__tests__/", "testing/", "spec/", + "specs/", "testdata/", "fixture/", "fixtures/", NULL}; + path = path ? path : ""; + for (int k = 0; DIRS[k]; k++) { + for (const char *p = path; (p = strstr(p, DIRS[k])) != NULL; p++) { + if (p == path || p[-SKIP_ONE] == '/') { + return true; + } + } + } + const char *base = strrchr(path, '/'); + base = base ? base + SKIP_ONE : path; + return strncmp(base, "test_", sizeof("test_") - SKIP_ONE) == 0 || strstr(base, "_test.") || + strstr(base, ".test.") || strstr(base, ".spec.") || strstr(base, "Test.") || + strstr(base, "Tests."); +} + +/* A test function, by its file or name. */ +static bool doc_is_test_function(const cbm_gbuf_node_t *fn) { + if (doc_is_test_path(fn ? fn->file_path : NULL)) { + return true; + } + const char *name = fn && fn->name ? fn->name : ""; + return strncmp(name, "test_", sizeof("test_") - SKIP_ONE) == 0 || + (strncmp(name, "Test", sizeof("Test") - SKIP_ONE) == 0 && + (isupper((unsigned char)name[4]) || name[4] == '_')); +} + +/* The section's tokens (heading + body); the caller frees each token. */ +static int doc_section_tokens(const cbm_gbuf_node_t *n, const char *body, char **tokens) { + int count = cbm_sem_tokenize(n->name, tokens, SEM_DOC_MAX_TOKENS); + if (body && count < SEM_DOC_MAX_TOKENS) { + count += cbm_sem_tokenize(body, tokens + count, SEM_DOC_MAX_TOKENS - count); + } + return count; +} + +/* Dot product and shared-term count of two ascending term lists. */ +static float doc_terms_dot(const int *ia, const float *wa, int na, const int *ib, const float *wb, + int nb, int *shared) { + float dot = 0.0F; + int a = 0; + int b = 0; + *shared = 0; + while (a < na && b < nb) { + if (ia[a] == ib[b]) { + dot += wa[a] * wb[b]; + (*shared)++; + a++; + b++; + } else if (ia[a] < ib[b]) { + a++; + } else { + b++; + } + } + return dot; +} + +static float terms_norm(const float *w, int n) { + float s = 0.0F; + for (int k = 0; k < n; k++) { + s += w[k] * w[k]; + } + return sqrtf(s); +} + +/* ── Phase 6c: whole files and the doc's home folder ── */ + +/* Whole-file candidates: a file's terms are the sum of its non-test + * functions' TF-IDF vectors and the TF-IDF vectors of its other definitions' + * names (a config file's keys are Variable nodes), cut to its + * SEM_DOC_FILE_TERMS heaviest terms (ties by token index), so a file's vector + * costs at most what one long function's does. */ +enum { SEM_DOC_FILE_TERMS = 64 }; +static const char *const DOC_FILE_DEF_LABELS[] = {"Class", "Struct", "Interface", "Enum", + "Type", "Trait", "Field", "Variable", + "Constant", "Macro", NULL}; + +typedef struct { + int count; + const cbm_gbuf_node_t **node; /* per file: its File node */ + int *off; /* count + 1: each file's terms in idx / w */ + int *idx; + float *w; + float *norm; + int *post_off; /* nterms + 1: the files holding each term */ + int *post_f; + CBMHashTable *by_path; /* file path (graph-owned) -> file index + 1 */ +} doc_files_t; + +typedef struct { + int idx; + float w; +} doc_term_t; + +static int cmp_term_idx(const void *pa, const void *pb) { + const doc_term_t *a = pa; + const doc_term_t *b = pb; + return (a->idx > b->idx) - (a->idx < b->idx); +} + +static int cmp_term_heavy(const void *pa, const void *pb) { + const doc_term_t *a = pa; + const doc_term_t *b = pb; + if (a->w != b->w) { + return a->w > b->w ? -1 : 1; + } + return (a->idx > b->idx) - (a->idx < b->idx); +} + +static void doc_files_free(doc_files_t *df) { + cbm_free(CBM_MEM_CLASS_SEMANTIC, df->node); + cbm_free(CBM_MEM_CLASS_SEMANTIC, df->off); + cbm_free(CBM_MEM_CLASS_SEMANTIC, df->idx); + cbm_free(CBM_MEM_CLASS_SEMANTIC, df->w); + cbm_free(CBM_MEM_CLASS_SEMANTIC, df->norm); + cbm_free(CBM_MEM_CLASS_SEMANTIC, df->post_off); + cbm_free(CBM_MEM_CLASS_SEMANTIC, df->post_f); + if (df->by_path) { + cbm_ht_free(df->by_path); + } + memset(df, 0, sizeof(*df)); +} + +static int doc_file_of(const doc_files_t *df, const char *path) { + return path ? (int)(intptr_t)cbm_ht_get(df->by_path, path) - SKIP_ONE : -SKIP_ONE; +} + +/* The scratch terms of one file: append n terms, growing the buffer. */ +typedef struct { + doc_term_t *t; + int n; + int cap; +} doc_scratch_t; + +static bool doc_scratch_add(doc_scratch_t *sc, const int *idx, const float *w, int n) { + if (sc->n + n > sc->cap) { + int cap = sc->cap ? sc->cap : CBM_SZ_256; + while (cap < sc->n + n) { + cap *= 2; + } + doc_term_t *t = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, sc->t, (size_t)cap * sizeof(*t)); + if (!t) { + return false; + } + sc->t = t; + sc->cap = cap; + } + for (int k = 0; k < n; k++) { + sc->t[sc->n++] = (doc_term_t){.idx = idx[k], .w = w[k]}; + } + return true; +} + +/* Sum the scratch terms per token and keep the heaviest SEM_DOC_FILE_TERMS, + * written at df->idx/w + off in ascending token order; returns how many. */ +static int doc_scratch_reduce(doc_scratch_t *sc, doc_files_t *df, int off) { + if (sc->n == 0) { + return 0; + } + qsort(sc->t, (size_t)sc->n, sizeof(*sc->t), cmp_term_idx); + int m = 0; + for (int k = 0; k < sc->n; k++) { + if (m > 0 && sc->t[m - SKIP_ONE].idx == sc->t[k].idx) { + sc->t[m - SKIP_ONE].w += sc->t[k].w; + } else { + sc->t[m++] = sc->t[k]; + } + } + if (m > SEM_DOC_FILE_TERMS) { + qsort(sc->t, (size_t)m, sizeof(*sc->t), cmp_term_heavy); + m = SEM_DOC_FILE_TERMS; + qsort(sc->t, (size_t)m, sizeof(*sc->t), cmp_term_idx); + } + for (int k = 0; k < m; k++) { + df->idx[off + k] = sc->t[k].idx; + df->w[off + k] = sc->t[k].w; + } + return m; +} + +/* The definitions (non-function) of every indexed non-test file, grouped by + * file: def_off (count + 1) into def_node. */ +static bool doc_files_defs(const cbm_gbuf_t *gbuf, const doc_files_t *df, int **def_off, + const cbm_gbuf_node_t ***def_node) { + *def_off = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)df->count + SKIP_ONE) * sizeof(int)); + if (!*def_off) { + return false; + } + for (int pass = 0; pass < 2; pass++) { + int *cursor = + pass ? cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)df->count * sizeof(int)) : NULL; + if (pass && !cursor) { + return false; + } + for (int l = 0; DOC_FILE_DEF_LABELS[l]; l++) { + const cbm_gbuf_node_t **nodes = NULL; + int n = 0; + if (cbm_gbuf_find_by_label(gbuf, DOC_FILE_DEF_LABELS[l], &nodes, &n) != 0) { + continue; + } + for (int i = 0; i < n; i++) { + int f = nodes[i]->name ? doc_file_of(df, nodes[i]->file_path) : -SKIP_ONE; + if (f >= 0 && !pass) { + (*def_off)[f + SKIP_ONE]++; + } else if (f >= 0) { + (*def_node)[(*def_off)[f] + cursor[f]++] = nodes[i]; + } + } + } + if (!pass) { + for (int f = 0; f < df->count; f++) { + (*def_off)[f + SKIP_ONE] += (*def_off)[f]; + } + *def_node = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, + ((size_t)(*def_off)[df->count] + SKIP_ONE) * sizeof(**def_node)); + if (!*def_node) { + return false; + } + } + cbm_free(CBM_MEM_CLASS_SEMANTIC, cursor); + } + return true; +} + +/* The scratch terms of one definition's name. */ +static bool doc_scratch_name(doc_scratch_t *sc, const cbm_sem_corpus_t *corpus, const char *name) { + char *tok[SEM_DOC_NAME_TOKENS]; + int idx[SEM_DOC_NAME_TOKENS]; + float w[SEM_DOC_NAME_TOKENS]; + int nt = cbm_sem_tokenize(name, tok, SEM_DOC_NAME_TOKENS); + int nw = nt > 0 ? cbm_sem_tfidf_terms(corpus, tok, NULL, nt, idx, w) : 0; + for (int t = 0; t < nt; t++) { + cbm_free(CBM_MEM_CLASS_SEMANTIC, tok[t]); + } + return doc_scratch_add(sc, idx, w, nw); +} + +/* Each file's term vector, norm, and the term -> files postings. */ +static bool doc_files_vectors(const cbm_gbuf_t *gbuf, doc_files_t *df, const cbm_sem_func_t *funcs, + const int *ffile, int func_count, const cbm_sem_corpus_t *corpus, + int nterms) { + int *fn_off = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)df->count + SKIP_ONE) * sizeof(int)); + int *fn_list = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)func_count + SKIP_ONE) * sizeof(int)); + int *def_off = NULL; + const cbm_gbuf_node_t **def_node = NULL; + doc_scratch_t sc = {0}; + bool ok = fn_off && fn_list && doc_files_defs(gbuf, df, &def_off, &def_node); + for (int f = 0; ok && f < func_count; f++) { + fn_off[ffile[f] + SKIP_ONE] += ffile[f] >= 0; + } + for (int i = 0; ok && i < df->count; i++) { + fn_off[i + SKIP_ONE] += fn_off[i]; + } + for (int f = 0; ok && f < func_count; f++) { + if (ffile[f] >= 0) { + fn_list[fn_off[ffile[f]]++] = f; /* fn_off[i] ends at file i's end: shifted below */ + } + } + for (int i = df->count; ok && i > 0; i--) { + fn_off[i] = fn_off[i - SKIP_ONE]; + } + if (ok) { + fn_off[0] = 0; + } + for (int i = 0; ok && i < df->count; i++) { + sc.n = 0; + for (int k = fn_off[i]; ok && k < fn_off[i + SKIP_ONE]; k++) { + const cbm_sem_func_t *fn = &funcs[fn_list[k]]; + ok = doc_scratch_add(&sc, fn->tfidf_indices, fn->tfidf_weights, fn->tfidf_len); + } + for (int k = def_off[i]; ok && k < def_off[i + SKIP_ONE]; k++) { + ok = doc_scratch_name(&sc, corpus, def_node[k]->name); + } + int m = ok ? doc_scratch_reduce(&sc, df, df->off[i]) : 0; + df->off[i + SKIP_ONE] = df->off[i] + m; + df->norm[i] = terms_norm(df->w + df->off[i], m); + for (int k = 0; k < m; k++) { + df->post_off[df->idx[df->off[i] + k] + SKIP_ONE]++; + } + } + cbm_free(CBM_MEM_CLASS_SEMANTIC, sc.t); + cbm_free(CBM_MEM_CLASS_SEMANTIC, fn_off); + cbm_free(CBM_MEM_CLASS_SEMANTIC, fn_list); + cbm_free(CBM_MEM_CLASS_SEMANTIC, def_off); + cbm_free(CBM_MEM_CLASS_SEMANTIC, def_node); + if (!ok) { + return false; + } + for (int t = 0; t < nterms; t++) { + df->post_off[t + SKIP_ONE] += df->post_off[t]; + } + df->post_f = + cbm_alloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)df->post_off[nterms] + SKIP_ONE) * sizeof(int)); + int *cursor = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)nterms + SKIP_ONE) * sizeof(int)); + ok = df->post_f && cursor; + for (int i = 0; ok && i < df->count; i++) { + for (int k = df->off[i]; k < df->off[i + SKIP_ONE]; k++) { + int t = df->idx[k]; + df->post_f[df->post_off[t] + cursor[t]++] = i; + } + } + cbm_free(CBM_MEM_CLASS_SEMANTIC, cursor); + return ok; +} + +/* The indexed non-test files (their File nodes) with their term vectors; + * ffile[f] = function f's file (-1: a test function or no File node). */ +static bool doc_files_build(const cbm_gbuf_t *gbuf, const cbm_sem_func_t *funcs, int func_count, + const bool *is_test, const cbm_sem_corpus_t *corpus, int nterms, + doc_files_t *df, int *ffile) { + memset(df, 0, sizeof(*df)); + const cbm_gbuf_node_t **files = NULL; + int nfiles = 0; + if (cbm_gbuf_find_by_label(gbuf, "File", &files, &nfiles) != 0) { + nfiles = 0; + } + df->by_path = cbm_ht_create_in(CBM_MEM_CLASS_SEMANTIC, (uint32_t)nfiles + SKIP_ONE); + df->node = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)nfiles + SKIP_ONE) * sizeof(*df->node)); + if (!df->by_path || !df->node) { + return false; + } + for (int i = 0; i < nfiles; i++) { + const char *path = files[i]->file_path; + if (path && path[0] && !doc_is_test_path(path) && !cbm_ht_has(df->by_path, path)) { + df->node[df->count] = files[i]; + cbm_ht_set(df->by_path, path, (void *)(intptr_t)(df->count + SKIP_ONE)); + df->count++; + } + } + for (int f = 0; f < func_count; f++) { + const cbm_gbuf_node_t *fn = cbm_gbuf_find_by_id(gbuf, funcs[f].node_id); + ffile[f] = is_test[f] || !fn ? -SKIP_ONE : doc_file_of(df, fn->file_path); + } + size_t cap = (size_t)df->count * SEM_DOC_FILE_TERMS + SKIP_ONE; + df->off = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)df->count + SKIP_ONE) * sizeof(int)); + df->idx = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, cap * sizeof(int)); + df->w = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, cap * sizeof(float)); + df->norm = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)df->count + SKIP_ONE) * sizeof(float)); + df->post_off = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)nterms + SKIP_ONE) * sizeof(int)); + return df->off && df->idx && df->w && df->norm && df->post_off && + doc_files_vectors(gbuf, df, funcs, ffile, func_count, corpus, nterms); +} + +/* A doc's home folder: the folder whose code it documents. From the doc's + * directory, a docs folder (doc, docs, documentation: the deepest one) + * documents its parent; then climb while the folder holds no indexed code + * (a non-test file with terms). "" = the repository root: the doc documents + * the whole project. Returns an allocated string (NULL: out of memory). */ +static bool doc_dir_is_docs(const char *name, size_t n) { + static const char *const DOCS[] = {"doc", "docs", "documentation", NULL}; + for (int k = 0; DOCS[k]; k++) { + if (strlen(DOCS[k]) == n && strncasecmp(name, DOCS[k], n) == 0) { + return true; + } + } + return false; +} + +static char *doc_home(const char *doc_path, const CBMHashTable *code_dirs) { + const char *slash = doc_path ? strrchr(doc_path, '/') : NULL; + size_t len = slash ? (size_t)(slash - doc_path) : 0; + char *dir = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, len + SKIP_ONE); + if (!dir) { + return NULL; + } + memcpy(dir, doc_path ? doc_path : "", len); + dir[len] = '\0'; + size_t cut = len + SKIP_ONE; /* the deepest docs folder's start (none) */ + for (size_t a = 0; a < len;) { + const char *end = strchr(dir + a, '/'); + size_t b = end ? (size_t)(end - dir) : len; + if (doc_dir_is_docs(dir + a, b - a)) { + cut = a; + } + a = b + SKIP_ONE; + } + if (cut <= len) { + len = cut > 0 ? cut - SKIP_ONE : 0; /* the docs folder's parent */ + dir[len] = '\0'; + } + while (len > 0 && !cbm_ht_has(code_dirs, dir)) { + const char *up = strrchr(dir, '/'); + len = up ? (size_t)(up - dir) : 0; + dir[len] = '\0'; + } + return dir; +} + +/* Every folder holding indexed code: the directories (and their ancestors) + * of the files with terms; keys owned by the table's caller (freed by + * doc_code_dirs_free). */ +static void doc_free_key(const char *key, void *value, void *userdata) { + (void)value; + (void)userdata; + cbm_free(CBM_MEM_CLASS_SEMANTIC, (void *)key); +} + +static void doc_code_dirs_free(CBMHashTable *dirs) { + if (dirs) { + cbm_ht_foreach(dirs, doc_free_key, NULL); + cbm_ht_free(dirs); + } +} + +static CBMHashTable *doc_code_dirs(const doc_files_t *df) { + CBMHashTable *dirs = cbm_ht_create_in(CBM_MEM_CLASS_SEMANTIC, CBM_SZ_256); + for (int i = 0; dirs && i < df->count; i++) { + const char *path = df->node[i]->file_path; + if (df->off[i + SKIP_ONE] == df->off[i]) { + continue; + } + for (const char *p = strchr(path, '/'); p; p = strchr(p + SKIP_ONE, '/')) { + size_t n = (size_t)(p - path); + char *key = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, n + SKIP_ONE); + if (!key) { + doc_code_dirs_free(dirs); + return NULL; + } + memcpy(key, path, n); + key[n] = '\0'; + if (cbm_ht_has(dirs, key)) { + cbm_free(CBM_MEM_CLASS_SEMANTIC, key); + } else { + cbm_ht_set(dirs, key, (void *)(intptr_t)SKIP_ONE); + } + } + } + return dirs; +} + +/* A candidate's position: inside the home (or the home folder itself), + * outside it, or no position (the home is the repository root). */ +static cbm_sem_doc_pos_t doc_position(const char *home, const char *path) { + if (!home || !home[0]) { + return CBM_SEM_DOC_POS_GLOBAL; + } + size_t n = strlen(home); + return path && strncmp(path, home, n) == 0 && (path[n] == '/' || path[n] == '\0') + ? CBM_SEM_DOC_POS_LOCAL + : CBM_SEM_DOC_POS_OUTSIDE; +} + +static const char *doc_position_name(cbm_sem_doc_pos_t pos) { + return pos == CBM_SEM_DOC_POS_LOCAL ? "local" + : pos == CBM_SEM_DOC_POS_OUTSIDE ? "outside" + : "global"; +} + +/* Per section: its doc's home, whether it is its doc's first section (the + * one a folder candidate starts from), and the Folder nodes by path. */ +typedef struct { + char **home; + bool *first; + CBMHashTable *folders; /* folder path (graph-owned) -> Folder node */ +} doc_place_t; + +static void doc_place_free(doc_place_t *pl, int nsec) { + for (int s = 0; pl->home && s < nsec; s++) { + cbm_free(CBM_MEM_CLASS_SEMANTIC, pl->home[s]); + } + cbm_free(CBM_MEM_CLASS_SEMANTIC, pl->home); + cbm_free(CBM_MEM_CLASS_SEMANTIC, pl->first); + if (pl->folders) { + cbm_ht_free(pl->folders); + } + memset(pl, 0, sizeof(*pl)); +} + +static bool doc_place_build(const cbm_gbuf_t *gbuf, const cbm_gbuf_node_t **secs, int nsec, + const doc_files_t *df, doc_place_t *pl) { + memset(pl, 0, sizeof(*pl)); + pl->home = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)nsec * sizeof(*pl->home)); + pl->first = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)nsec * sizeof(*pl->first)); + pl->folders = cbm_ht_create_in(CBM_MEM_CLASS_SEMANTIC, CBM_SZ_256); + CBMHashTable *firsts = cbm_ht_create_in(CBM_MEM_CLASS_SEMANTIC, CBM_SZ_256); + CBMHashTable *dirs = doc_code_dirs(df); + bool ok = pl->home && pl->first && pl->folders && firsts && dirs; + for (int s = 0; ok && s < nsec; s++) { + pl->home[s] = doc_home(secs[s]->file_path, dirs); + ok = pl->home[s] != NULL; + const char *path = secs[s]->file_path; + if (ok && path) { + int prev = (int)(intptr_t)cbm_ht_get(firsts, path) - SKIP_ONE; + if (prev < 0 || secs[s]->start_line < secs[prev]->start_line) { + cbm_ht_set(firsts, path, (void *)(intptr_t)(s + SKIP_ONE)); + } + } + } + for (int s = 0; ok && s < nsec; s++) { + const char *path = secs[s]->file_path; + pl->first[s] = path && (int)(intptr_t)cbm_ht_get(firsts, path) == s + SKIP_ONE; + } + const cbm_gbuf_node_t **folders = NULL; + int nfold = 0; + if (ok && cbm_gbuf_find_by_label(gbuf, "Folder", &folders, &nfold) == 0) { + for (int i = 0; i < nfold; i++) { + if (folders[i]->file_path && folders[i]->file_path[0]) { + cbm_ht_set(pl->folders, folders[i]->file_path, (void *)folders[i]); + } + } + } + if (firsts) { + cbm_ht_free(firsts); + } + doc_code_dirs_free(dirs); + return ok; +} + +typedef struct { + const cbm_sem_func_t *funcs; + int func_count; + const cbm_sem_corpus_t *corpus; + const cbm_gbuf_node_t **secs; + int nsec; + const int *post_off; + const int *post_f; + int key_df; + const float *fnorm; + const bool *is_test; + const int *name_off; /* per function: its name tokens' corpus indices */ + const int *name_idx; + const char *const *fpath; /* per function: its file */ + const char *const *home; /* per section: its doc's home ("" = the root) */ + const doc_files_t *files; + int file_key_df; + doc_result_t *res; + _Atomic int next; +} doc_ctx_t; + +/* Name-field evidence of one hit: the function's name tokens that the + * section holds (sidx ascending), and the highest idf among them. */ +static void doc_name_evidence(const doc_ctx_t *dc, const int *sidx, int ns, doc_hit_t *hit) { + hit->name_shared = 0; + hit->name_idf = 0.0F; + for (int k = dc->name_off[hit->func]; k < dc->name_off[hit->func + SKIP_ONE]; k++) { + int t = dc->name_idx[k]; + int lo = 0; + int hi = ns - SKIP_ONE; + while (lo <= hi) { + int mid = (lo + hi) / 2; + if (sidx[mid] == t) { + float idf = cbm_sem_corpus_idf_at(dc->corpus, t); + hit->name_shared++; + hit->name_idf = idf > hit->name_idf ? idf : hit->name_idf; + break; + } + if (sidx[mid] < t) { + lo = mid + SKIP_ONE; + } else { + hi = mid - SKIP_ONE; + } + } + } +} + +/* The section's best CBM_SEM_DOC_FILE_K whole files, retrieved by the + * section's key terms over the files' postings, scored by cosine. */ +static void doc_section_files(doc_ctx_t *dc, int s, const int *sidx, const float *sw, int ns, + float snorm, int *fstamp, doc_cand_t *fc) { + const doc_files_t *df = dc->files; + doc_result_t *r = &dc->res[s]; + int nc = 0; + for (int k = 0; k < ns && snorm > 0.0F; k++) { + int t = sidx[k]; + if (df->post_off[t + SKIP_ONE] - df->post_off[t] > dc->file_key_df) { + continue; + } + for (int p = df->post_off[t]; p < df->post_off[t + SKIP_ONE]; p++) { + int f = df->post_f[p]; + if (fstamp[f] != s) { + fstamp[f] = s; + fc[nc++] = (doc_cand_t){.func = f}; + } + } + } + for (int c = 0; c < nc; c++) { + int f = fc[c].func; + int len = df->off[f + SKIP_ONE] - df->off[f]; + float dot = doc_terms_dot(sidx, sw, ns, df->idx + df->off[f], df->w + df->off[f], len, + &fc[c].shared); + float denom = snorm * df->norm[f]; + fc[c].score = denom > 0.0F ? dot / denom : 0.0F; + } + if (nc > 1) { + qsort(fc, (size_t)nc, sizeof(fc[0]), cmp_doc_cand); + } + for (int c = 0; c < nc && c < CBM_SEM_DOC_FILE_K; c++) { + r->files[r->nfiles++] = (doc_file_hit_t){ + .file = fc[c].func, .tfidf = fc[c].score, .rank = c + SKIP_ONE, .shared = fc[c].shared}; + } +} + +/* Up to CBM_SEM_DOC_LOCAL_K functions inside the section's home that rank + * below the top CBM_SEM_DOC_TOP_K, at CBM_SEM_DOC_MIN_SCORE or more (cands + * sorted best first). */ +static void doc_section_local(doc_ctx_t *dc, int s, const doc_cand_t *cands, int nc, + const int *sidx, int ns) { + doc_result_t *r = &dc->res[s]; + const char *home = dc->home[s]; + for (int c = CBM_SEM_DOC_TOP_K; home && home[0] && c < nc && r->nlocal < CBM_SEM_DOC_LOCAL_K; + c++) { + if (cands[c].score < CBM_SEM_DOC_MIN_SCORE) { + break; + } + if (doc_position(home, dc->fpath[cands[c].func]) != CBM_SEM_DOC_POS_LOCAL) { + continue; + } + doc_hit_t *hit = &r->local[r->nlocal++]; + *hit = (doc_hit_t){.func = cands[c].func, + .tfidf = cands[c].score, + .rank = c + SKIP_ONE, + .shared = cands[c].shared}; + doc_name_evidence(dc, sidx, ns, hit); + } +} + +static void doc_section_one(doc_ctx_t *dc, int s, int *stamp, doc_cand_t *cands, int *sidx, + float *sw, char *body, int *fstamp, doc_cand_t *fc) { + doc_result_t *r = &dc->res[s]; + r->n = 0; + r->nlocal = 0; + r->nfiles = 0; + if (!doc_section_gated(dc->secs[s], body)) { + return; + } + char *tokens[SEM_DOC_MAX_TOKENS]; + int count = doc_section_tokens(dc->secs[s], body, tokens); + int ns = cbm_sem_tfidf_terms(dc->corpus, tokens, NULL, count, sidx, sw); + for (int t = 0; t < count; t++) { + cbm_free(CBM_MEM_CLASS_SEMANTIC, tokens[t]); + } + float snorm = terms_norm(sw, ns); + int nc = 0; + for (int k = 0; k < ns && snorm > 0.0F; k++) { + int t = sidx[k]; + if (dc->post_off[t + SKIP_ONE] - dc->post_off[t] > dc->key_df) { + continue; + } + for (int p = dc->post_off[t]; p < dc->post_off[t + SKIP_ONE]; p++) { + int f = dc->post_f[p]; + if (stamp[f] != s && !dc->is_test[f]) { + stamp[f] = s; + cands[nc++] = (doc_cand_t){.func = f}; + } + } + } + for (int c = 0; c < nc; c++) { + const cbm_sem_func_t *fn = &dc->funcs[cands[c].func]; + float dot = doc_terms_dot(sidx, sw, ns, fn->tfidf_indices, fn->tfidf_weights, fn->tfidf_len, + &cands[c].shared); + float denom = snorm * dc->fnorm[cands[c].func]; + cands[c].score = denom > 0.0F ? dot / denom : 0.0F; + } + if (nc > 1) { + qsort(cands, (size_t)nc, sizeof(cands[0]), cmp_doc_cand); + } + for (int c = 0; c < nc && c < SEM_DOC_TOP_K; c++) { + doc_hit_t *hit = &r->hits[r->n++]; + *hit = (doc_hit_t){.func = cands[c].func, + .tfidf = cands[c].score, + .rank = c + SKIP_ONE, + .shared = cands[c].shared}; + doc_name_evidence(dc, sidx, ns, hit); + } + doc_section_local(dc, s, cands, nc, sidx, ns); + doc_section_files(dc, s, sidx, sw, ns, snorm, fstamp, fc); +} + +static void doc_section_worker(int worker_id, void *ctx_ptr) { + (void)worker_id; + doc_ctx_t *dc = ctx_ptr; + int nf = dc->files->count; + int *stamp = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)dc->func_count * sizeof(int)); + doc_cand_t *cands = + cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)dc->func_count * sizeof(doc_cand_t)); + int *fstamp = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)nf + SKIP_ONE) * sizeof(int)); + doc_cand_t *fc = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)nf + SKIP_ONE) * sizeof(*fc)); + int *sidx = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)SEM_DOC_MAX_TOKENS * sizeof(int)); + float *sw = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)SEM_DOC_MAX_TOKENS * sizeof(float)); + char *body = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, SEM_DOC_TEXT_MAX); + if (stamp && cands && fstamp && fc && sidx && sw && body) { + for (int f = 0; f < dc->func_count; f++) { + stamp[f] = -1; + } + for (int f = 0; f < nf; f++) { + fstamp[f] = -1; + } + while (true) { + int s = atomic_fetch_add_explicit(&dc->next, SKIP_ONE, memory_order_relaxed); + if (s >= dc->nsec) { + break; + } + doc_section_one(dc, s, stamp, cands, sidx, sw, body, fstamp, fc); + } + } + cbm_free(CBM_MEM_CLASS_SEMANTIC, stamp); + cbm_free(CBM_MEM_CLASS_SEMANTIC, cands); + cbm_free(CBM_MEM_CLASS_SEMANTIC, fstamp); + cbm_free(CBM_MEM_CLASS_SEMANTIC, fc); + cbm_free(CBM_MEM_CLASS_SEMANTIC, sidx); + cbm_free(CBM_MEM_CLASS_SEMANTIC, sw); + cbm_free(CBM_MEM_CLASS_SEMANTIC, body); +} + +static const char *const DOC_KIND_NAME[CBM_SEM_DOC_KIND_COUNT] = {"function", "local", "file", + "folder"}; + +/* The doc's home Folder node, for the doc's first section (NULL: another + * section, or the home is the repository root). */ +static const cbm_gbuf_node_t *doc_home_folder(const doc_place_t *pl, int s) { + return pl->first[s] && pl->home[s][0] ? cbm_ht_get(pl->folders, pl->home[s]) : NULL; +} + +static cbm_sem_doc_pos_t doc_target_position(const char *home, const cbm_gbuf_node_t *target, + cbm_sem_doc_kind_t kind) { + return kind == CBM_SEM_DOC_KIND_FOLDER ? CBM_SEM_DOC_POS_LOCAL + : doc_position(home, target->file_path); +} + +/* One CBM_SEM_DOC_SIGNALS line: section and target qualified names, tfidf, + * rank, shared terms, both locations, the name evidence, then the kind, the + * position and the doc's home ("." = the repository root). */ +static void doc_signal_line(FILE *out, const cbm_gbuf_node_t *sec, const cbm_gbuf_node_t *t, + const doc_hit_t *hit, cbm_sem_doc_kind_t kind, const char *home) { + (void)fprintf(out, "%s\t%s\t%.6f\t%d\t%d\t%s:%d-%d\t%s:%d-%d\t%d\t%.6f\t%s\t%s\t%s\n", + sec->qualified_name ? sec->qualified_name : "?", + t && t->qualified_name ? t->qualified_name : "?", (double)hit->tfidf, hit->rank, + hit->shared, sec->file_path ? sec->file_path : "?", sec->start_line, + sec->end_line, t && t->file_path ? t->file_path : "?", t ? t->start_line : 0, + t ? t->end_line : 0, hit->name_shared, (double)hit->name_idf, DOC_KIND_NAME[kind], + t ? doc_position_name(doc_target_position(home, t, kind)) : "?", + home[0] ? home : "."); +} + +static void doc_section_lines(FILE *out, const cbm_gbuf_t *gbuf, const doc_ctx_t *dc, + const doc_place_t *pl, int s) { + const cbm_gbuf_node_t *sec = dc->secs[s]; + const doc_result_t *r = &dc->res[s]; + for (int h = 0; h < r->n + r->nlocal; h++) { + const doc_hit_t *hit = h < r->n ? &r->hits[h] : &r->local[h - r->n]; + const cbm_gbuf_node_t *fn = cbm_gbuf_find_by_id(gbuf, dc->funcs[hit->func].node_id); + doc_signal_line(out, sec, fn, hit, + h < r->n ? CBM_SEM_DOC_KIND_FUNCTION : CBM_SEM_DOC_KIND_LOCAL, pl->home[s]); + } + for (int h = 0; h < r->nfiles; h++) { + const doc_file_hit_t *fh = &r->files[h]; + doc_hit_t hit = {.tfidf = fh->tfidf, .rank = fh->rank, .shared = fh->shared}; + doc_signal_line(out, sec, dc->files->node[fh->file], &hit, CBM_SEM_DOC_KIND_FILE, + pl->home[s]); + } + const cbm_gbuf_node_t *folder = doc_home_folder(pl, s); + if (folder) { + doc_hit_t hit = {.rank = SKIP_ONE}; + doc_signal_line(out, sec, folder, &hit, CBM_SEM_DOC_KIND_FOLDER, pl->home[s]); + } +} + +/* Does the section already link the target exactly (a MENTIONS edge)? A + * candidate for that pair would only repeat a fact the graph holds. */ +static bool doc_pair_linked(const cbm_gbuf_t *gbuf, int64_t sec_id, int64_t fn_id) { + const cbm_gbuf_edge_t **edges = NULL; + int n = 0; + if (cbm_gbuf_find_edges_by_source_type(gbuf, sec_id, "MENTIONS", &edges, &n) != 0) { + return false; + } + for (int i = 0; i < n; i++) { + if (edges[i]->target_id == fn_id) { + return true; + } + } + return false; +} + +typedef struct { + cbm_doc_candidate_t *rows; + int n; +} doc_rows_t; + +/* Append the candidate (section -> target) when its judged p is above 0 and + * the graph does not link the pair exactly; false: out of memory (the row at + * rw->n may hold partial strings). */ +static bool doc_row_add(doc_rows_t *rw, const cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *sec, + const cbm_gbuf_node_t *target, const doc_hit_t *hit, + cbm_sem_doc_kind_t kind, const char *home) { + if (!target || !target->qualified_name || !sec->qualified_name) { + return true; + } + cbm_sem_doc_pos_t pos = doc_target_position(home, target, kind); + cbm_sem_doc_format_t fmt = cbm_sem_doc_format(sec->file_path); + float p = kind == CBM_SEM_DOC_KIND_FOLDER + ? cbm_sem_doc_folder_p(fmt, sec->file_path) + : cbm_sem_doc_calibrated_p(fmt, kind, pos, hit->tfidf); + if (p <= 0.0F || doc_pair_linked(gbuf, sec->id, target->id)) { + return true; /* not stored for this band, kind or format; or linked exactly */ + } + char ev[CBM_SZ_128]; + snprintf(ev, sizeof(ev), + "{\"shared_terms\":%d,\"name_tokens\":%d,\"kind\":\"%s\",\"position\":\"%s\"}", + hit->shared, hit->name_shared, DOC_KIND_NAME[kind], doc_position_name(pos)); + rw->rows[rw->n] = (cbm_doc_candidate_t){ + .section_qn = cbm_mem_strdup(CBM_MEM_CLASS_STORE, sec->qualified_name), + .target_qn = cbm_mem_strdup(CBM_MEM_CLASS_STORE, target->qualified_name), + .rank = hit->rank, + .score = hit->tfidf, + .p = cbm_sem_p_2dp(p), + .evidence = cbm_mem_strdup(CBM_MEM_CLASS_STORE, ev)}; + const cbm_doc_candidate_t *row = &rw->rows[rw->n]; + if (!row->section_qn || !row->target_qn || !row->evidence) { + return false; + } + rw->n++; + return true; +} + +static bool doc_section_rows(doc_rows_t *rw, const cbm_gbuf_t *gbuf, const doc_ctx_t *dc, + const doc_place_t *pl, int s) { + const cbm_gbuf_node_t *sec = dc->secs[s]; + const doc_result_t *r = &dc->res[s]; + bool ok = true; + for (int h = 0; ok && h < r->n + r->nlocal; h++) { + const doc_hit_t *hit = h < r->n ? &r->hits[h] : &r->local[h - r->n]; + if (h < r->n && (hit->rank > CBM_SEM_DOC_TOP_K || hit->tfidf < CBM_SEM_DOC_MIN_SCORE)) { + continue; + } + ok = + doc_row_add(rw, gbuf, sec, cbm_gbuf_find_by_id(gbuf, dc->funcs[hit->func].node_id), hit, + h < r->n ? CBM_SEM_DOC_KIND_FUNCTION : CBM_SEM_DOC_KIND_LOCAL, pl->home[s]); + } + for (int h = 0; ok && h < r->nfiles; h++) { + const doc_file_hit_t *fh = &r->files[h]; + doc_hit_t hit = {.tfidf = fh->tfidf, .rank = fh->rank, .shared = fh->shared}; + ok = doc_row_add(rw, gbuf, sec, dc->files->node[fh->file], &hit, CBM_SEM_DOC_KIND_FILE, + pl->home[s]); + } + doc_hit_t first = {.rank = SKIP_ONE}; + return ok && doc_row_add(rw, gbuf, sec, doc_home_folder(pl, s), &first, CBM_SEM_DOC_KIND_FOLDER, + pl->home[s]); +} + +/* The stored candidates of every section -- its best CBM_SEM_DOC_TOP_K + * functions at CBM_SEM_DOC_MIN_SCORE or more, its local functions below them, + * its best files, and (for a doc's first section) the doc's home folder -- + * each with its judged p, in CBM_MEM_CLASS_STORE for + * cbm_store_doc_candidates_free; p at two decimals, as on + * SEMANTICALLY_RELATED edges. */ +static cbm_doc_candidate_t *doc_candidate_rows(const cbm_gbuf_t *gbuf, const doc_ctx_t *dc, + const doc_place_t *pl, int *out_count) { + int n = 0; + for (int s = 0; s < dc->nsec; s++) { + const doc_result_t *r = &dc->res[s]; + n += (r->n < CBM_SEM_DOC_TOP_K ? r->n : CBM_SEM_DOC_TOP_K) + r->nlocal + r->nfiles + + SKIP_ONE; + } + *out_count = 0; + doc_rows_t rw = {.rows = n > 0 ? cbm_calloc(CBM_MEM_CLASS_STORE, (size_t)n * sizeof(*rw.rows)) + : NULL}; + if (!rw.rows) { + return NULL; + } + for (int s = 0; s < dc->nsec; s++) { + if (!doc_section_rows(&rw, gbuf, dc, pl, s)) { + cbm_store_doc_candidates_free(rw.rows, rw.n + SKIP_ONE); + return NULL; + } + } + *out_count = rw.n; + return rw.rows; +} + +static void phase6c_doc_sections(cbm_pipeline_ctx_t *ctx, const cbm_sem_func_t *funcs, + int func_count, const cbm_sem_corpus_t *corpus, int worker_count) { + const cbm_gbuf_t *gbuf = ctx->gbuf; + if (!corpus || func_count <= 0) { + return; + } + const cbm_gbuf_node_t **found = NULL; + int nsec = 0; + if (cbm_gbuf_find_by_label(gbuf, "Section", &found, &nsec) != 0 || nsec == 0) { + return; + } + int nterms = cbm_sem_corpus_token_count(corpus); + const cbm_gbuf_node_t **secs = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)nsec * sizeof(*secs)); + int *post_off = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)nterms + SKIP_ONE) * sizeof(int)); + float *fnorm = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)func_count * sizeof(float)); + bool *is_test = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)func_count * sizeof(bool)); + int *name_off = + cbm_calloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)func_count + SKIP_ONE) * sizeof(int)); + int *name_idx = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, + ((size_t)func_count * SEM_DOC_NAME_TOKENS + SKIP_ONE) * sizeof(int)); + doc_result_t *res = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)nsec * sizeof(doc_result_t)); + const char **fn_paths = + cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)func_count * sizeof(*fn_paths)); + int *ffile = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)func_count * sizeof(int)); + doc_files_t files = {0}; + doc_place_t place = {0}; + int *post_f = NULL; + int *cursor = NULL; + FILE *out = NULL; + if (secs && post_off && fnorm && is_test && name_off && name_idx && res && fn_paths && ffile) { + for (int f = 0; f < func_count; f++) { + for (int k = 0; k < funcs[f].tfidf_len; k++) { + post_off[funcs[f].tfidf_indices[k] + SKIP_ONE]++; + } + fnorm[f] = terms_norm(funcs[f].tfidf_weights, funcs[f].tfidf_len); + const cbm_gbuf_node_t *fn = cbm_gbuf_find_by_id(gbuf, funcs[f].node_id); + is_test[f] = doc_is_test_function(fn); + fn_paths[f] = fn && fn->file_path ? fn->file_path : ""; + char *tok[SEM_DOC_NAME_TOKENS]; + int nt = fn && fn->name ? cbm_sem_tokenize(fn->name, tok, SEM_DOC_NAME_TOKENS) : 0; + int base = name_off[f]; + int m = 0; + for (int t = 0; t < nt; t++) { + int idx = cbm_sem_corpus_token_index(corpus, tok[t]); + bool dup = false; + for (int u = 0; u < m; u++) { + dup = dup || name_idx[base + u] == idx; + } + if (idx >= 0 && !dup) { + name_idx[base + m++] = idx; + } + cbm_free(CBM_MEM_CLASS_SEMANTIC, tok[t]); + } + name_off[f + SKIP_ONE] = base + m; + } + for (int t = 0; t < nterms; t++) { + post_off[t + SKIP_ONE] += post_off[t]; + } + post_f = + cbm_alloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)post_off[nterms] + SKIP_ONE) * sizeof(int)); + cursor = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)nterms + SKIP_ONE) * sizeof(int)); + } + if (post_f && cursor) { + for (int f = 0; f < func_count; f++) { + for (int k = 0; k < funcs[f].tfidf_len; k++) { + int t = funcs[f].tfidf_indices[k]; + post_f[post_off[t] + cursor[t]++] = f; + } + } + memcpy(secs, found, (size_t)nsec * sizeof(*secs)); + qsort(secs, (size_t)nsec, sizeof(*secs), cmp_section_node); + } + bool placed = + post_f && cursor && + doc_files_build(gbuf, funcs, func_count, is_test, corpus, nterms, &files, ffile) && + doc_place_build(gbuf, secs, nsec, &files, &place); + if (placed) { + int key_df = func_count / SEM_DOC_KEY_DF_DIV; + if (key_df < SEM_DOC_KEY_DF_MIN) { + key_df = SEM_DOC_KEY_DF_MIN; + } + int file_key_df = files.count / SEM_DOC_KEY_DF_DIV; + if (file_key_df < SEM_DOC_KEY_DF_MIN) { + file_key_df = SEM_DOC_KEY_DF_MIN; + } + doc_ctx_t dc = {.funcs = funcs, + .func_count = func_count, + .corpus = corpus, + .secs = secs, + .nsec = nsec, + .post_off = post_off, + .post_f = post_f, + .key_df = key_df, + .fnorm = fnorm, + .is_test = is_test, + .name_off = name_off, + .name_idx = name_idx, + .fpath = fn_paths, + .home = (const char *const *)place.home, + .files = &files, + .file_key_df = file_key_df, + .res = res}; + atomic_init(&dc.next, 0); + cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false}; + cbm_parallel_for(worker_count, doc_section_worker, &dc, opts); + int row_count = 0; + cbm_doc_candidate_t *rows = doc_candidate_rows(gbuf, &dc, &place, &row_count); + cbm_pipeline_set_doc_candidates(ctx->pipeline, rows, row_count); + cbm_log_info("pass.semantic.doc_candidates", "sections", itoa_log(nsec), "files", + itoa_log(files.count), "rows", itoa_log(row_count)); + char path[CBM_SZ_1K]; + out = cbm_safe_getenv("CBM_SEM_DOC_SIGNALS", path, sizeof(path), NULL) + ? cbm_fopen(path, "w") + : NULL; + for (int s = 0; out && s < nsec; s++) { + doc_section_lines(out, gbuf, &dc, &place, s); + } + /* CBM_SEM_DOC_FUNCS=: every function with its location and the + * test flag, for measuring candidates the TF-IDF generator does not + * produce (judged samples only). */ + char fpath[CBM_SZ_1K]; + FILE *fout = cbm_safe_getenv("CBM_SEM_DOC_FUNCS", fpath, sizeof(fpath), NULL) + ? cbm_fopen(fpath, "w") + : NULL; + for (int f = 0; fout && f < func_count; f++) { + const cbm_gbuf_node_t *fn = cbm_gbuf_find_by_id(gbuf, funcs[f].node_id); + (void)fprintf(fout, "%s\t%s:%d-%d\t%d\n", + fn && fn->qualified_name ? fn->qualified_name : "?", + fn && fn->file_path ? fn->file_path : "?", fn ? fn->start_line : 0, + fn ? fn->end_line : 0, is_test[f] ? 1 : 0); + } + if (fout) { + (void)fclose(fout); + } + } + if (out) { + (void)fclose(out); + } + cbm_free(CBM_MEM_CLASS_SEMANTIC, cursor); + cbm_free(CBM_MEM_CLASS_SEMANTIC, post_f); + cbm_free(CBM_MEM_CLASS_SEMANTIC, secs); + cbm_free(CBM_MEM_CLASS_SEMANTIC, post_off); + cbm_free(CBM_MEM_CLASS_SEMANTIC, fnorm); + cbm_free(CBM_MEM_CLASS_SEMANTIC, is_test); + cbm_free(CBM_MEM_CLASS_SEMANTIC, name_off); + cbm_free(CBM_MEM_CLASS_SEMANTIC, name_idx); + cbm_free(CBM_MEM_CLASS_SEMANTIC, res); + cbm_free(CBM_MEM_CLASS_SEMANTIC, fn_paths); + cbm_free(CBM_MEM_CLASS_SEMANTIC, ffile); + doc_place_free(&place, nsec); + doc_files_free(&files); +} + int cbm_pipeline_pass_semantic_edges(cbm_pipeline_ctx_t *ctx) { /* Controlled by pipeline mode (moderate/full), not env var */ cbm_log_info("pass.start", "pass", "semantic_edges"); @@ -1900,6 +3187,9 @@ int cbm_pipeline_pass_semantic_edges(cbm_pipeline_ctx_t *ctx) { sem_mem_mark("6_score"); + /* Phase 6c: doc sections -> function candidates (doc_link_candidates). */ + phase6c_doc_sections(ctx, funcs, func_count, corpus, worker_count); + /* Phase 7: Cleanup */ CBM_PROF_START(t_phase7); free_lsh_buckets(band_buckets); diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 179a5c981..2d66a6c7e 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -273,6 +273,11 @@ struct cbm_pipeline { int doc_link_row_count; bool doc_links_failed; bool doc_links_ran; + /* Doc -> code candidates from pass_semantic_edges, published with the + * generation (heap rows, cbm_store_doc_candidates_free in + * cbm_pipeline_free). NULL when the pass found none or did not run. */ + cbm_doc_candidate_t *doc_candidates; + int doc_candidate_count; /* Deterministic test-only seam at the final publication boundary. Kept * per pipeline so concurrent test/process activity cannot cross-trigger. */ @@ -915,6 +920,23 @@ void cbm_pipeline_set_lsp_surfaces(cbm_pipeline_t *p, cbm_lsp_surface_row_t *row p->surface_row_count = count; } +void cbm_pipeline_set_doc_candidates(cbm_pipeline_t *p, cbm_doc_candidate_t *rows, int count) { + if (!p) { + cbm_store_doc_candidates_free(rows, count); + return; + } + cbm_store_doc_candidates_free(p->doc_candidates, p->doc_candidate_count); + p->doc_candidates = rows; + p->doc_candidate_count = rows ? count : 0; +} + +const cbm_doc_candidate_t *cbm_pipeline_doc_candidates(const cbm_pipeline_t *p, int *count) { + if (count) { + *count = p ? p->doc_candidate_count : 0; + } + return p ? p->doc_candidates : NULL; +} + static void pipeline_release_test_config(cbm_pipeline_t *p) { if (!p) { return; @@ -995,6 +1017,9 @@ void cbm_pipeline_free(cbm_pipeline_t *p) { p->surface_rows = NULL; p->surface_row_count = 0; pipeline_reset_doc_links(p); + cbm_store_doc_candidates_free(p->doc_candidates, p->doc_candidate_count); + p->doc_candidates = NULL; + p->doc_candidate_count = 0; cbm_git_context_free(&p->git_ctx); /* gbuf, store, registry freed during/after run */ /* Defensive owner cleanup; normal runs release after publication. */ @@ -3075,6 +3100,14 @@ static int publish_staged_impl(char *stage_path, const cbm_pipeline_generation_t ok = cbm_store_adr_store(store, generation->project, generation->adr_content) == CBM_STORE_OK; } + /* Doc -> code candidates belong to the generation too: replaced inside + * the staging store (the table is created here), so a reader never sees + * candidates of another graph. */ + if (ok && !generation->doc_candidates_in_place) { + ok = + cbm_store_doc_candidates_replace(store, generation->project, generation->doc_candidates, + generation->doc_candidate_count) == CBM_STORE_OK; + } if (ok) { ok = cbm_store_set_format_version(store, CBM_INDEX_FORMAT_VERSION) == CBM_STORE_OK; @@ -3366,6 +3399,8 @@ static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_hash_t *bas .doc_link_row_count = p->doc_link_row_count, /* every full route runs a doc-link phase; none having run is a fault */ .doc_links_failed = p->doc_links_failed || !p->doc_links_ran, + .doc_candidates = p->doc_candidates, + .doc_candidate_count = p->doc_candidate_count, }; free(db_dir); diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index bc3beb7e6..330d36d89 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -2082,7 +2082,8 @@ static int dump_and_persist(cbm_gbuf_t *gbuf, const char *db_path, const char *p const cbm_coverage_row_t *cov, int cov_count, const cbm_coverage_meta_t *meta_template, const cbm_lsp_surface_row_t *surface_rows, int surface_row_count, - legacy_doc_rows_t doc) { + legacy_doc_rows_t doc, const cbm_doc_candidate_t *doc_candidates, + int doc_candidate_count) { struct timespec t; cbm_clock_gettime(CLOCK_MONOTONIC, &t); cbm_pipeline_generation_t generation = { @@ -2101,6 +2102,8 @@ static int dump_and_persist(cbm_gbuf_t *gbuf, const char *db_path, const char *p .doc_link_rows = doc.rows, .doc_link_row_count = doc.count, .doc_links_failed = doc.failed, + .doc_candidates = doc_candidates, + .doc_candidate_count = doc_candidate_count, }; int rc = cbm_pipeline_publish_generation(&generation); cbm_log_info("incremental.dump", "rc", itoa_buf(rc), "elapsed_ms", @@ -3169,6 +3172,7 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char .doc_link_rows = doc_rows, .doc_link_row_count = doc_row_count, .doc_links_failed = doc_failed, + .doc_candidates_in_place = true, /* the clone keeps the previous rows */ }; cbm_store_close(staging); staging = NULL; @@ -3796,9 +3800,13 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil * would satisfy a future closure plan with yesterday's surface; an * empty table just routes the next incremental to a full rebuild. */ legacy_doc_rows_t doc_out = {.rows = doc_rows, .count = doc_row_count, .failed = doc_failed}; - int persist_rc = - dump_and_persist(existing, db_path, project, cbm_pipeline_cancelled_ptr(p), manifest, - manifest_count, saved_adr, cov, cov_n, &coverage_meta, NULL, 0, doc_out); + /* Doc candidates: the post-passes ran the semantic pass over the whole + * rehydrated project, so this run's rows are complete. */ + int doc_cand_n = 0; + const cbm_doc_candidate_t *doc_cands = cbm_pipeline_doc_candidates(p, &doc_cand_n); + int persist_rc = dump_and_persist(existing, db_path, project, cbm_pipeline_cancelled_ptr(p), + manifest, manifest_count, saved_adr, cov, cov_n, + &coverage_meta, NULL, 0, doc_out, doc_cands, doc_cand_n); cbm_doclinks_free_rows(doc_rows, doc_row_count); cbm_pipeline_free_semantic_manifest(manifest, manifest_count); free(saved_adr); diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 87b9c160b..7ecddbb4e 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -1274,8 +1274,13 @@ int cbm_pipeline_build_fresh_semantic_manifest(cbm_pipeline_t *p, const char *pr * sections and PDF pages hold MENTIONS edges into the code, ADR records * are nodes, and AsciiDoc files, PDFs and a Sphinx project's extra * source suffixes are indexed. An index built before has none of this - * for its unchanged documents, so it rebuilds once on upgrade. */ -enum { CBM_SEMANTIC_INDEX_VERSION = 6 }; + * for its unchanged documents, so it rebuilds once on upgrade. + * 7: SEMANTICALLY_RELATED changed: TF-IDF compares vocabulary, string + * properties are read with their escapes, admission is best-first and + * every edge carries "p"; doc_link_candidates holds doc section -> code + * candidates. An index built before has the old pairs and no + * candidates, so it rebuilds once on upgrade. */ +enum { CBM_SEMANTIC_INDEX_VERSION = 7 }; typedef struct { cbm_gbuf_t *gbuf; @@ -1305,6 +1310,16 @@ typedef struct { const cbm_doc_link_row_t *doc_link_rows; int doc_link_row_count; bool doc_links_failed; + /* Doc -> code candidates of this generation (pass_semantic_edges; may be + * NULL: none found, or a mode without the semantic pass), written into + * the staging store with the graph. */ + const cbm_doc_candidate_t *doc_candidates; + int doc_candidate_count; + /* True when the staging store already holds this generation's + * candidates: the delta clone keeps the previous generation's rows (its + * proxy buffer has no section text or function tokens to recompute + * them); a row whose node is gone resolves to nothing when read. */ + bool doc_candidates_in_place; } cbm_pipeline_generation_t; /* Serialize and fully populate a sibling staging database, then atomically @@ -1377,6 +1392,12 @@ void cbm_pipeline_set_doc_link_rows(cbm_pipeline_t *p, cbm_doc_link_row_t *rows, void cbm_pipeline_take_doc_link_rows(cbm_pipeline_t *p, cbm_doc_link_row_t **rows, int *count, bool *failed, bool *ran); +/* Hand the run's doc -> code candidates (pass_semantic_edges) to the + * pipeline. Takes ownership (rows and strings in CBM_MEM_CLASS_STORE, released + * with cbm_store_doc_candidates_free); a NULL p frees them. NULL/0 clears. */ +void cbm_pipeline_set_doc_candidates(cbm_pipeline_t *p, cbm_doc_candidate_t *rows, int count); +const cbm_doc_candidate_t *cbm_pipeline_doc_candidates(const cbm_pipeline_t *p, int *count); + /* Pipeline accessors for incremental use */ const char *cbm_pipeline_repo_path(const cbm_pipeline_t *p); const cbm_test_declarations_t *cbm_pipeline_test_declarations(const cbm_pipeline_t *p); diff --git a/src/semantic/semantic.c b/src/semantic/semantic.c index 2aa1df5a3..af738151e 100644 --- a/src/semantic/semantic.c +++ b/src/semantic/semantic.c @@ -1619,6 +1619,47 @@ static float small_cosine(const float *a, const float *b, int dims) { return denom < CBM_SEM_DENOM_EPS ? 0.0F : dot / denom; } +static int cmp_int_asc(const void *pa, const void *pb) { + int a = *(const int *)pa; + int b = *(const int *)pb; + return (a > b) - (a < b); +} + +/* Terms are keyed by corpus token, so two functions share a term only when + * they share the word. (Until 2026-10 the key was the token's POSITION in the + * function: any two functions of similar length "shared" their first terms, + * and the TF-IDF signal measured length, not vocabulary.) */ +int cbm_sem_tfidf_terms(const cbm_sem_corpus_t *corpus, char *const *tokens, const int *token_index, + int count, int *indices, float *weights) { + if (!corpus || !indices || !weights || count <= 0) { + return 0; + } + int known = 0; + for (int t = 0; t < count; t++) { + int idx = token_index ? token_index[t] : cbm_sem_corpus_token_index(corpus, tokens[t]); + if (idx >= 0 && cbm_sem_corpus_idf_at(corpus, idx) > 0.0F) { + indices[known++] = idx; + } + } + if (known > 1) { + qsort(indices, (size_t)known, sizeof(indices[0]), cmp_int_asc); + } + /* Each run of one index becomes one term; its weight is the idf added once + * per occurrence (equal indices have equal idf). */ + int terms = 0; + for (int k = 0; k < known; k++) { + float idf = cbm_sem_corpus_idf_at(corpus, indices[k]); + if (terms > 0 && indices[terms - SKIP_ONE] == indices[k]) { + weights[terms - SKIP_ONE] += idf; + } else { + indices[terms] = indices[k]; + weights[terms] = idf; + terms++; + } + } + return terms; +} + /* Sparse cosine over two pre-sorted (index, weight) vectors. Returns 0 when * either side is empty or the magnitude product is below the epsilon guard. */ static float sparse_tfidf_cosine(const cbm_sem_func_t *a, const cbm_sem_func_t *b) { @@ -1651,49 +1692,47 @@ static float sparse_tfidf_cosine(const cbm_sem_func_t *a, const cbm_sem_func_t * return denom > CBM_SEM_DENOM_EPS ? (dot / denom) : 0.0F; } -float cbm_sem_combined_score(const cbm_sem_func_t *a, const cbm_sem_func_t *b, - const cbm_sem_config_t *cfg) { - if (!a || !b || !cfg) { - return 0.0F; +void cbm_sem_signal_values(const cbm_sem_func_t *a, const cbm_sem_func_t *b, + cbm_sem_signals_t *out) { + memset(out, 0, sizeof(*out)); + out->proximity = CBM_SEM_UNIT_POS; + if (!a || !b) { + return; } + if (a->has_minhash && b->has_minhash) { + double j = cbm_minhash_jaccard((const cbm_minhash_t *)a->minhash, + (const cbm_minhash_t *)b->minhash); + out->minhash = (float)j; + out->near_copy = j >= CBM_MINHASH_JACCARD_THRESHOLD; + } + out->tfidf = sparse_tfidf_cosine(a, b); + out->ri = cbm_rsq_ip(&a->ri_code, &b->ri_code); /* signal 2: Random Indexing */ + out->api = cbm_rsq_ip(&a->api_code, &b->api_code); /* signal 4: API signatures */ + out->type = cbm_rsq_ip(&a->type_code, &b->type_code); /* signal 5: type signatures */ + out->decorator = cbm_rsq_ip(&a->deco_code, &b->deco_code); /* signal 7: decorators */ + /* signals 8+9+11: structural profile + data flow + Halstead */ + out->struct_profile = + small_cosine(a->struct_profile, b->struct_profile, CBM_SEM_AST_PROFILE_DIMS); + out->proximity = cbm_sem_proximity(a->file_path, b->file_path); /* signal 6: multiplier */ +} +float cbm_sem_combine(const cbm_sem_signals_t *s, const cbm_sem_config_t *cfg) { /* Short-circuit: if MinHash Jaccard is already above the SIMILAR_TO threshold, * the pass_similarity pipeline already emitted a SIMILAR_TO edge for this pair. * Returning 0 here avoids flooding top-k with cross-service copy-paste boilerplate * (logging_middleware, shared push/pull handlers) that SIMILAR_TO already covers, * freeing the edge budget for true semantic leaps (vocabulary-bridged relations). */ - if (a->has_minhash && b->has_minhash) { - double early_j = cbm_minhash_jaccard((const cbm_minhash_t *)a->minhash, - (const cbm_minhash_t *)b->minhash); - if (early_j >= CBM_MINHASH_JACCARD_THRESHOLD) { - return 0.0F; - } - } - - float score = cfg->w_tfidf * sparse_tfidf_cosine(a, b); - - /* Signal 2: Random Indexing */ - score += cfg->w_ri * cbm_rsq_ip(&a->ri_code, &b->ri_code); - - /* Signal 3: MinHash Jaccard */ - if (a->has_minhash && b->has_minhash) { - double j = cbm_minhash_jaccard((const cbm_minhash_t *)a->minhash, - (const cbm_minhash_t *)b->minhash); - score += cfg->w_minhash * (float)j; + if (!s || !cfg || s->near_copy) { + return 0.0F; } - - /* Signal 4: API Signatures */ - score += cfg->w_api * cbm_rsq_ip(&a->api_code, &b->api_code); - - /* Signal 5: Type Signatures */ - score += cfg->w_type * cbm_rsq_ip(&a->type_code, &b->type_code); - - /* Signal 7: Decorator Pattern */ - score += cfg->w_decorator * cbm_rsq_ip(&a->deco_code, &b->deco_code); - - /* Signal 8+9+11: Structural profile + data flow + Halstead */ - float sp_score = small_cosine(a->struct_profile, b->struct_profile, CBM_SEM_AST_PROFILE_DIMS); - score += cfg->w_struct_profile * sp_score; + /* The terms in the historical order (float addition is not associative). */ + float score = cfg->w_tfidf * s->tfidf; + score += cfg->w_ri * s->ri; + score += cfg->w_minhash * s->minhash; + score += cfg->w_api * s->api; + score += cfg->w_type * s->type; + score += cfg->w_decorator * s->decorator; + score += cfg->w_struct_profile * s->struct_profile; /* Signal 6: Module proximity (multiplier, not additive). * Proximity returns [1.0, 1.10] — a same-file/same-dir boost for ranking. @@ -1701,17 +1740,249 @@ float cbm_sem_combined_score(const cbm_sem_func_t *a, const cbm_sem_func_t *b, * cosine-similarity range. Without the clamp, a 0.95 base × 1.10 proximity * would emit a 1.045 score which violates the semantic of "similarity" and * breaks downstream consumers that expect a normalized value. */ - score *= cbm_sem_proximity(a->file_path, b->file_path); + score *= s->proximity; if (score > CBM_SEM_UNIT_POS) { score = CBM_SEM_UNIT_POS; } if (score < 0.0F) { score = 0.0F; } - return score; } +typedef struct { + float score; + int pos; +} sem_rank_t; + +static int cmp_rank_best_first(const void *pa, const void *pb) { + const sem_rank_t *a = pa; + const sem_rank_t *b = pb; + if (a->score != b->score) { + return a->score > b->score ? -1 : 1; + } + return (a->pos > b->pos) - (a->pos < b->pos); +} + +/* Best-first, not first-come: when a function has more candidate partners + * than its budget, it keeps its best ones. Judged on held-out repositories + * (2026-10-07): the pairs first-come admitted over budget were 0 of 17 + * related, the ones best-first admits instead 7 of 20. */ +bool cbm_sem_admit_best_first(const float *scores, const int *fa, const int *fb, + const bool *eligible, int n, int max_edges, int *counts, + bool *admitted) { + if (n <= 0) { + return true; + } + memset(admitted, 0, (size_t)n * sizeof(admitted[0])); + sem_rank_t *order = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)n * sizeof(sem_rank_t)); + if (!order) { + return false; + } + int m = 0; + for (int k = 0; k < n; k++) { + if (eligible[k]) { + order[m++] = (sem_rank_t){.score = scores[k], .pos = k}; + } + } + qsort(order, (size_t)m, sizeof(order[0]), cmp_rank_best_first); + for (int r = 0; r < m; r++) { + int k = order[r].pos; + if (counts[fa[k]] < max_edges && counts[fb[k]] < max_edges) { + counts[fa[k]]++; + counts[fb[k]]++; + admitted[k] = true; + } + } + cbm_free(CBM_MEM_CLASS_SEMANTIC, order); + return true; +} + +/* Until 2026-10 the reader stopped at the first '"' and kept escapes raw: + * a docstring holding a quote lost everything after it, and "\nreturn" gave + * the token "nreturn". */ +const char *cbm_sem_json_str(const char *json, const char *key, char *buf, int bufsize) { + if (!json || !key || !buf || bufsize <= 0) { + return NULL; + } + char search[CBM_SZ_64]; + snprintf(search, sizeof(search), "\"%s\":\"", key); + const char *p = strstr(json, search); + if (!p) { + return NULL; + } + p += strlen(search); + int n = 0; + for (; *p && *p != '"' && n + SKIP_ONE < bufsize; p++) { + char c = *p; + if (c == '\\' && p[SKIP_ONE]) { + p++; + switch (*p) { + case 'n': + case 't': + case 'r': + case 'b': + case 'f': + c = ' '; + break; + case 'u': + c = ' '; + for (int k = 0; k < 4 && isxdigit((unsigned char)p[SKIP_ONE]); k++) { + p++; + } + break; + default: + c = *p; /* \" \\ \/ */ + break; + } + } + buf[n++] = c; + } + buf[n] = '\0'; + return buf; +} + +/* Score bands at the quartiles of the admitted scores (0.773, 0.802, 0.838 at + * the stored score's three decimals, so one shown score always carries one p), + * each band's p the judged share of related pairs, pooled where a higher band + * judged lower: the upper three pool to one. Judged 2026-10-07 in eleven + * held-out repositories: the pairs this scoring admits that the previous one + * did not (100, blind, three reading rounds) and the earlier samples' pairs it + * keeps (65), each weighted by its share of the admitted set. Ships as the + * edge property "p". */ +static const float SEM_P_CUTS[] = {0.773F}; +static const float SEM_P_BANDS[] = {0.626F, 0.760F}; + +float cbm_sem_calibrated_p(float score) { + size_t band = 0; + while (band < sizeof(SEM_P_CUTS) / sizeof(SEM_P_CUTS[0]) && score >= SEM_P_CUTS[band]) { + band++; + } + return SEM_P_BANDS[band]; +} + +/* Doc -> code candidates by their tfidf: each band's p the judged share of + * sections that are about the function (2026-10-07: 274 blind-judged + * candidates -- a section's best five non-test functions -- from five held-out + * repositories, three reading rounds; the top three measured bands pooled to + * stay monotone). Below the first cut (judged 0.06) nothing is stored. */ +enum { SEM_DOC_P_NBANDS = 4 }; +static const float SEM_DOC_P_CUTS[SEM_DOC_P_NBANDS - 1] = {CBM_SEM_DOC_MIN_SCORE, 0.30F, 0.40F}; +/* Per format, p below the first cut and in each band; 0 = not stored (a + * band judged below 0.20). Blind-judged, three reading rounds, pooled where + * a higher band judged lower: + * Markdown by kind and position since SEM7: SEM_DOC_P_MD_* below (its + * first curve, 2026-10-07: 274 candidates of five repositories, + * 0.06 / 0.36 / 0.42 / 0.55, did not separate positions). + * reST 2026-10-08, 120 candidates, three Sphinx projects; 0.20-0.30 + * judged 0.125, so reST starts at 0.30. + * AsciiDoc 2026-10-08, 114 candidates, one repository (the only one of + * its search that qualified). + * PDF no qualifying repository: not measured, nothing stored. */ +static const float SEM_DOC_P_BANDS[CBM_SEM_DOC_FMT_COUNT][SEM_DOC_P_NBANDS] = { + [CBM_SEM_DOC_FMT_OTHER] = {0.0F, 0.0F, 0.0F, 0.0F}, + [CBM_SEM_DOC_FMT_MARKDOWN] = {0.0F, 0.0F, 0.0F, 0.0F}, /* SEM_DOC_P_MD_* */ + [CBM_SEM_DOC_FMT_RST] = {0.0F, 0.0F, 0.35F, 0.625F}, + [CBM_SEM_DOC_FMT_ADOC] = {0.0F, 0.225F, 0.25F, 0.50F}, + [CBM_SEM_DOC_FMT_PDF] = {0.0F, 0.0F, 0.0F, 0.0F}, +}; + +cbm_sem_doc_format_t cbm_sem_doc_format(const char *path) { + static const struct { + const char *ext; + cbm_sem_doc_format_t fmt; + } exts[] = {{".md", CBM_SEM_DOC_FMT_MARKDOWN}, {".mdx", CBM_SEM_DOC_FMT_MARKDOWN}, + {".markdown", CBM_SEM_DOC_FMT_MARKDOWN}, {".rst", CBM_SEM_DOC_FMT_RST}, + {".rest", CBM_SEM_DOC_FMT_RST}, {".txt", CBM_SEM_DOC_FMT_RST}, + {".adoc", CBM_SEM_DOC_FMT_ADOC}, {".asciidoc", CBM_SEM_DOC_FMT_ADOC}, + {".asc", CBM_SEM_DOC_FMT_ADOC}, {".pdf", CBM_SEM_DOC_FMT_PDF}}; + const char *dot = path ? strrchr(path, '.') : NULL; + if (!dot || strchr(dot, '/')) { + return CBM_SEM_DOC_FMT_OTHER; + } + for (size_t i = 0; i < sizeof(exts) / sizeof(exts[0]); i++) { + if (strcasecmp(dot, exts[i].ext) == 0) { + return exts[i].fmt; + } + } + return CBM_SEM_DOC_FMT_OTHER; +} + +double cbm_sem_p_2dp(float p) { + /* 1e-4 on the x100 scale: a float holds 0.325 as 0.32499998..., 1.2e-6 + * below the half; judged values have at most three decimals, so the + * nearest one that must round down (x.xx4) is 0.1 away */ + return (double)(long)((double)p * 100.0 + 0.5 + 1e-4) / 100.0; +} + +/* Markdown by kind and position (SEM7, 2026-10-08: 490 blind-judged + * candidates from four monorepos with per-component docs, three reading + * rounds, pooled where a higher band judged lower). A doc's position decides + * most: functions outside the doc's home were right 6 of 90 (none stored); + * inside it 0.18 / 0.33 / 0.53. A doc of the whole project: SEM7's and + * sample 4's judged items pooled (the user's choice; 158 items, nine + * repositories): 0.17 / 0.22 / 0.51. Local extras (home functions below the + * top 5) 19 of 60 = 0.32 at every band; whole files 0.12 below 0.40, then + * 0.30. Below 0.20 nothing. */ +static const float SEM_DOC_P_MD_FUNCTION[CBM_SEM_DOC_POS_COUNT][SEM_DOC_P_NBANDS] = { + [CBM_SEM_DOC_POS_GLOBAL] = {0.0F, 0.0F, 0.217F, 0.509F}, + [CBM_SEM_DOC_POS_LOCAL] = {0.0F, 0.0F, 0.325F, 0.525F}, + [CBM_SEM_DOC_POS_OUTSIDE] = {0.0F, 0.0F, 0.0F, 0.0F}, +}; +static const float SEM_DOC_P_MD_LOCAL[SEM_DOC_P_NBANDS] = {0.0F, 0.317F, 0.317F, 0.317F}; +static const float SEM_DOC_P_MD_FILE[SEM_DOC_P_NBANDS] = {0.0F, 0.0F, 0.0F, 0.30F}; + +float cbm_sem_doc_calibrated_p(cbm_sem_doc_format_t fmt, cbm_sem_doc_kind_t kind, + cbm_sem_doc_pos_t pos, float score) { + if ((int)fmt < 0 || fmt >= CBM_SEM_DOC_FMT_COUNT || (int)pos < 0 || + pos >= CBM_SEM_DOC_POS_COUNT) { + return 0.0F; + } + size_t band = 0; + while (band < SEM_DOC_P_NBANDS - 1 && score >= SEM_DOC_P_CUTS[band]) { + band++; + } + if (fmt != CBM_SEM_DOC_FMT_MARKDOWN) { + /* measured for function candidates only, without position (SEM6) */ + return kind == CBM_SEM_DOC_KIND_FUNCTION ? SEM_DOC_P_BANDS[fmt][band] : 0.0F; + } + switch (kind) { + case CBM_SEM_DOC_KIND_FUNCTION: + return SEM_DOC_P_MD_FUNCTION[pos][band]; + case CBM_SEM_DOC_KIND_LOCAL: + return pos == CBM_SEM_DOC_POS_LOCAL ? SEM_DOC_P_MD_LOCAL[band] : 0.0F; + case CBM_SEM_DOC_KIND_FILE: + return SEM_DOC_P_MD_FILE[band]; + default: + return 0.0F; /* a folder candidate: cbm_sem_doc_folder_p */ + } +} + +/* A doc's home folder as its candidate (SEM7): a README or index doc was + * right 26 of 30 (0.87); any other doc 5 of 30 (not stored). Markdown only. */ +float cbm_sem_doc_folder_p(cbm_sem_doc_format_t fmt, const char *doc_path) { + if (fmt != CBM_SEM_DOC_FMT_MARKDOWN || !doc_path) { + return 0.0F; + } + const char *base = strrchr(doc_path, '/'); + base = base ? base + 1 : doc_path; + const char *dot = strrchr(base, '.'); + size_t n = dot ? (size_t)(dot - base) : strlen(base); + bool readme = (n == sizeof("readme") - 1 && strncasecmp(base, "readme", n) == 0) || + (n == sizeof("index") - 1 && strncasecmp(base, "index", n) == 0); + return readme ? 0.867F : 0.0F; +} + +float cbm_sem_combined_score(const cbm_sem_func_t *a, const cbm_sem_func_t *b, + const cbm_sem_config_t *cfg) { + if (!a || !b || !cfg) { + return 0.0F; + } + cbm_sem_signals_t s; + cbm_sem_signal_values(a, b, &s); + return cbm_sem_combine(&s, cfg); +} + /* ── Graph diffusion ─────────────────────────────────────────────── */ void cbm_sem_diffuse(cbm_sem_vec_t *combined, const cbm_sem_vec_t *neighbors, int neighbor_count, diff --git a/src/semantic/semantic.h b/src/semantic/semantic.h index 1f99a54c0..66f4cc0a1 100644 --- a/src/semantic/semantic.h +++ b/src/semantic/semantic.h @@ -206,12 +206,123 @@ const char *cbm_sem_corpus_token_at(const cbm_sem_corpus_t *corpus, int index, /* Free corpus. */ void cbm_sem_corpus_free(cbm_sem_corpus_t *corpus); +/* The TF-IDF terms of one function: one entry per distinct corpus token whose + * idf is above 0, weighted term frequency x idf, in ascending token-index + * order (the order the TF-IDF cosine merges on). `token_index[t]` is the + * corpus index of tokens[t] (-1 when unknown); NULL looks each token up by + * name. `indices` and `weights` hold at least `count` entries. Returns the + * number of terms written. */ +int cbm_sem_tfidf_terms(const cbm_sem_corpus_t *corpus, char *const *tokens, const int *token_index, + int count, int *indices, float *weights); + /* ── Combined scoring ────────────────────────────────────────────── */ /* Compute combined similarity score between two functions. */ float cbm_sem_combined_score(const cbm_sem_func_t *a, const cbm_sem_func_t *b, const cbm_sem_config_t *cfg); +/* The signal values cbm_sem_combined_score weighs, for measuring them one by + * one: each similarity in [0, 1] (minhash 0 when either side has none), + * proximity a multiplier in [1.0, 1.10]. `near_copy`: the pair is a SIMILAR_TO + * copy, which the combined score sets to 0. */ +typedef struct { + float tfidf; + float ri; + float minhash; + float api; + float type; + float decorator; + float struct_profile; + float proximity; + bool near_copy; +} cbm_sem_signals_t; + +void cbm_sem_signal_values(const cbm_sem_func_t *a, const cbm_sem_func_t *b, + cbm_sem_signals_t *out); + +/* cbm_sem_combined_score from the signal values (the same arithmetic). */ +float cbm_sem_combine(const cbm_sem_signals_t *s, const cbm_sem_config_t *cfg); + +/* Best-first admission under a per-function edge budget. Pair k joins + * functions fa[k] and fb[k] with scores[k]; pairs are taken by descending + * score, ties in ascending k (the caller's canonical order), and pair k is + * admitted when eligible[k] and both functions have fewer than max_edges + * admitted pairs. counts holds one zeroed slot per function and is updated. + * Returns false (nothing admitted) when the order cannot be allocated. */ +bool cbm_sem_admit_best_first(const float *scores, const int *fa, const int *fb, + const bool *eligible, int n, int max_edges, int *counts, + bool *admitted); + +/* The string value of "key" in a properties JSON object, unescaped into buf + * (cut at bufsize - 1); NULL when the key is absent. Escapes are honoured: + * \" \\ \/ keep their character, \n \t \r \b \f and \uXXXX become a space (a + * separator for the tokenizer), so a value holding a quote is read whole. */ +const char *cbm_sem_json_str(const char *json, const char *key, char *buf, int bufsize); + +/* The probability that an admitted SEMANTICALLY_RELATED pair is related, by + * its score as stored (three decimals): the judged precision of blind-judged + * held-out samples (the edge property "p"). */ +float cbm_sem_calibrated_p(float score); + +/* Doc -> code candidates: a section's best CBM_SEM_DOC_TOP_K non-test + * functions by TF-IDF, stored when their tfidf is at least + * CBM_SEM_DOC_MIN_SCORE, each with the judged probability that the section is + * about the function. */ +#define CBM_SEM_DOC_TOP_K 5 +#define CBM_SEM_DOC_MIN_SCORE 0.20F + +/* A section's document format, by its file's extension: candidates are + * judged, and stored, per format. */ +typedef enum { + CBM_SEM_DOC_FMT_OTHER = 0, + CBM_SEM_DOC_FMT_MARKDOWN, + CBM_SEM_DOC_FMT_RST, /* .rst, and .txt: a .txt file only has sections as Sphinx source */ + CBM_SEM_DOC_FMT_ADOC, + CBM_SEM_DOC_FMT_PDF, + CBM_SEM_DOC_FMT_COUNT, +} cbm_sem_doc_format_t; +cbm_sem_doc_format_t cbm_sem_doc_format(const char *path); + +/* Where a candidate sits relative to the doc's home folder (the folder whose + * code the doc documents, see pass_semantic_edges.c): the home is the + * repository root (the doc documents the whole project), or the candidate is + * inside or outside the home. */ +typedef enum { + CBM_SEM_DOC_POS_GLOBAL = 0, + CBM_SEM_DOC_POS_LOCAL, + CBM_SEM_DOC_POS_OUTSIDE, + CBM_SEM_DOC_POS_COUNT, +} cbm_sem_doc_pos_t; + +/* What a candidate points at: one of a section's best CBM_SEM_DOC_TOP_K + * functions; a function inside the home below that top K (at most + * CBM_SEM_DOC_LOCAL_K per section); a whole file; the doc's home folder. */ +typedef enum { + CBM_SEM_DOC_KIND_FUNCTION = 0, + CBM_SEM_DOC_KIND_LOCAL, + CBM_SEM_DOC_KIND_FILE, + CBM_SEM_DOC_KIND_FOLDER, + CBM_SEM_DOC_KIND_COUNT, +} cbm_sem_doc_kind_t; +#define CBM_SEM_DOC_LOCAL_K 2 +#define CBM_SEM_DOC_FILE_K 3 + +/* The probability that a section of format fmt is about a candidate of this + * kind and position with this tfidf, from the judged curves; 0 when that band + * is not stored (judged below 0.20) or has no judged curve. A folder + * candidate has no score: cbm_sem_doc_folder_p. */ +float cbm_sem_doc_calibrated_p(cbm_sem_doc_format_t fmt, cbm_sem_doc_kind_t kind, + cbm_sem_doc_pos_t pos, float score); + +/* The probability that the doc at doc_path documents its home folder, by + * the doc's kind (a README or index file, or any other doc); 0 = not + * stored. */ +float cbm_sem_doc_folder_p(cbm_sem_doc_format_t fmt, const char *doc_path); + +/* p as stored and shown: two decimals, half up (0.225 -> 0.23, 0.325 -> + * 0.33; the epsilon absorbs the float representation of a judged x.xx5). */ +double cbm_sem_p_2dp(float p); + /* Module proximity multiplier based on file paths. */ float cbm_sem_proximity(const char *path_a, const char *path_b); diff --git a/src/store/store.c b/src/store/store.c index 4f76915d5..b974e5e39 100644 --- a/src/store/store.c +++ b/src/store/store.c @@ -10746,6 +10746,178 @@ int cbm_adr_validate_section_keys(const char **keys, int count, char *errbuf, in return CBM_STORE_OK; } +/* ── Doc -> code candidates ─────────────────────────────────────── */ + +static const char DOC_CANDIDATES_DDL[] = + "CREATE TABLE IF NOT EXISTS doc_link_candidates (" + " project TEXT NOT NULL," + " section_qn TEXT NOT NULL," + " target_qn TEXT NOT NULL," + " rank INTEGER NOT NULL," + " score REAL NOT NULL," + " p REAL NOT NULL," + " evidence TEXT NOT NULL DEFAULT ''" + ");" + "CREATE INDEX IF NOT EXISTS idx_doc_link_candidates_section " + "ON doc_link_candidates(project, section_qn);" + "CREATE INDEX IF NOT EXISTS idx_doc_link_candidates_target " + "ON doc_link_candidates(project, target_qn);"; + +int cbm_store_doc_candidates_replace(cbm_store_t *s, const char *project, + const cbm_doc_candidate_t *rows, int count) { + if (!s || !s->db || !project || count < 0 || (count > 0 && !rows)) { + return CBM_STORE_ERR; + } + if (exec_sql(s, DOC_CANDIDATES_DDL) != CBM_STORE_OK || exec_sql(s, "BEGIN;") != CBM_STORE_OK) { + return CBM_STORE_ERR; + } + sqlite3_stmt *del = NULL; + sqlite3_stmt *ins = NULL; + bool ok = sqlite3_prepare_v2(s->db, "DELETE FROM doc_link_candidates WHERE project = ?1;", + CBM_NOT_FOUND, &del, NULL) == SQLITE_OK && + sqlite3_prepare_v2(s->db, + "INSERT INTO doc_link_candidates " + "(project, section_qn, target_qn, rank, score, p, evidence) " + "VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7);", + CBM_NOT_FOUND, &ins, NULL) == SQLITE_OK; + if (ok) { + bind_text(del, SKIP_ONE, project); + ok = sqlite3_step(del) == SQLITE_DONE; + } + for (int i = 0; ok && i < count; i++) { + if (!rows[i].section_qn || !rows[i].target_qn) { + continue; + } + bind_text(ins, SKIP_ONE, project); + bind_text(ins, ST_COL_2, rows[i].section_qn); + bind_text(ins, ST_COL_3, rows[i].target_qn); + sqlite3_bind_int(ins, ST_COL_4, rows[i].rank); + sqlite3_bind_double(ins, ST_COL_5, rows[i].score); + sqlite3_bind_double(ins, ST_COL_6, rows[i].p); + bind_text(ins, CBM_SZ_7, rows[i].evidence ? rows[i].evidence : ""); + ok = sqlite3_step(ins) == SQLITE_DONE; + sqlite3_reset(ins); + } + sqlite3_finalize(del); + sqlite3_finalize(ins); + if (!ok) { + store_set_error_sqlite(s, "doc_candidates replace"); + (void)exec_sql(s, "ROLLBACK;"); + return CBM_STORE_ERR; + } + return exec_sql(s, "COMMIT;"); +} + +/* Does the database hold the candidates table? CBM_STORE_NOT_FOUND: no (a + * database written by an older build). */ +static int doc_candidates_table(cbm_store_t *s) { + sqlite3_stmt *probe = NULL; + if (sqlite3_prepare_v2(s->db, + "SELECT 1 FROM sqlite_master WHERE type='table' AND " + "name='doc_link_candidates' LIMIT 1;", + CBM_NOT_FOUND, &probe, NULL) != SQLITE_OK) { + store_set_error_sqlite(s, "doc_candidates probe"); + return CBM_STORE_ERR; + } + int rc = sqlite3_step(probe); + sqlite3_finalize(probe); + if (rc == SQLITE_DONE) { + return CBM_STORE_NOT_FOUND; + } + if (rc != SQLITE_ROW) { + store_set_error_sqlite(s, "doc_candidates probe step"); + return CBM_STORE_ERR; + } + return CBM_STORE_OK; +} + +/* Run a prepared candidates query (project ?1, key ?2, limit ?3; columns + * section_qn, target_qn, rank, score, p, evidence) into *out; finalizes it. */ +static int doc_candidates_read(sqlite3_stmt *stmt, const char *project, const char *key, int limit, + cbm_doc_candidate_t **out, int *count) { + bind_text(stmt, SKIP_ONE, project); + bind_text(stmt, ST_COL_2, key); + sqlite3_bind_int(stmt, ST_COL_3, limit); + cbm_doc_candidate_t *rows = cbm_calloc(CBM_MEM_CLASS_STORE, (size_t)limit * sizeof(*rows)); + if (!rows) { + sqlite3_finalize(stmt); + return CBM_STORE_ERR; + } + int n = 0; + while (n < limit && sqlite3_step(stmt) == SQLITE_ROW) { + rows[n].section_qn = + cbm_mem_strdup(CBM_MEM_CLASS_STORE, (const char *)sqlite3_column_text(stmt, 0)); + rows[n].target_qn = + cbm_mem_strdup(CBM_MEM_CLASS_STORE, (const char *)sqlite3_column_text(stmt, SKIP_ONE)); + rows[n].rank = sqlite3_column_int(stmt, ST_COL_2); + rows[n].score = sqlite3_column_double(stmt, ST_COL_3); + rows[n].p = sqlite3_column_double(stmt, ST_COL_4); + rows[n].evidence = + cbm_mem_strdup(CBM_MEM_CLASS_STORE, (const char *)sqlite3_column_text(stmt, ST_COL_5)); + n++; + } + sqlite3_finalize(stmt); + *out = rows; + *count = n; + return CBM_STORE_OK; +} + +static int doc_candidates_query(cbm_store_t *s, const char *sql, const char *project, + const char *key, int limit, cbm_doc_candidate_t **out, int *count) { + *out = NULL; + *count = 0; + int rc = doc_candidates_table(s); + if (rc != CBM_STORE_OK) { + return rc; + } + sqlite3_stmt *stmt = NULL; + if (sqlite3_prepare_v2(s->db, sql, CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) { + store_set_error_sqlite(s, "doc_candidates get"); + return CBM_STORE_ERR; + } + return doc_candidates_read(stmt, project, key, limit, out, count); +} + +int cbm_store_doc_candidates_get(cbm_store_t *s, const char *project, const char *section_qn, + const char *target_qn, int limit, cbm_doc_candidate_t **out, + int *count) { + if (!s || !s->db || !project || !out || !count || (!section_qn == !target_qn) || limit <= 0) { + return CBM_STORE_ERR; + } + const char *sql = section_qn ? "SELECT section_qn, target_qn, rank, score, p, evidence FROM " + "doc_link_candidates WHERE project=?1 AND section_qn=?2 " + "ORDER BY p DESC, score DESC, target_qn LIMIT ?3;" + : "SELECT section_qn, target_qn, rank, score, p, evidence FROM " + "doc_link_candidates WHERE project=?1 AND target_qn=?2 " + "ORDER BY p DESC, score DESC, section_qn LIMIT ?3;"; + return doc_candidates_query(s, sql, project, section_qn ? section_qn : target_qn, limit, out, + count); +} + +int cbm_store_doc_candidates_for_path(cbm_store_t *s, const char *project, const char *file_path, + int limit, cbm_doc_candidate_t **out, int *count) { + if (!s || !s->db || !project || !file_path || !file_path[0] || !out || !count || limit <= 0) { + return CBM_STORE_ERR; + } + static const char SQL[] = "SELECT c.section_qn, c.target_qn, c.rank, c.score, c.p, c.evidence " + "FROM doc_link_candidates c JOIN nodes n " + "ON n.project = c.project AND n.qualified_name = c.target_qn " + "WHERE c.project=?1 AND ((n.label='File' AND n.file_path=?2) OR " + "(n.label='Folder' AND n.file_path <> '' AND " + "substr(?2, 1, length(n.file_path) + 1) = n.file_path || '/')) " + "ORDER BY c.p DESC, c.score DESC, c.section_qn LIMIT ?3;"; + return doc_candidates_query(s, SQL, project, file_path, limit, out, count); +} + +void cbm_store_doc_candidates_free(cbm_doc_candidate_t *rows, int count) { + for (int i = 0; rows && i < count; i++) { + cbm_free(CBM_MEM_CLASS_STORE, (void *)rows[i].section_qn); + cbm_free(CBM_MEM_CLASS_STORE, (void *)rows[i].target_qn); + cbm_free(CBM_MEM_CLASS_STORE, (void *)rows[i].evidence); + } + cbm_free(CBM_MEM_CLASS_STORE, rows); +} + void cbm_adr_sections_free(cbm_adr_sections_t *s) { if (!s) { return; diff --git a/src/store/store.h b/src/store/store.h index 8dc67ee48..2a8d64048 100644 --- a/src/store/store.h +++ b/src/store/store.h @@ -1083,6 +1083,41 @@ int cbm_store_adr_update_sections(cbm_store_t *s, const char *project, const cha const char **values, int count, cbm_adr_t *out); void cbm_store_adr_free(cbm_adr_t *adr); +/* ── Doc -> code candidates (for the agent to verify) ───────────── */ + +/* One documentation section -> function candidate, with the calibrated + * probability that the section is about the function. Candidates, not links: + * never graph edges; a short ranked list per section for the agent to judge. */ +typedef struct { + const char *section_qn; + const char *target_qn; + int rank; + double score; + double p; + const char *evidence; /* JSON: {"terms":n,"name_tokens":m} */ +} cbm_doc_candidate_t; + +/* Replace the project's candidates. The table is created here, at publish + * (not in the schema): a database written by an older build has none, and + * readers treat that as "no candidates". */ +int cbm_store_doc_candidates_replace(cbm_store_t *s, const char *project, + const cbm_doc_candidate_t *rows, int count); + +/* The candidates of one section (section_qn) or for one function (target_qn; + * pass the other NULL), highest p first, then score, at most limit. *out is + * allocated in CBM_MEM_CLASS_STORE (cbm_store_doc_candidates_free). + * CBM_STORE_NOT_FOUND: no table. */ +int cbm_store_doc_candidates_get(cbm_store_t *s, const char *project, const char *section_qn, + const char *target_qn, int limit, cbm_doc_candidate_t **out, + int *count); + +/* The candidates that point at a file as a whole: at its File node, or at a + * Folder node that holds it (a doc's home folder), highest p first, then + * score, at most limit. Allocation and CBM_STORE_NOT_FOUND as above. */ +int cbm_store_doc_candidates_for_path(cbm_store_t *s, const char *project, const char *file_path, + int limit, cbm_doc_candidate_t **out, int *count); +void cbm_store_doc_candidates_free(cbm_doc_candidate_t *rows, int count); + /* ADR section parsing/rendering (pure functions, no store needed) */ enum { PROPS_MAX = 16 }; diff --git a/tests/test_mcp.c b/tests/test_mcp.c index 5d107a8b3..abd19c124 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -2546,6 +2546,98 @@ TEST(tool_get_file_outline_returns_bounded_filtered_columnar_rows_issue469) { PASS(); } +/* get_file_outline shows the sections possibly about the file as a whole or + * about a folder holding it (highest p first, kind shown), in both formats, + * on the first page only; a candidate for a function in the file is not one + * of them. */ +TEST(tool_get_file_outline_shows_doc_candidates_of_file_and_folders) { + cbm_mcp_server_t *srv = setup_mcp_with_data(); + cbm_store_t *store = cbm_mcp_server_store(srv); + ASSERT_NOT_NULL(store); + ASSERT_EQ(cbm_store_upsert_project(store, "doc-outline", "/tmp/doc-outline"), CBM_STORE_OK); + cbm_mcp_server_set_project(srv, "doc-outline"); + const struct { + const char *label, *qn, *path; + int start; + } nodes[] = {{"File", "doc-outline.pkg.mail.parse.__file__", "pkg/mail/parse.py", 0}, + {"Function", "doc-outline.pkg.mail.parse.parse_mail", "pkg/mail/parse.py", 3}, + {"Folder", "doc-outline.pkg.mail", "pkg/mail", 0}, + {"Section", "doc-outline.pkg.mail.README.Mail", "pkg/mail/README.md", 1}, + {"Section", "doc-outline.docs.parsing.Parsing", "docs/parsing.md", 5}}; + for (size_t i = 0; i < sizeof(nodes) / sizeof(nodes[0]); i++) { + cbm_node_t node = {.project = "doc-outline", + .label = nodes[i].label, + .name = nodes[i].qn, + .qualified_name = nodes[i].qn, + .file_path = nodes[i].path, + .start_line = nodes[i].start, + .end_line = nodes[i].start + 2}; + ASSERT_GT(cbm_store_upsert_node(store, &node), 0); + } + const cbm_doc_candidate_t rows[] = { + {.section_qn = "doc-outline.docs.parsing.Parsing", + .target_qn = "doc-outline.pkg.mail.parse.__file__", + .rank = 1, + .score = 0.41, + .p = 0.5, + .evidence = "{\"shared_terms\":4,\"name_tokens\":0,\"kind\":\"file\"," + "\"position\":\"global\"}"}, + {.section_qn = "doc-outline.pkg.mail.README.Mail", + .target_qn = "doc-outline.pkg.mail", + .rank = 1, + .p = 0.8, + .evidence = "{\"shared_terms\":0,\"name_tokens\":0,\"kind\":\"folder\"," + "\"position\":\"local\"}"}, + {.section_qn = "doc-outline.docs.parsing.Parsing", + .target_qn = "doc-outline.pkg.mail.parse.parse_mail", + .rank = 1, + .score = 0.5, + .p = 0.9, + .evidence = "{\"shared_terms\":5,\"name_tokens\":1,\"kind\":\"function\"," + "\"position\":\"global\"}"}, + }; + ASSERT_EQ(cbm_store_doc_candidates_replace(store, "doc-outline", rows, 3), CBM_STORE_OK); + + char *raw = + cbm_mcp_handle_tool(srv, "get_file_outline", + "{\"project\":\"doc-outline\",\"file_path\":\"pkg/mail/parse.py\"," + "\"format\":\"json\"}"); + char *text = extract_text_content(raw); + free(raw); + ASSERT_NOT_NULL(text); + yyjson_doc *doc = yyjson_read(text, strlen(text), 0); + ASSERT_NOT_NULL(doc); + yyjson_val *in = yyjson_obj_get(yyjson_doc_get_root(doc), "possibly_described_in"); + ASSERT_EQ((int)yyjson_arr_size(in), 2); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(yyjson_arr_get(in, 0), "section")), + "doc-outline.pkg.mail.README.Mail"); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(yyjson_arr_get(in, 0), "kind")), "folder"); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(yyjson_arr_get(in, 1), "kind")), "file"); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(yyjson_arr_get(in, 1), "file_path")), + "docs/parsing.md"); + ASSERT_NOT_NULL(yyjson_obj_get(yyjson_doc_get_root(doc), "candidates_note")); + yyjson_doc_free(doc); + free(text); + + raw = cbm_mcp_handle_tool(srv, "get_file_outline", + "{\"project\":\"doc-outline\",\"file_path\":\"pkg/mail/parse.py\"}"); + ASSERT_NOT_NULL(raw); + ASSERT_NOT_NULL(strstr(raw, "possibly_described_in")); + ASSERT_NOT_NULL(strstr(raw, "doc-outline.pkg.mail.README.Mail")); + ASSERT_NOT_NULL(strstr(raw, "folder")); + free(raw); + + raw = cbm_mcp_handle_tool(srv, "get_file_outline", + "{\"project\":\"doc-outline\",\"file_path\":\"pkg/mail/parse.py\"," + "\"offset\":1}"); + ASSERT_NOT_NULL(raw); + ASSERT_NULL(strstr(raw, "possibly_described_in")); + free(raw); + + cbm_mcp_server_free(srv); + PASS(); +} + TEST(tool_get_file_outline_validates_json_path_limit_and_cancel_issue469) { cbm_mcp_server_t *srv = setup_mcp_with_data(); cbm_store_t *store = cbm_mcp_server_store(srv); @@ -7618,9 +7710,10 @@ TEST(tool_trace_call_path_qn_fallback_frees_name_miss) { cbm_store_free_nodes(by_name, by_name_count); char *resp = cbm_mcp_server_handle( - srv, "{\"jsonrpc\":\"2.0\",\"id\":63,\"method\":\"tools/call\"," - "\"params\":{\"name\":\"trace_call_path\",\"arguments\":{\"function_name\":" - "\"qnfb-proj.src.qnfb_entry\",\"project\":\"qnfb-proj\",\"direction\":\"outbound\"}}}"); + srv, + "{\"jsonrpc\":\"2.0\",\"id\":63,\"method\":\"tools/call\"," + "\"params\":{\"name\":\"trace_call_path\",\"arguments\":{\"function_name\":" + "\"qnfb-proj.src.qnfb_entry\",\"project\":\"qnfb-proj\",\"direction\":\"outbound\"}}}"); ASSERT_NOT_NULL(resp); char *inner = extract_text_content(resp); ASSERT_NOT_NULL(inner); @@ -18508,6 +18601,93 @@ TEST(snippet_exact_qn) { PASS(); } +/* Doc -> code candidates on get_code_snippet: a Section lists the functions + * it is possibly about (highest p first, evidence included), a function the + * sections that possibly describe it; a row whose other end no longer exists + * is skipped, and a node without rows gets no candidate keys at all. */ +TEST(snippet_doc_candidates_both_directions) { + char tmp[256]; + cbm_mcp_server_t *srv = setup_snippet_server(tmp, sizeof(tmp)); + ASSERT_NOT_NULL(srv); + cbm_store_t *st = cbm_mcp_server_store(srv); + cbm_node_t sec = {.project = "test-project", + .label = "Section", + .name = "Orders", + .qualified_name = "test-project.README.Orders", + .file_path = "README.md", + .start_line = 1, + .end_line = 4}; + ASSERT_TRUE(cbm_store_upsert_node(st, &sec) > 0); + const cbm_doc_candidate_t rows[] = { + {.section_qn = "test-project.README.Orders", + .target_qn = "test-project.cmd.server.main.HandleRequest", + .rank = 2, + .score = 0.25, + .p = 0.36, + .evidence = "{\"shared_terms\":2,\"name_tokens\":0}"}, + {.section_qn = "test-project.README.Orders", + .target_qn = "test-project.cmd.server.main.ProcessOrder", + .rank = 1, + .score = 0.45, + .p = 0.548, + .evidence = "{\"shared_terms\":5,\"name_tokens\":2,\"kind\":\"function\"," + "\"position\":\"local\"}"}, + {.section_qn = "test-project.README.Orders", + .target_qn = "test-project.gone.Missing", + .rank = 3, + .score = 0.31, + .p = 0.42, + .evidence = "{\"shared_terms\":1,\"name_tokens\":0}"}, + }; + ASSERT_EQ(cbm_store_doc_candidates_replace(st, "test-project", rows, 3), CBM_STORE_OK); + + char *resp = call_snippet(srv, "{\"qualified_name\":\"test-project.README.Orders\"," + "\"project\":\"test-project\"}"); + ASSERT_NOT_NULL(resp); + yyjson_doc *doc = yyjson_read(resp, strlen(resp), 0); + ASSERT_NOT_NULL(doc); + yyjson_val *about = yyjson_obj_get(yyjson_doc_get_root(doc), "possibly_about"); + ASSERT_EQ((int)yyjson_arr_size(about), 2); + yyjson_val *first = yyjson_arr_get(about, 0); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(first, "qualified_name")), + "test-project.cmd.server.main.ProcessOrder"); + ASSERT_FLOAT_EQ(yyjson_get_num(yyjson_obj_get(first, "p")), 0.548, 1e-9); + ASSERT_EQ((int)yyjson_get_int(yyjson_obj_get(first, "shared_terms")), 5); + ASSERT_EQ((int)yyjson_get_int(yyjson_obj_get(first, "start_line")), 7); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(first, "label")), "Function"); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(first, "kind")), "function"); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(first, "position")), "local"); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(yyjson_arr_get(about, 1), "qualified_name")), + "test-project.cmd.server.main.HandleRequest"); + ASSERT_NOT_NULL(yyjson_obj_get(yyjson_doc_get_root(doc), "candidates_note")); + yyjson_doc_free(doc); + free(resp); + + resp = call_snippet(srv, "{\"qualified_name\":\"test-project.cmd.server.main.ProcessOrder\"," + "\"project\":\"test-project\"}"); + ASSERT_NOT_NULL(resp); + doc = yyjson_read(resp, strlen(resp), 0); + ASSERT_NOT_NULL(doc); + yyjson_val *in = yyjson_obj_get(yyjson_doc_get_root(doc), "possibly_described_in"); + ASSERT_EQ((int)yyjson_arr_size(in), 1); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(yyjson_arr_get(in, 0), "section")), + "test-project.README.Orders"); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(yyjson_arr_get(in, 0), "file_path")), "README.md"); + yyjson_doc_free(doc); + free(resp); + + resp = call_snippet(srv, "{\"qualified_name\":\"test-project.cmd.server.Run\"," + "\"project\":\"test-project\"}"); + ASSERT_NOT_NULL(resp); + ASSERT_NULL(strstr(resp, "possibly_")); + ASSERT_NULL(strstr(resp, "candidates_note")); + free(resp); + + cbm_mcp_server_free(srv); + cleanup_snippet_dir(tmp); + PASS(); +} + /* ── TestSnippet_QNSuffix ─────────────────────────────────────── */ TEST(snippet_qn_suffix) { @@ -23701,6 +23881,7 @@ SUITE(mcp) { RUN_TEST(tool_compare_graphs_validation_and_scan_cap_are_atomic_issue525); RUN_TEST(tool_compare_graphs_cancel_and_readonly_handles_release_issue525); RUN_TEST(tool_get_file_outline_returns_bounded_filtered_columnar_rows_issue469); + RUN_TEST(tool_get_file_outline_shows_doc_candidates_of_file_and_folders); RUN_TEST(tool_get_file_outline_validates_json_path_limit_and_cancel_issue469); RUN_TEST(tool_search_graph_basic); RUN_TEST(tool_search_graph_empty_hint_describes_combined_filters_issue1366); @@ -23952,6 +24133,7 @@ SUITE(mcp) { /* Snippet resolution (port of snippet_test.go) */ RUN_TEST(snippet_exact_qn); + RUN_TEST(snippet_doc_candidates_both_directions); RUN_TEST(snippet_qn_suffix); RUN_TEST(snippet_unique_short_name); RUN_TEST(snippet_c_macro_namespace_c1); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index a2861d054..ddf7bc5a5 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -14,6 +14,7 @@ #include "pipeline/pipeline_internal.h" #include "pipeline/artifact.h" #include "store/store.h" +#include "semantic/semantic.h" #include "git/git_context.h" #include "foundation/dump_verify.h" #include "foundation/sha256.h" @@ -18810,6 +18811,590 @@ TEST(pipeline_semantic_batched_matches_unbatched) { PASS(); } +/* CBM_SEM_PAIR_SIGNALS (+ _FLOOR) is the semantic engine's measuring + * harness: it records each scored pair with its signal values, from a floor + * below the threshold up. Recording must never change the graph: the same + * SEMANTICALLY_RELATED rows with and without it, one admitted line per edge, + * and every pair under the threshold recorded but never admitted. */ +TEST(pipeline_semantic_pair_signals_never_change_the_graph) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_semsig_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + static const char *const subjects[] = {"User", "Server", "Client", "Cache", "Queue", "Mail"}; + for (size_t i = 0; i < sizeof(subjects) / sizeof(subjects[0]); i++) { + char file[64]; + snprintf(file, sizeof(file), "%s_config.go", subjects[i]); + write_sem_family(tmp, file, subjects[i]); + } + char db_plain[512]; + char db_signals[512]; + char tsv[512]; + snprintf(db_plain, sizeof(db_plain), "%s/plain.db", tmp); + snprintf(db_signals, sizeof(db_signals), "%s/signals.db", tmp); + snprintf(tsv, sizeof(tsv), "%s/pairs.tsv", tmp); + + cbm_setenv("CBM_SEMANTIC_THRESHOLD", "0.3", 1); /* as above: admits the near-duplicates */ + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_plain, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + cbm_pipeline_free(p); + + cbm_setenv("CBM_SEM_PAIR_SIGNALS", tsv, 1); + cbm_setenv("CBM_SEM_PAIR_SIGNALS_FLOOR", "0.05", 1); + p = cbm_pipeline_new(tmp, db_signals, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + int rc = cbm_pipeline_run(p); + cbm_unsetenv("CBM_SEM_PAIR_SIGNALS"); + cbm_unsetenv("CBM_SEM_PAIR_SIGNALS_FLOOR"); + cbm_unsetenv("CBM_SEMANTIC_THRESHOLD"); + cbm_pipeline_free(p); + ASSERT_EQ(rc, 0); + + cbm_store_t *sp = cbm_store_open_path(db_plain); + cbm_store_t *ss = cbm_store_open_path(db_signals); + ASSERT_NOT_NULL(sp); + ASSERT_NOT_NULL(ss); + int edges = 0; + ASSERT_TRUE( + sem_same_rows(cbm_store_get_db(sp), cbm_store_get_db(ss), + "SELECT s.qualified_name, t.qualified_name, e.properties FROM edges e " + "JOIN nodes s ON s.id = e.source_id JOIN nodes t ON t.id = e.target_id " + "WHERE e.type = 'SEMANTICALLY_RELATED' ORDER BY 1, 2", + &edges)); + ASSERT_GT(edges, 0); + cbm_store_close(sp); + cbm_store_close(ss); + + /* qa qb score tfidf ri minhash api type decorator struct proximity + * same_file admitted loc_a loc_b */ + FILE *f = cbm_fopen(tsv, "r"); + ASSERT_NOT_NULL(f); + char line[4096]; + int lines = 0; + int admitted = 0; + int below = 0; + while (fgets(line, sizeof(line), f)) { + char *field[16]; + int n = 0; + for (char *tok = strtok(line, "\t\n"); tok && n < 16; tok = strtok(NULL, "\t\n")) { + field[n++] = tok; + } + ASSERT_EQ(n, 15); + float score = strtof(field[2], NULL); + bool is_admitted = strcmp(field[12], "1") == 0; + if (score < 0.3F) { + below++; + ASSERT_FALSE(is_admitted); + } + admitted += is_admitted ? 1 : 0; + lines++; + } + fclose(f); + ASSERT_EQ(admitted, edges); + ASSERT_GT(below, 0); /* the floor recorded pairs under the threshold */ + ASSERT_GT(lines, admitted); + th_rmtree(tmp); + PASS(); +} + +/* Every SEMANTICALLY_RELATED edge carries "p": the calibrated probability for + * its score AS STORED (three decimals), so one shown score has one p. */ +TEST(pipeline_semantic_edges_carry_p) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_semp_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + static const char *const subjects[] = {"User", "Server", "Client", "Cache", "Queue", "Mail"}; + for (size_t i = 0; i < sizeof(subjects) / sizeof(subjects[0]); i++) { + char file[64]; + snprintf(file, sizeof(file), "%s_config.go", subjects[i]); + write_sem_family(tmp, file, subjects[i]); + } + char db[512]; + snprintf(db, sizeof(db), "%s/p.db", tmp); + cbm_setenv("CBM_SEMANTIC_THRESHOLD", "0.3", 1); /* as above: admits the near-duplicates */ + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + int rc = cbm_pipeline_run(p); + cbm_unsetenv("CBM_SEMANTIC_THRESHOLD"); + cbm_pipeline_free(p); + ASSERT_EQ(rc, 0); + + cbm_store_t *s = cbm_store_open_path(db); + ASSERT_NOT_NULL(s); + sqlite3_stmt *st = NULL; + ASSERT_EQ(sqlite3_prepare_v2(cbm_store_get_db(s), + "SELECT properties FROM edges WHERE type = 'SEMANTICALLY_RELATED'", + -1, &st, NULL), + SQLITE_OK); + int edges = 0; + while (sqlite3_step(st) == SQLITE_ROW) { + const char *props = (const char *)sqlite3_column_text(st, 0); + ASSERT_NOT_NULL(props); + yyjson_doc *doc = yyjson_read(props, strlen(props), 0); + ASSERT_NOT_NULL(doc); + yyjson_val *score = yyjson_obj_get(yyjson_doc_get_root(doc), "score"); + yyjson_val *pv = yyjson_obj_get(yyjson_doc_get_root(doc), "p"); + ASSERT_TRUE(score && yyjson_is_num(score)); + ASSERT_TRUE(pv && yyjson_is_num(pv)); + double want = cbm_sem_calibrated_p((float)yyjson_get_num(score)); + ASSERT_FLOAT_EQ(yyjson_get_num(pv), want, 0.006); + yyjson_doc_free(doc); + edges++; + } + sqlite3_finalize(st); + cbm_store_close(s); + ASSERT_GT(edges, 0); + th_rmtree(tmp); + PASS(); +} + +/* The p a stored candidate row must carry: the curve of the row's kind and + * position (read from its evidence) at its score, two decimals; -1 for an + * unknown kind or position. *is_function: a function candidate (top 5). */ +static double doc_row_want(const char *evidence, cbm_sem_doc_format_t fmt, float score, + bool *is_function) { + static const struct { + const char *name; + cbm_sem_doc_kind_t kind; + } KINDS[] = {{"\"kind\":\"function\"", CBM_SEM_DOC_KIND_FUNCTION}, + {"\"kind\":\"local\"", CBM_SEM_DOC_KIND_LOCAL}, + {"\"kind\":\"file\"", CBM_SEM_DOC_KIND_FILE}}; + static const struct { + const char *name; + cbm_sem_doc_pos_t pos; + } POSITIONS[] = {{"\"position\":\"global\"", CBM_SEM_DOC_POS_GLOBAL}, + {"\"position\":\"local\"", CBM_SEM_DOC_POS_LOCAL}, + {"\"position\":\"outside\"", CBM_SEM_DOC_POS_OUTSIDE}}; + int k = -1; + int p = -1; + for (int i = 0; evidence && i < 3; i++) { + k = k < 0 && strstr(evidence, KINDS[i].name) ? i : k; + p = p < 0 && strstr(evidence, POSITIONS[i].name) ? i : p; + } + *is_function = k == 0; + return k < 0 || p < 0 ? -1.0 + : cbm_sem_p_2dp(cbm_sem_doc_calibrated_p(fmt, KINDS[k].kind, + POSITIONS[p].pos, score)); +} + +/* Count a section's stored doc -> code candidates (section QN containing + * heading) and check each row's shape: a function within the top + * CBM_SEM_DOC_TOP_K (top: the best of them), score at or above the floor, p + * the curve of its kind and position at two decimals, never a test + * function. Returns -1 on a bad row. */ +static int doc_candidate_rows_for(cbm_store_t *s, const char *heading, char *top, size_t top_sz) { + sqlite3_stmt *st = NULL; + if (sqlite3_prepare_v2(cbm_store_get_db(s), + "SELECT target_qn, rank, score, p, evidence FROM doc_link_candidates " + "WHERE instr(section_qn, ?1) > 0 ORDER BY rank", + -1, &st, NULL) != SQLITE_OK) { + return -1; + } + sqlite3_bind_text(st, 1, heading, -1, SQLITE_TRANSIENT); + int n = 0; + bool ok = true; + bool have_top = false; + while (ok && sqlite3_step(st) == SQLITE_ROW) { + const char *target = (const char *)sqlite3_column_text(st, 0); + int rank = sqlite3_column_int(st, 1); + double score = sqlite3_column_double(st, 2); + double p = sqlite3_column_double(st, 3); + bool is_function = false; + double want = doc_row_want((const char *)sqlite3_column_text(st, 4), + CBM_SEM_DOC_FMT_MARKDOWN, (float)score, &is_function); + ok = target && rank >= 1 && (!is_function || rank <= CBM_SEM_DOC_TOP_K) && + score >= (double)CBM_SEM_DOC_MIN_SCORE && p == want && !strstr(target, ".Test"); + if (ok && is_function && !have_top && top) { + snprintf(top, top_sz, "%s", target); + have_top = true; + } + n++; + } + sqlite3_finalize(st); + return ok ? n : -1; +} + +/* Doc -> code candidates end to end: a full index publishes a section's + * calibrated candidates (the best match first, a test function never, a + * changelog section none), and a delta-route reindex keeps them -- its proxy + * buffer cannot recompute them, so the clone's rows must survive. */ +TEST(pipeline_doc_candidates_published_and_kept_by_delta) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_doccand_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + static const char *const subjects[] = {"User", "Server", "Client", "Cache", "Queue", "Mail"}; + for (size_t i = 0; i < sizeof(subjects) / sizeof(subjects[0]); i++) { + char file[64]; + snprintf(file, sizeof(file), "%s_config.go", subjects[i]); + write_sem_family(tmp, file, subjects[i]); + } + /* A test whose comment is the section's own text: the best match of all, + * and still never a candidate. */ + write_temp_file(tmp, "mail_config_test.go", + "package main\n\n" + "// TestParseMailConfig: the mail config file is parsed by ParseMailConfig,\n" + "// which loads the mail config from disk and validates the mail config\n" + "// fields before anything reads them. Missing fields fail the validation,\n" + "// and the parsed mail config can be written back to disk afterwards.\n" + "func TestParseMailConfig(t *testing.T) {\n" + "\tParseMailConfig(\"mail.conf\")\n}\n"); + write_temp_file(tmp, "README.md", + "# Mail configuration\n\n" + "The mail config file is parsed by ParseMailConfig, which loads the mail " + "config from disk and validates the mail config fields before anything " + "reads them. Missing fields fail the validation, and the parsed mail " + "config can be written back to disk afterwards.\n\n" + "## Changelog\n\n" + "The mail config loader now validates missing fields and writes the mail " + "config back to disk after parsing, so a parsed mail config file always " + "carries every field it needs.\n"); + char db[512]; + snprintf(db, sizeof(db), "%s/doccand.db", tmp); + + cbm_pipeline_incremental_test_reset_faults(); + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + cbm_pipeline_free(p); + + cbm_store_t *s = cbm_store_open_path(db); + ASSERT_NOT_NULL(s); + char top[512] = ""; + int mail = doc_candidate_rows_for(s, "Mail-configuration", top, sizeof(top)); + int changelog = doc_candidate_rows_for(s, "Changelog", NULL, 0); + cbm_store_close(s); + ASSERT_GT(mail, 0); + ASSERT_NOT_NULL(strstr(top, "Mail")); + ASSERT_EQ(changelog, 0); + + /* Delta route: an edit that keeps every definition name (closure repair + * declines added names and new files -- the name must match the fixture's + * case exactly, or a case-sensitive filesystem gets a new file); the + * candidates are kept as they were. */ + char edited[512]; + snprintf(edited, sizeof(edited), "%s/User_config.go", tmp); + FILE *ef = cbm_fopen(edited, "a"); + ASSERT_NOT_NULL(ef); + (void)fputs("\n// edited after the first index\n", ef); + (void)fclose(ef); + cbm_pipeline_incremental_test_reset_faults(); + p = cbm_pipeline_new(tmp, db, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + ASSERT_EQ(cbm_pipeline_incremental_test_last_route(), CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR); + cbm_pipeline_free(p); + s = cbm_store_open_path(db); + ASSERT_NOT_NULL(s); + char top_after[512] = ""; + int mail_after = doc_candidate_rows_for(s, "Mail-configuration", top_after, sizeof(top_after)); + cbm_store_close(s); + ASSERT_EQ(mail_after, mail); + ASSERT_STR_EQ(top_after, top); + th_rmtree(tmp); + cbm_pipeline_incremental_test_reset_faults(); + PASS(); +} + +/* One Python family per subject: parse / load / validate / write, each with + * a docstring, so the functions' corpus has the subject words to match. */ +static void write_py_family(const char *base, const char *subject) { + char path[512]; + snprintf(path, sizeof(path), "%s/%s.py", base, subject); + char body[4096]; + snprintf(body, sizeof(body), + "def parse_%s_config(path):\n" + " \"\"\"Parse the %s config file and return the parsed %s settings.\"\"\"\n" + " return load_%s_config(path)\n\n" + "def load_%s_config(path):\n" + " \"\"\"Load the %s config from disk and validate every %s field.\"\"\"\n" + " cfg = {}\n validate_%s_config(cfg)\n return cfg\n\n" + "def validate_%s_config(cfg):\n" + " \"\"\"Check the %s config for missing fields.\"\"\"\n" + " return cfg is not None\n\n" + "def write_%s_config(path, cfg):\n" + " \"\"\"Write the %s config back to disk after validating it.\"\"\"\n" + " validate_%s_config(cfg)\n", + subject, subject, subject, subject, subject, subject, subject, subject, subject, + subject, subject, subject, subject); + th_write_file(path, body); +} + +/* The candidate count, and the bad rows, of one section (its QN contains + * heading and its file ends in ext): p must be the format's curve value at + * two decimals and the tfidf at or above the format's first stored band. */ +static int doc_rows_of(cbm_store_t *s, const char *heading, const char *ext, + cbm_sem_doc_format_t fmt, const char *forbidden_target, int *bad) { + sqlite3_stmt *st = NULL; + *bad = 0; + if (sqlite3_prepare_v2( + cbm_store_get_db(s), + "SELECT c.target_qn, c.score, c.p, c.evidence FROM doc_link_candidates c " + "JOIN nodes n ON n.qualified_name = c.section_qn " + "WHERE instr(c.section_qn, ?1) > 0 AND n.file_path LIKE ?2", + -1, &st, NULL) != SQLITE_OK) { + return -1; + } + char like[32]; + snprintf(like, sizeof(like), "%%%s", ext); + sqlite3_bind_text(st, 1, heading, -1, SQLITE_TRANSIENT); + sqlite3_bind_text(st, 2, like, -1, SQLITE_TRANSIENT); + int n = 0; + while (sqlite3_step(st) == SQLITE_ROW) { + const char *target = (const char *)sqlite3_column_text(st, 0); + float score = (float)sqlite3_column_double(st, 1); + double p = sqlite3_column_double(st, 2); + bool is_function = false; + double want = + doc_row_want((const char *)sqlite3_column_text(st, 3), fmt, score, &is_function); + *bad += want <= 0.0 || p != want || + (forbidden_target && target && strstr(target, forbidden_target)); + n++; + } + sqlite3_finalize(st); + return n; +} + +/* Candidates never repeat an exact link: a section whose code span names a + * function (a MENTIONS edge) does not also list that function as a + * candidate. And each format carries its own judged curve: reST stores only + * from tfidf 0.30, with reST's p. */ +TEST(pipeline_doc_candidates_skip_exact_links_and_use_format_curves) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_docfmt_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + static const char *const subjects[] = {"mail", "cache", "queue", "user", "server", "client"}; + for (size_t i = 0; i < sizeof(subjects) / sizeof(subjects[0]); i++) { + write_py_family(tmp, subjects[i]); + } + char docs[512]; + snprintf(docs, sizeof(docs), "%s/docs", tmp); + ASSERT_TRUE(cbm_mkdir_p(docs, 0755)); + write_temp_file(tmp, "docs/mail.md", + "# Mail settings\n\n" + "The mail config file is parsed by `mail.parse_mail_config`, which loads " + "the mail config from disk and validates every mail field before the " + "parsed mail settings are returned to the caller for use.\n"); + write_temp_file(tmp, "docs/mail.rst", + "Mail loading\n============\n\n" + "Load the mail config from disk and validate every mail field: missing " + "mail fields fail the validation, and the loaded mail config is written " + "back to disk after validating it.\n"); + char db[512]; + snprintf(db, sizeof(db), "%s/docfmt.db", tmp); + cbm_pipeline_incremental_test_reset_faults(); + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + cbm_pipeline_free(p); + + cbm_store_t *s = cbm_store_open_path(db); + ASSERT_NOT_NULL(s); + /* the exact link exists ... */ + sqlite3_stmt *st = NULL; + ASSERT_EQ(sqlite3_prepare_v2(cbm_store_get_db(s), + "SELECT count(*) FROM edges e JOIN nodes a ON a.id = e.source_id " + "JOIN nodes b ON b.id = e.target_id WHERE e.type = 'MENTIONS' " + "AND a.label = 'Section' AND a.file_path = 'docs/mail.md' " + "AND b.name = 'parse_mail_config'", + -1, &st, NULL), + SQLITE_OK); + ASSERT_EQ(sqlite3_step(st), SQLITE_ROW); + int linked = sqlite3_column_int(st, 0); + sqlite3_finalize(st); + /* ... and is not repeated as a candidate, while the rest of the section's + * candidates are stored on the Markdown curve */ + int md_bad = 0; + int rst_bad = 0; + int md = doc_rows_of(s, "Mail-settings", ".md", CBM_SEM_DOC_FMT_MARKDOWN, ".parse_mail_config", + &md_bad); + int rst = doc_rows_of(s, "Mail-loading", ".rst", CBM_SEM_DOC_FMT_RST, NULL, &rst_bad); + cbm_store_close(s); + th_rmtree(tmp); + cbm_pipeline_incremental_test_reset_faults(); + ASSERT_EQ(linked, 1); + ASSERT_GT(md, 0); + ASSERT_EQ(md_bad, 0); + ASSERT_GT(rst, 0); + ASSERT_EQ(rst_bad, 0); + PASS(); +} + +/* Lines of a CBM_SEM_DOC_SIGNALS dump whose section file, kind, position, + * home and target file equal the given values (NULL: any). */ +static int doc_signal_count(const char *dump, const char *sec_file, const char *kind, + const char *pos, const char *home, const char *target_file) { + FILE *f = cbm_fopen(dump, "r"); + if (!f) { + return -1; + } + char line[4096]; + int n = 0; + while (fgets(line, sizeof(line), f)) { + line[strcspn(line, "\r\n")] = '\0'; + char *col[12]; + int nc = 0; + for (char *p = line; nc < 12; nc++) { + col[nc] = p; + char *tab = strchr(p, '\t'); + if (!tab) { + nc++; + break; + } + *tab = '\0'; + p = tab + 1; + } + if (nc != 12) { + continue; + } + char *colon = strrchr(col[5], ':'); + if (colon) { + *colon = '\0'; + } + colon = strrchr(col[6], ':'); + if (colon) { + *colon = '\0'; + } + n += (!sec_file || strcmp(col[5], sec_file) == 0) && (!kind || strcmp(col[9], kind) == 0) && + (!pos || strcmp(col[10], pos) == 0) && (!home || strcmp(col[11], home) == 0) && + (!target_file || strcmp(col[6], target_file) == 0); + } + fclose(f); + return n; +} + +/* Document position: a doc's home is its folder (a docs folder documents its + * parent; a folder without code climbs to one with code; the root = the + * whole project); a candidate inside the home is local, elsewhere outside; + * a doc with a home below the root gets its home folder as a candidate; + * whole files are candidates; local functions below the top 5 are added. */ +TEST(pipeline_doc_candidates_home_position_and_kinds) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_docpos_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char dir[512]; + static const char *const pkgs[] = {"pkg/mail", "pkg/mail/sub", "pkg/cache", "pkg/queue", + "pkg/mail/docs", "docs", "guides"}; + for (size_t i = 0; i < sizeof(pkgs) / sizeof(pkgs[0]); i++) { + snprintf(dir, sizeof(dir), "%s/%s", tmp, pkgs[i]); + ASSERT_TRUE(cbm_mkdir_p(dir, 0755)); + } + snprintf(dir, sizeof(dir), "%s/pkg/mail", tmp); + write_py_family(dir, "mail"); + snprintf(dir, sizeof(dir), "%s/pkg/mail/sub", tmp); + write_py_family(dir, "mail"); + snprintf(dir, sizeof(dir), "%s/pkg/cache", tmp); + write_py_family(dir, "cache"); + snprintf(dir, sizeof(dir), "%s/pkg/queue", tmp); + write_py_family(dir, "queue"); + static const char body[] = + "The mail config file is parsed, loaded from disk and validated: every mail " + "field is checked, missing mail fields fail the validation, and the parsed mail " + "settings are written back to disk after validating the mail config.\n"; + static const char *const doc_files[] = {"pkg/mail/README.md", "pkg/mail/docs/guide.md", + "docs/overview.md", "guides/notes.md"}; + for (size_t i = 0; i < sizeof(doc_files) / sizeof(doc_files[0]); i++) { + char text[1024]; + snprintf(text, sizeof(text), "# Mail config %d\n\n%s", (int)i, body); + write_temp_file(tmp, doc_files[i], text); + } + char db[512]; + char dump[512]; + snprintf(db, sizeof(db), "%s/docpos.db", tmp); + snprintf(dump, sizeof(dump), "%s/signals.tsv", tmp); + cbm_setenv("CBM_SEM_DOC_SIGNALS", dump, 1); + cbm_pipeline_incremental_test_reset_faults(); + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + int rc = cbm_pipeline_run(p); + cbm_pipeline_free(p); + cbm_unsetenv("CBM_SEM_DOC_SIGNALS"); + ASSERT_EQ(rc, 0); + + const char *readme = "pkg/mail/README.md"; + const char *guide = "pkg/mail/docs/guide.md"; + /* homes: the package; its docs folder documents the package; top-level + * docs and a folder without code document the whole project */ + int readme_lines = doc_signal_count(dump, readme, NULL, NULL, NULL, NULL); + int readme_home = doc_signal_count(dump, readme, NULL, NULL, "pkg/mail", NULL); + int guide_lines = doc_signal_count(dump, guide, NULL, NULL, NULL, NULL); + int guide_home = doc_signal_count(dump, guide, NULL, NULL, "pkg/mail", NULL); + int overview_lines = doc_signal_count(dump, "docs/overview.md", NULL, NULL, NULL, NULL); + int overview_root = doc_signal_count(dump, "docs/overview.md", NULL, "global", ".", NULL); + int notes_lines = doc_signal_count(dump, "guides/notes.md", NULL, NULL, NULL, NULL); + int notes_root = doc_signal_count(dump, "guides/notes.md", NULL, "global", ".", NULL); + /* positions */ + int local_fn = + doc_signal_count(dump, readme, "function", "local", "pkg/mail", "pkg/mail/mail.py"); + int local_sub = + doc_signal_count(dump, readme, NULL, "local", "pkg/mail", "pkg/mail/sub/mail.py"); + int outside_fn = doc_signal_count(dump, readme, "function", "outside", NULL, NULL); + int wrong_local = doc_signal_count(dump, readme, NULL, "local", NULL, "pkg/cache/cache.py"); + /* kinds */ + int folders = doc_signal_count(dump, NULL, "folder", "local", NULL, "pkg/mail"); + int folder_readme = doc_signal_count(dump, readme, "folder", "local", "pkg/mail", "pkg/mail"); + int root_folders = doc_signal_count(dump, "docs/overview.md", "folder", NULL, NULL, NULL) + + doc_signal_count(dump, "guides/notes.md", "folder", NULL, NULL, NULL); + int files = doc_signal_count(dump, readme, "file", "local", NULL, NULL); + int extras = doc_signal_count(dump, readme, "local", "local", "pkg/mail", NULL); + /* stored: the README's home folder (a README: p 0.87), not the guide's + * (another doc), no function outside the home, every row on its curve */ + cbm_store_t *s = cbm_store_open_path(db); + ASSERT_NOT_NULL(s); + sqlite3_stmt *st = NULL; + ASSERT_EQ(sqlite3_prepare_v2(cbm_store_get_db(s), + "SELECT c.score, c.p, c.evidence, n.file_path FROM " + "doc_link_candidates c JOIN nodes n ON n.qualified_name = " + "c.section_qn", + -1, &st, NULL), + SQLITE_OK); + int stored_folders = 0; + int stored_outside = 0; + int off_curve = 0; + double folder_p = 0.0; + while (sqlite3_step(st) == SQLITE_ROW) { + const char *ev = (const char *)sqlite3_column_text(st, 2); + const char *path = (const char *)sqlite3_column_text(st, 3); + double p = sqlite3_column_double(st, 1); + if (ev && strstr(ev, "\"kind\":\"folder\"")) { + stored_folders++; + folder_p = p; + off_curve += strcmp(path, readme) != 0; + continue; + } + stored_outside += + ev && strstr(ev, "\"position\":\"outside\"") && strstr(ev, "\"kind\":\"function\""); + bool is_function = false; + off_curve += p != doc_row_want(ev, CBM_SEM_DOC_FMT_MARKDOWN, + (float)sqlite3_column_double(st, 0), &is_function); + } + sqlite3_finalize(st); + cbm_store_close(s); + th_rmtree(tmp); + cbm_pipeline_incremental_test_reset_faults(); + ASSERT_GT(readme_lines, 0); + ASSERT_EQ(readme_home, readme_lines); + ASSERT_GT(guide_lines, 0); + ASSERT_EQ(guide_home, guide_lines); + ASSERT_GT(overview_lines, 0); + ASSERT_EQ(overview_root, overview_lines); + ASSERT_GT(notes_lines, 0); + ASSERT_EQ(notes_root, notes_lines); + ASSERT_GT(local_fn, 0); + ASSERT_GT(local_sub, 0); + ASSERT_GT(outside_fn, 0); + ASSERT_EQ(wrong_local, 0); + ASSERT_EQ(folders, 2); /* README and the docs guide; one per doc, never per section */ + ASSERT_EQ(folder_readme, 1); + ASSERT_EQ(root_folders, 0); + ASSERT_GT(files, 0); + ASSERT_GT(extras, 0); + ASSERT_EQ(stored_folders, 1); + ASSERT_FLOAT_EQ(folder_p, 0.87, 1e-9); + ASSERT_EQ(stored_outside, 0); + ASSERT_EQ(off_curve, 0); + PASS(); +} + TEST(pipeline_semantic_edges_no_functions) { char tmp[256]; snprintf(tmp, sizeof(tmp), "/tmp/cbm_nofunc_XXXXXX"); @@ -19722,6 +20307,11 @@ SUITE(pipeline) { RUN_TEST(pipeline_markdown_and_config_prose_reaches_fts_body); RUN_TEST(pipeline_semantic_edges_no_functions); RUN_TEST(pipeline_semantic_batched_matches_unbatched); + RUN_TEST(pipeline_semantic_pair_signals_never_change_the_graph); + RUN_TEST(pipeline_semantic_edges_carry_p); + RUN_TEST(pipeline_doc_candidates_published_and_kept_by_delta); + RUN_TEST(pipeline_doc_candidates_skip_exact_links_and_use_format_curves); + RUN_TEST(pipeline_doc_candidates_home_position_and_kinds); RUN_TEST(pipeline_cpp_static_factory_pointer_receiver_issue1153); RUN_TEST(pipeline_cpp_static_factory_reference_receiver_issue1153); RUN_TEST(pipeline_cpp_static_factory_unique_ptr_receiver_issue1153); diff --git a/tests/test_semantic.c b/tests/test_semantic.c index e56c02c23..f1ebd09f7 100644 --- a/tests/test_semantic.c +++ b/tests/test_semantic.c @@ -355,6 +355,311 @@ TEST(sem_corpus_index_accessors_match_name_lookups) { PASS(); } +/* ── TF-IDF terms and the combined score ─────────────────────────── */ + +static int sem_terms_of(const cbm_sem_corpus_t *c, char **tokens, int n, cbm_sem_func_t *f, + int *indices, float *weights) { + memset(f, 0, sizeof(*f)); + f->tfidf_indices = indices; + f->tfidf_weights = weights; + f->tfidf_len = cbm_sem_tfidf_terms(c, tokens, NULL, n, indices, weights); + return f->tfidf_len; +} + +/* Two functions share a TF-IDF term only when they share the word. Until + * 2026-10 the term key was the token's POSITION: any two functions of equal + * length matched on every term, so the signal measured length (a held-out + * judged sample: admitted-edge precision 0.713 -> 0.742 with the fix). */ +TEST(sem_tfidf_terms_compare_vocabulary_not_positions) { + cbm_sem_corpus_t *c = cbm_sem_corpus_new(); + ASSERT_NOT_NULL(c); + static char *parse[] = {"parse", "header", "value"}; + static char *render[] = {"render", "widget", "frame"}; + static char *reparse[] = {"value", "parse", "header", "parse", "absent"}; + static char *other[] = {"queue", "drain"}; + cbm_sem_corpus_add_doc(c, (const char **)parse, 3); + cbm_sem_corpus_add_doc(c, (const char **)render, 3); + cbm_sem_corpus_add_doc(c, (const char **)reparse, 4); /* "absent" stays out */ + cbm_sem_corpus_add_doc(c, (const char **)other, 2); + cbm_sem_corpus_finalize(c); + + cbm_sem_func_t fa; + cbm_sem_func_t fb; + cbm_sem_func_t fc; + int ia[3]; + int ib[3]; + int ic[5]; + float wa[3]; + float wb[3]; + float wc[5]; + ASSERT_EQ(sem_terms_of(c, parse, 3, &fa, ia, wa), 3); + ASSERT_EQ(sem_terms_of(c, render, 3, &fb, ib, wb), 3); + /* A repeated word is one term, its weight twice the idf; an unknown word + * is no term; terms ascend by token index. */ + ASSERT_EQ(sem_terms_of(c, reparse, 5, &fc, ic, wc), 3); + ASSERT_TRUE(ic[0] < ic[1] && ic[1] < ic[2]); + int at_parse = ic[0] == cbm_sem_corpus_token_index(c, "parse") ? 0 + : ic[1] == cbm_sem_corpus_token_index(c, "parse") ? 1 + : 2; + ASSERT_EQ(ic[at_parse], cbm_sem_corpus_token_index(c, "parse")); + ASSERT_FLOAT_EQ(wc[at_parse], 2.0F * cbm_sem_corpus_idf(c, "parse"), 1e-6); + + cbm_sem_signals_t s; + cbm_sem_signal_values(&fa, &fb, &s); + ASSERT_FLOAT_EQ(s.tfidf, 0.0F, 1e-6); /* same length, no shared word */ + cbm_sem_signal_values(&fa, &fc, &s); + ASSERT_TRUE(s.tfidf > 0.9F); /* the same words, other order and length */ + cbm_sem_corpus_free(c); + PASS(); +} + +/* cbm_sem_combine is the weighted sum of the signals, times the proximity + * multiplier, clamped to [0, 1]; a SIMILAR_TO near copy scores 0. And + * cbm_sem_combined_score is exactly that, applied to cbm_sem_signal_values. */ +TEST(sem_combine_weighs_signals) { + cbm_sem_config_t cfg = cbm_sem_get_config(); + cbm_sem_signals_t s = {.tfidf = 0.5F, .ri = 0.25F, .proximity = 1.0F}; + ASSERT_FLOAT_EQ(cbm_sem_combine(&s, &cfg), cfg.w_tfidf * 0.5F + cfg.w_ri * 0.25F, 1e-6); + s.proximity = 1.10F; + ASSERT_FLOAT_EQ(cbm_sem_combine(&s, &cfg), (cfg.w_tfidf * 0.5F + cfg.w_ri * 0.25F) * 1.10F, + 1e-6); + cbm_sem_signals_t all = {.tfidf = 1.0F, + .ri = 1.0F, + .minhash = 1.0F, + .api = 1.0F, + .type = 1.0F, + .decorator = 1.0F, + .struct_profile = 1.0F, + .proximity = 1.10F}; + ASSERT_FLOAT_EQ(cbm_sem_combine(&all, &cfg), 1.0F, 0.0); /* clamped */ + all.near_copy = true; + ASSERT_FLOAT_EQ(cbm_sem_combine(&all, &cfg), 0.0F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_combine(NULL, &cfg), 0.0F, 0.0); + + cbm_sem_corpus_t *c = cbm_sem_corpus_new(); + ASSERT_NOT_NULL(c); + static char *one[] = {"open", "socket", "port"}; + static char *two[] = {"open", "socket", "host", "retry"}; + cbm_sem_corpus_add_doc(c, (const char **)one, 3); + cbm_sem_corpus_add_doc(c, (const char **)two, 4); + cbm_sem_corpus_finalize(c); + cbm_sem_func_t fa; + cbm_sem_func_t fb; + int ia[3]; + int ib[4]; + float wa[3]; + float wb[4]; + sem_terms_of(c, one, 3, &fa, ia, wa); + sem_terms_of(c, two, 4, &fb, ib, wb); + fa.file_path = "net/dial.go"; + fb.file_path = "net/listen.go"; + cbm_sem_signals_t v; + cbm_sem_signal_values(&fa, &fb, &v); + ASSERT_FLOAT_EQ(cbm_sem_combined_score(&fa, &fb, &cfg), cbm_sem_combine(&v, &cfg), 0.0); + cbm_sem_corpus_free(c); + PASS(); +} + +/* Over budget, a function keeps its BEST partners, not its first ones (a + * held-out judged sample: the pairs first-come admitted over budget were 0 of + * 17 related). Equal scores keep the canonical order; an ineligible pair is + * never admitted and takes no budget. */ +TEST(sem_admit_best_first_keeps_the_best) { + /* function 0 has four partners in canonical order, budget 2 */ + const float scores[] = {0.80F, 0.90F, 0.95F, 0.99F, 0.90F}; + const int fa[] = {0, 0, 0, 0, 5}; + const int fb[] = {1, 2, 3, 4, 6}; + const bool eligible[] = {true, true, true, false, true}; + int counts[7] = {0}; + bool admitted[5]; + ASSERT_TRUE(cbm_sem_admit_best_first(scores, fa, fb, eligible, 5, 2, counts, admitted)); + ASSERT_FALSE(admitted[0]); /* first-come would have kept 0.80 */ + ASSERT_TRUE(admitted[1]); + ASSERT_TRUE(admitted[2]); + ASSERT_FALSE(admitted[3]); /* the best score, but recorded only */ + ASSERT_TRUE(admitted[4]); + ASSERT_EQ(counts[0], 2); + ASSERT_EQ(counts[4], 0); + + /* ties: the earlier pair in canonical order wins the last slot */ + const float tied[] = {0.85F, 0.85F}; + const int ta[] = {0, 0}; + const int tb[] = {1, 2}; + const bool all[] = {true, true}; + int tcounts[3] = {0}; + bool tadmitted[2]; + ASSERT_TRUE(cbm_sem_admit_best_first(tied, ta, tb, all, 2, 1, tcounts, tadmitted)); + ASSERT_TRUE(tadmitted[0]); + ASSERT_FALSE(tadmitted[1]); + PASS(); +} + +/* p by the stored (three-decimal) score: the judged bands, monotone. */ +TEST(sem_calibrated_p_bands) { + ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.750F), 0.626F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.772F), 0.626F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.773F), 0.760F, 0.0); + /* the cut sits at the stored precision (0.773), not the raw quartile: + * a stored "0.773" and a stored "0.772" never share a band */ + ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.7729F), 0.626F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.838F), 0.760F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(1.000F), 0.760F, 0.0); + float last = 0.0F; + for (int milli = 750; milli <= 1000; milli++) { + float p = cbm_sem_calibrated_p((float)milli / 1000.0F); + ASSERT_TRUE(p >= last); + last = p; + } + PASS(); +} + +static float docp(cbm_sem_doc_format_t fmt, cbm_sem_doc_kind_t kind, cbm_sem_doc_pos_t pos, + float score) { + return cbm_sem_doc_calibrated_p(fmt, kind, pos, score); +} + +/* Doc -> code candidates, Markdown (sample 7): p by kind and position -- + * functions outside the doc's home never stored, inside it from 0.30, for a + * doc of the whole project (samples 4 and 7 pooled) from 0.30 too; local + * extras 0.32 inside the home + * only; whole files from 0.40; nothing below CBM_SEM_DOC_MIN_SCORE; every + * curve monotone. A format without a judged curve stores nothing. */ +TEST(sem_doc_calibrated_p_bands) { + const cbm_sem_doc_format_t md = CBM_SEM_DOC_FMT_MARKDOWN; + const cbm_sem_doc_kind_t fn = CBM_SEM_DOC_KIND_FUNCTION; + const cbm_sem_doc_pos_t global = CBM_SEM_DOC_POS_GLOBAL; + const cbm_sem_doc_pos_t local = CBM_SEM_DOC_POS_LOCAL; + const cbm_sem_doc_pos_t outside = CBM_SEM_DOC_POS_OUTSIDE; + ASSERT_FLOAT_EQ(docp(md, fn, local, 0.199F), 0.0F, 0.0); + ASSERT_FLOAT_EQ(docp(md, fn, local, CBM_SEM_DOC_MIN_SCORE), 0.0F, 0.0); + ASSERT_FLOAT_EQ(docp(md, fn, local, 0.30F), 0.325F, 0.0); + ASSERT_FLOAT_EQ(docp(md, fn, local, 0.40F), 0.525F, 0.0); + ASSERT_FLOAT_EQ(docp(md, fn, global, 0.299F), 0.0F, 0.0); + ASSERT_FLOAT_EQ(docp(md, fn, global, 0.30F), 0.217F, 0.0); + ASSERT_FLOAT_EQ(docp(md, fn, global, 0.40F), 0.509F, 0.0); + ASSERT_FLOAT_EQ(docp(md, fn, outside, 0.90F), 0.0F, 0.0); + ASSERT_FLOAT_EQ(docp(md, CBM_SEM_DOC_KIND_LOCAL, local, 0.199F), 0.0F, 0.0); + ASSERT_FLOAT_EQ(docp(md, CBM_SEM_DOC_KIND_LOCAL, local, 0.20F), 0.317F, 0.0); + ASSERT_FLOAT_EQ(docp(md, CBM_SEM_DOC_KIND_LOCAL, local, 0.90F), 0.317F, 0.0); + ASSERT_FLOAT_EQ(docp(md, CBM_SEM_DOC_KIND_FILE, outside, 0.399F), 0.0F, 0.0); + ASSERT_FLOAT_EQ(docp(md, CBM_SEM_DOC_KIND_FILE, outside, 0.40F), 0.30F, 0.0); + ASSERT_FLOAT_EQ(docp(md, CBM_SEM_DOC_KIND_FOLDER, local, 0.90F), 0.0F, 0.0); + static const cbm_sem_doc_kind_t KINDS[] = {CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_KIND_LOCAL, + CBM_SEM_DOC_KIND_FILE}; + for (size_t k = 0; k < sizeof(KINDS) / sizeof(KINDS[0]); k++) { + for (int pos = 0; pos < CBM_SEM_DOC_POS_COUNT; pos++) { + float last = 0.0F; + for (int milli = 0; milli <= 1000; milli++) { + float s = (float)milli / 1000.0F; + float p = docp(md, KINDS[k], (cbm_sem_doc_pos_t)pos, s); + ASSERT_TRUE(p >= last); + last = p; + ASSERT_TRUE(pos != CBM_SEM_DOC_POS_OUTSIDE || KINDS[k] == CBM_SEM_DOC_KIND_FILE || + p == 0.0F); + ASSERT_FLOAT_EQ(docp(CBM_SEM_DOC_FMT_OTHER, KINDS[k], (cbm_sem_doc_pos_t)pos, s), + 0.0F, 0.0); + ASSERT_FLOAT_EQ(docp(CBM_SEM_DOC_FMT_PDF, KINDS[k], (cbm_sem_doc_pos_t)pos, s), + 0.0F, 0.0); + } + } + } + ASSERT_FLOAT_EQ(docp(CBM_SEM_DOC_FMT_COUNT, fn, global, 0.9F), 0.0F, 0.0); + /* a home folder: README and index docs only, Markdown only */ + ASSERT_FLOAT_EQ(cbm_sem_doc_folder_p(md, "pkg/mail/README.md"), 0.867F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_doc_folder_p(md, "pkg/mail/docs/index.mdx"), 0.867F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_doc_folder_p(md, "pkg/mail/readme"), 0.867F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_doc_folder_p(md, "pkg/mail/guide.md"), 0.0F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_doc_folder_p(md, "pkg/mail/README-old.md"), 0.0F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_doc_folder_p(CBM_SEM_DOC_FMT_RST, "pkg/mail/index.rst"), 0.0F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_doc_folder_p(md, NULL), 0.0F, 0.0); + /* reST (sample 6): 0.20-0.30 judged 0.125, not stored */ + const cbm_sem_doc_format_t rst = CBM_SEM_DOC_FMT_RST; + ASSERT_FLOAT_EQ( + cbm_sem_doc_calibrated_p(rst, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, 0.25F), + 0.0F, 0.0); + ASSERT_FLOAT_EQ( + cbm_sem_doc_calibrated_p(rst, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, 0.30F), + 0.35F, 0.0); + ASSERT_FLOAT_EQ( + cbm_sem_doc_calibrated_p(rst, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, 0.40F), + 0.625F, 0.0); + /* AsciiDoc (sample 6): every band at or above the floor is stored */ + const cbm_sem_doc_format_t adoc = CBM_SEM_DOC_FMT_ADOC; + ASSERT_FLOAT_EQ( + cbm_sem_doc_calibrated_p(adoc, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, 0.199F), + 0.0F, 0.0); + ASSERT_FLOAT_EQ( + cbm_sem_doc_calibrated_p(adoc, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, 0.20F), + 0.225F, 0.0); + ASSERT_FLOAT_EQ( + cbm_sem_doc_calibrated_p(adoc, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, 0.35F), + 0.25F, 0.0); + ASSERT_FLOAT_EQ( + cbm_sem_doc_calibrated_p(adoc, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, 0.90F), + 0.50F, 0.0); + for (int milli = 0; milli <= 1000; milli++) { + float s = (float)milli / 1000.0F; + ASSERT_TRUE( + cbm_sem_doc_calibrated_p(rst, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, s) >= + cbm_sem_doc_calibrated_p(rst, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, + s - 0.001F)); + ASSERT_TRUE( + cbm_sem_doc_calibrated_p(adoc, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, s) >= + cbm_sem_doc_calibrated_p(adoc, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, + s - 0.001F)); + } + PASS(); +} + +/* p is stored and shown at two decimals, half up, whatever the float + * representation of the judged value (0.225F is 0.22499999...). */ +TEST(sem_p_two_decimals_half_up) { + ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.225F), 0.23, 1e-12); + ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.325F), 0.33, 1e-12); /* float 0.32499998... */ + ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.525F), 0.53, 1e-12); /* float 0.52499997... */ + ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.324F), 0.32, 1e-12); + ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.317F), 0.32, 1e-12); + ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.867F), 0.87, 1e-12); + ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.625F), 0.63, 1e-12); + ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.548F), 0.55, 1e-12); + ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.36F), 0.36, 1e-12); + ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.0F), 0.0, 1e-12); + PASS(); +} + +/* A section's format comes from its file's extension, case-insensitively; + * a .txt section is reStructuredText (plain text has no sections). */ +TEST(sem_doc_format_by_extension) { + ASSERT_EQ(cbm_sem_doc_format("docs/guide.md"), CBM_SEM_DOC_FMT_MARKDOWN); + ASSERT_EQ(cbm_sem_doc_format("README.MARKDOWN"), CBM_SEM_DOC_FMT_MARKDOWN); + ASSERT_EQ(cbm_sem_doc_format("docs/index.rst"), CBM_SEM_DOC_FMT_RST); + ASSERT_EQ(cbm_sem_doc_format("docs/topics/db.txt"), CBM_SEM_DOC_FMT_RST); + ASSERT_EQ(cbm_sem_doc_format("manual/intro.adoc"), CBM_SEM_DOC_FMT_ADOC); + ASSERT_EQ(cbm_sem_doc_format("paper.pdf"), CBM_SEM_DOC_FMT_PDF); + ASSERT_EQ(cbm_sem_doc_format("src/main.go"), CBM_SEM_DOC_FMT_OTHER); + ASSERT_EQ(cbm_sem_doc_format("v1.2/NOTES"), CBM_SEM_DOC_FMT_OTHER); + ASSERT_EQ(cbm_sem_doc_format(NULL), CBM_SEM_DOC_FMT_OTHER); + PASS(); +} + +/* A properties string holding a quote is read whole, and escapes are not + * glued to the next word (until 2026-10 the read stopped at the first quote + * and kept "\n" raw, so "\nreturn" became the token "nreturn"). */ +TEST(sem_json_str_honours_escapes) { + static const char json[] = + "{\"docstring\":\"Say \\\"hi\\\" to\\nthe user \\u00e9 now\",\"bt\":\"x\"}"; + char buf[64]; + ASSERT_NOT_NULL(cbm_sem_json_str(json, "docstring", buf, sizeof(buf))); + ASSERT_STR_EQ(buf, "Say \"hi\" to the user now"); + ASSERT_NOT_NULL(cbm_sem_json_str(json, "bt", buf, sizeof(buf))); + ASSERT_STR_EQ(buf, "x"); + ASSERT_NULL(cbm_sem_json_str(json, "absent", buf, sizeof(buf))); + char small[6]; + ASSERT_NOT_NULL(cbm_sem_json_str(json, "docstring", small, sizeof(small))); + ASSERT_STR_EQ(small, "Say \""); /* cut at the buffer, still terminated */ + PASS(); +} + TEST(sem_corpus_add_null_doc) { cbm_sem_corpus_t *c = cbm_sem_corpus_new(); ASSERT_NOT_NULL(c); @@ -531,6 +836,14 @@ SUITE(semantic) { RUN_TEST(sem_corpus_add_one_doc); RUN_TEST(sem_corpus_idf); RUN_TEST(sem_corpus_index_accessors_match_name_lookups); + RUN_TEST(sem_tfidf_terms_compare_vocabulary_not_positions); + RUN_TEST(sem_combine_weighs_signals); + RUN_TEST(sem_admit_best_first_keeps_the_best); + RUN_TEST(sem_calibrated_p_bands); + RUN_TEST(sem_doc_calibrated_p_bands); + RUN_TEST(sem_doc_format_by_extension); + RUN_TEST(sem_p_two_decimals_half_up); + RUN_TEST(sem_json_str_honours_escapes); RUN_TEST(sem_corpus_add_null_doc); RUN_TEST(sem_corpus_free_null); RUN_TEST(sem_get_config_defaults); diff --git a/tests/test_store_arch.c b/tests/test_store_arch.c index 4e9f59068..23ea5c935 100644 --- a/tests/test_store_arch.c +++ b/tests/test_store_arch.c @@ -275,7 +275,8 @@ TEST(arch_path_scoping) { ASSERT_TRUE(whole_pkg_nodes > scoped_pkg_nodes); ASSERT_EQ(scoped_pkg_nodes, 1); - ASSERT_TRUE(cbm_store_count_nodes(s, "pscope") > cbm_store_count_nodes_scoped(s, "pscope", "apps/foo")); + ASSERT_TRUE(cbm_store_count_nodes(s, "pscope") > + cbm_store_count_nodes_scoped(s, "pscope", "apps/foo")); cbm_architecture_info_t scoped_slash; memset(&scoped_slash, 0, sizeof(scoped_slash)); @@ -621,6 +622,145 @@ TEST(adr_delete_not_found) { PASS(); } +/* Doc -> code candidates: a database without the table reads as "none"; + * a replace swaps exactly one project's rows; reads go highest p first (then + * score) in both directions. */ +TEST(doc_candidates_replace_and_get) { + cbm_store_t *s = cbm_store_open_memory(); + ASSERT_NOT_NULL(s); + cbm_doc_candidate_t *got = NULL; + int n = -1; + ASSERT_EQ(cbm_store_doc_candidates_get(s, "p", "p.secA", NULL, 5, &got, &n), + CBM_STORE_NOT_FOUND); + + const cbm_doc_candidate_t rows[] = { + {.section_qn = "p.secA", + .target_qn = "p.f2", + .rank = 2, + .score = 0.25, + .p = 0.36, + .evidence = "{\"shared_terms\":3,\"name_tokens\":0}"}, + {.section_qn = "p.secA", + .target_qn = "p.f1", + .rank = 1, + .score = 0.45, + .p = 0.548, + .evidence = "{\"shared_terms\":6,\"name_tokens\":1}"}, + {.section_qn = "p.secB", + .target_qn = "p.f1", + .rank = 1, + .score = 0.32, + .p = 0.42, + .evidence = ""}, + }; + ASSERT_EQ(cbm_store_doc_candidates_replace(s, "p", rows, 3), CBM_STORE_OK); + const cbm_doc_candidate_t other = {.section_qn = "q.secA", + .target_qn = "q.f9", + .rank = 1, + .score = 0.5, + .p = 0.548, + .evidence = ""}; + ASSERT_EQ(cbm_store_doc_candidates_replace(s, "q", &other, 1), CBM_STORE_OK); + + ASSERT_EQ(cbm_store_doc_candidates_get(s, "p", "p.secA", NULL, 5, &got, &n), CBM_STORE_OK); + ASSERT_EQ(n, 2); + ASSERT_STR_EQ(got[0].target_qn, "p.f1"); + ASSERT_EQ(got[0].rank, 1); + ASSERT_STR_EQ(got[0].evidence, "{\"shared_terms\":6,\"name_tokens\":1}"); + ASSERT_STR_EQ(got[1].target_qn, "p.f2"); + cbm_store_doc_candidates_free(got, n); + + ASSERT_EQ(cbm_store_doc_candidates_get(s, "p", NULL, "p.f1", 5, &got, &n), CBM_STORE_OK); + ASSERT_EQ(n, 2); + ASSERT_STR_EQ(got[0].section_qn, "p.secA"); + ASSERT_STR_EQ(got[1].section_qn, "p.secB"); + cbm_store_doc_candidates_free(got, n); + + ASSERT_EQ(cbm_store_doc_candidates_get(s, "p", NULL, "p.f1", 1, &got, &n), CBM_STORE_OK); + ASSERT_EQ(n, 1); + cbm_store_doc_candidates_free(got, n); + + /* a replace drops every earlier row of the project, and only of it */ + ASSERT_EQ(cbm_store_doc_candidates_replace(s, "p", &rows[2], 1), CBM_STORE_OK); + ASSERT_EQ(cbm_store_doc_candidates_get(s, "p", "p.secA", NULL, 5, &got, &n), CBM_STORE_OK); + ASSERT_EQ(n, 0); + cbm_store_doc_candidates_free(got, n); + ASSERT_EQ(cbm_store_doc_candidates_get(s, "p", "p.secB", NULL, 5, &got, &n), CBM_STORE_OK); + ASSERT_EQ(n, 1); + cbm_store_doc_candidates_free(got, n); + ASSERT_EQ(cbm_store_doc_candidates_get(s, "q", "q.secA", NULL, 5, &got, &n), CBM_STORE_OK); + ASSERT_EQ(n, 1); + cbm_store_doc_candidates_free(got, n); + + ASSERT_EQ(cbm_store_doc_candidates_replace(s, "p", NULL, 0), CBM_STORE_OK); + ASSERT_EQ(cbm_store_doc_candidates_get(s, "p", NULL, "p.f1", 5, &got, &n), CBM_STORE_OK); + ASSERT_EQ(n, 0); + cbm_store_doc_candidates_free(got, n); + + /* exactly one of section / target */ + ASSERT_EQ(cbm_store_doc_candidates_get(s, "p", NULL, NULL, 5, &got, &n), CBM_STORE_ERR); + ASSERT_EQ(cbm_store_doc_candidates_get(s, "p", "p.secA", "p.f1", 5, &got, &n), CBM_STORE_ERR); + cbm_store_close(s); + PASS(); +} + +/* Whole-file candidates of a path: the rows whose target is the path's File + * node or a Folder holding it -- never a sibling folder sharing a name prefix + * (pkg/mailer, pk), a function in the file, or another project's rows. */ +TEST(doc_candidates_for_path_file_and_folders) { + cbm_store_t *s = cbm_store_open_memory(); + ASSERT_NOT_NULL(s); + cbm_doc_candidate_t *got = NULL; + int n = -1; + ASSERT_EQ(cbm_store_doc_candidates_for_path(s, "p", "pkg/mail/parse.py", 5, &got, &n), + CBM_STORE_NOT_FOUND); + cbm_store_upsert_project(s, "p", "/tmp/p"); + const struct { + const char *label, *qn, *path; + } nodes[] = {{"File", "p.pkg.mail.parse.__file__", "pkg/mail/parse.py"}, + {"Function", "p.pkg.mail.parse.parse_mail", "pkg/mail/parse.py"}, + {"Folder", "p.pkg.mail", "pkg/mail"}, + {"Folder", "p.pkg", "pkg"}, + {"Folder", "p.pkg.mailer", "pkg/mailer"}, + {"Folder", "p.pk", "pk"}}; + for (size_t i = 0; i < sizeof(nodes) / sizeof(nodes[0]); i++) { + cbm_node_t node = {.project = "p", + .label = nodes[i].label, + .name = nodes[i].qn, + .qualified_name = nodes[i].qn, + .file_path = nodes[i].path}; + ASSERT_TRUE(cbm_store_upsert_node(s, &node) > 0); + } + const cbm_doc_candidate_t rows[] = { + {.section_qn = "p.docs.a", .target_qn = "p.pkg.mail.parse.__file__", .p = 0.5}, + {.section_qn = "p.docs.b", .target_qn = "p.pkg.mail", .p = 0.7}, + {.section_qn = "p.docs.c", .target_qn = "p.pkg", .p = 0.3}, + {.section_qn = "p.docs.d", .target_qn = "p.pkg.mailer", .p = 0.9}, + {.section_qn = "p.docs.e", .target_qn = "p.pk", .p = 0.9}, + {.section_qn = "p.docs.f", .target_qn = "p.pkg.mail.parse.parse_mail", .p = 0.95}, + }; + ASSERT_EQ(cbm_store_doc_candidates_replace(s, "p", rows, 6), CBM_STORE_OK); + const cbm_doc_candidate_t other = { + .section_qn = "q.docs.a", .target_qn = "p.pkg.mail", .p = 0.99}; + ASSERT_EQ(cbm_store_doc_candidates_replace(s, "q", &other, 1), CBM_STORE_OK); + + ASSERT_EQ(cbm_store_doc_candidates_for_path(s, "p", "pkg/mail/parse.py", 5, &got, &n), + CBM_STORE_OK); + ASSERT_EQ(n, 3); + ASSERT_STR_EQ(got[0].section_qn, "p.docs.b"); /* highest p first */ + ASSERT_STR_EQ(got[1].section_qn, "p.docs.a"); + ASSERT_STR_EQ(got[2].section_qn, "p.docs.c"); + cbm_store_doc_candidates_free(got, n); + + ASSERT_EQ(cbm_store_doc_candidates_for_path(s, "p", "pkg/mail/parse.py", 1, &got, &n), + CBM_STORE_OK); + ASSERT_EQ(n, 1); + cbm_store_doc_candidates_free(got, n); + ASSERT_EQ(cbm_store_doc_candidates_for_path(s, "p", "", 5, &got, &n), CBM_STORE_ERR); + cbm_store_close(s); + PASS(); +} + TEST(adr_parse_sections_basic) { cbm_adr_sections_t sec = cbm_adr_parse_sections("## PURPOSE\nFoo\n\n## STACK\nBar"); ASSERT_EQ(sec.count, 2); @@ -969,11 +1109,11 @@ static int adr_check_splice(const char *in, const char *name, const char *body, return 0; } -#define CHECK_SPLICE(in, name, body, expect) \ - do { \ - if (adr_check_splice((in), (name), (body), (expect)) != 0) { \ - return 1; \ - } \ +#define CHECK_SPLICE(in, name, body, expect) \ + do { \ + if (adr_check_splice((in), (name), (body), (expect)) != 0) { \ + return 1; \ + } \ } while (0) /* THE acceptance property. Every document below is a case where rebuilding @@ -1088,9 +1228,9 @@ TEST(adr_splice_mixed_line_endings) { TEST(adr_splice_matches_headings_across_line_endings) { adr_name_collect_t c; memset(&c, 0, sizeof(c)); - ASSERT_EQ(cbm_adr_scan_headings("## PURPOSE\r\nFoo\r\n\r\n## STACK\r\nBar", - adr_collect_names, &c), - CBM_STORE_OK); + ASSERT_EQ( + cbm_adr_scan_headings("## PURPOSE\r\nFoo\r\n\r\n## STACK\r\nBar", adr_collect_names, &c), + CBM_STORE_OK); ASSERT_STR_EQ(c.buf, "[PURPOSE][STACK]"); /* A fenced block with CRLF still hides its heading, and still closes. */ @@ -1695,6 +1835,8 @@ SUITE(store_arch) { RUN_TEST(adr_upsert); RUN_TEST(adr_delete); RUN_TEST(adr_delete_not_found); + RUN_TEST(doc_candidates_replace_and_get); + RUN_TEST(doc_candidates_for_path_file_and_folders); RUN_TEST(adr_parse_sections_basic); RUN_TEST(adr_parse_sections_all_six); RUN_TEST(adr_parse_sections_non_canonical);