From 968d4a4be9f69a34a2654bbc6add8a4b3e50da4f Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Wed, 7 Oct 2026 18:05:49 +0200 Subject: [PATCH 1/7] fix(semantic): TF-IDF compares vocabulary, not token positions The TF-IDF signal keyed each term by the token's POSITION in the function, so any two functions of similar length shared their first terms whatever their words: 0.20 of every SEMANTICALLY_RELATED score measured length, not vocabulary. Terms are now keyed by corpus token (cbm_sem_tfidf_terms: one term per distinct word, weight term frequency x idf, ascending token index, which the sparse cosine merges on). Measured on a blind judged sample: 222 function pairs from six held-out repositories, stratified by score band, read in three rounds. The share of admitted SEMANTICALLY_RELATED edges a reader judges related rises from 0.713 to 0.742, with about 19% fewer edges (fastapi 507 -> 426, django 64 -> 66). The same sample shows that no score threshold reaches 0.90 precision, so the documented "~95% at 0.75" did not hold before this change either. The harness that measured it ships with the fix, off by default: CBM_SEM_PAIR_SIGNALS= writes one line per scored pair (both qualified names, the score, every signal value, whether the pair was admitted, both locations); CBM_SEM_PAIR_SIGNALS_FLOOR also records pairs below the threshold, never admitting them. cbm_sem_combined_score is now cbm_sem_signal_values + cbm_sem_combine (same arithmetic, same order). CBM_SEMANTIC_INDEX_VERSION 3 -> 4: indexes built before score pairs differently. Tests: sem_tfidf_terms_compare_vocabulary_not_positions, sem_combine_weighs_signals, pipeline_semantic_pair_signals_never_change_the_graph; each is RED with its behavior reverted. Signed-off-by: Martin Vogel --- src/pipeline/pass_semantic_edges.c | 95 ++++++++++++++++------ src/pipeline/pipeline_internal.h | 9 ++- src/semantic/semantic.c | 122 ++++++++++++++++++++--------- src/semantic/semantic.h | 31 ++++++++ tests/test_pipeline.c | 87 ++++++++++++++++++++ tests/test_semantic.c | 107 +++++++++++++++++++++++++ 6 files changed, 390 insertions(+), 61 deletions(-) diff --git a/src/pipeline/pass_semantic_edges.c b/src/pipeline/pass_semantic_edges.c index 14597289e..a0bcb636d 100644 --- a/src/pipeline/pass_semantic_edges.c +++ b/src/pipeline/pass_semantic_edges.c @@ -71,6 +71,7 @@ typedef struct { int64_t target_id; float score; bool same_file; + bool below; /* under the threshold: recorded for CBM_SEM_PAIR_SIGNALS only, never admitted */ /* Canonical admission keys (determinism): func index of the discovering * side, its candidate rank, and the partner's func index. The sequential * admission pass replays pairs in (i, c) order so which pairs win the @@ -93,7 +94,7 @@ static void deferred_buf_init(deferred_edge_buf_t *buf) { } static void deferred_buf_push(deferred_edge_buf_t *buf, int64_t src, int64_t tgt, float score, - bool same_file, int i, int j, int c) { + bool same_file, bool below, int i, int j, int c) { if (buf->count >= buf->cap) { int nc = buf->cap < CBM_SZ_256 ? CBM_SZ_256 : buf->cap * GROW; deferred_edge_t *grown = @@ -108,6 +109,7 @@ static void deferred_buf_push(deferred_edge_buf_t *buf, int64_t src, int64_t tgt .target_id = tgt, .score = score, .same_file = same_file, + .below = below, .i = i, .j = j, .c = c}; @@ -724,19 +726,10 @@ static void vec_build_worker(int worker_id, void *ctx_ptr) { token_index[t] = cbm_sem_corpus_token_index(vc->corpus, tokens[t]); } - /* TF-IDF weights */ + /* TF-IDF terms, keyed by corpus token (cbm_sem_tfidf_terms). */ int *indices = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)tc * sizeof(int)); float *weights = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)tc * sizeof(float)); - int tfidf_len = 0; - for (int t = 0; t < tc; t++) { - float idf = token_index ? cbm_sem_corpus_idf_at(vc->corpus, token_index[t]) - : cbm_sem_corpus_idf(vc->corpus, tokens[t]); - if (idf > 0.0F) { - indices[tfidf_len] = t; - weights[tfidf_len] = idf; - tfidf_len++; - } - } + int tfidf_len = cbm_sem_tfidf_terms(vc->corpus, tokens, token_index, tc, indices, weights); vc->funcs[f].tfidf_indices = indices; vc->funcs[f].tfidf_weights = weights; vc->funcs[f].tfidf_len = tfidf_len; @@ -833,6 +826,7 @@ typedef struct { uint64_t *signatures; int *edge_counts; /* budget applied sequentially in phase6b (determinism) */ cbm_sem_config_t cfg; + float record_floor; /* kept from here up; under the threshold: signals dump only */ int func_count; /* LSH buckets (read-only during scoring) */ @@ -927,13 +921,13 @@ static void score_try_emit(score_ctx_t *sc, int i, int j, int c, deferred_edge_b return; } float score = cbm_sem_combined_score(&sc->funcs[i], &sc->funcs[j], &sc->cfg); - if (score < sc->cfg.threshold) { + if (score < sc->record_floor) { return; } bool same_file = sc->funcs[i].file_path && sc->funcs[j].file_path && strcmp(sc->funcs[i].file_path, sc->funcs[j].file_path) == 0; - deferred_buf_push(my_buf, sc->funcs[i].node_id, sc->funcs[j].node_id, score, same_file, i, j, - c); + deferred_buf_push(my_buf, sc->funcs[i].node_id, sc->funcs[j].node_id, score, same_file, + score < sc->cfg.threshold, i, j, c); } static void score_worker(int worker_id, void *ctx_ptr) { @@ -1138,8 +1132,31 @@ static int cmp_deferred_edge_canonical(const void *pa, const void *pb) { * and apply the per-node max_edges budget HERE — single-threaded — so the * admitted edge set is a pure function of the (canonically sorted) inputs, * independent of worker count and scheduling. */ +/* CBM_SEM_PAIR_SIGNALS=: one line per recorded pair (from the record + * floor up, CBM_SEM_PAIR_SIGNALS_FLOOR, default the threshold), in canonical + * order: both qualified names, the score, every signal value and whether the + * pair was admitted. The measuring harness for the semantic engine: judged + * samples are drawn from it, stratified by score. Off unless set. */ +static void pair_signals_line(FILE *f, const cbm_gbuf_t *gbuf, const cbm_sem_func_t *funcs, + const deferred_edge_t *de, bool admitted) { + cbm_sem_signals_t s; + cbm_sem_signal_values(&funcs[de->i], &funcs[de->j], &s); + const cbm_gbuf_node_t *a = cbm_gbuf_find_by_id(gbuf, de->source_id); + const cbm_gbuf_node_t *b = cbm_gbuf_find_by_id(gbuf, de->target_id); + (void)fprintf( + f, + "%s\t%s\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.4f\t%d\t%d" + "\t%s:%d-%d\t%s:%d-%d\n", + a && a->qualified_name ? a->qualified_name : "?", + b && b->qualified_name ? b->qualified_name : "?", (double)de->score, (double)s.tfidf, + (double)s.ri, (double)s.minhash, (double)s.api, (double)s.type, (double)s.decorator, + (double)s.struct_profile, (double)s.proximity, de->same_file ? 1 : 0, admitted ? 1 : 0, + a && a->file_path ? a->file_path : "?", a ? a->start_line : 0, a ? a->end_line : 0, + b && b->file_path ? b->file_path : "?", b ? b->start_line : 0, b ? b->end_line : 0); +} + static int phase6b_merge_edges(cbm_gbuf_t *gbuf, deferred_edge_buf_t *worker_bufs, int worker_count, - int *edge_counts, int max_edges) { + int *edge_counts, int max_edges, const cbm_sem_func_t *funcs) { int total_pairs = 0; for (int w = 0; w < worker_count; w++) { total_pairs += worker_bufs[w].count; @@ -1175,20 +1192,34 @@ static int phase6b_merge_edges(cbm_gbuf_t *gbuf, deferred_edge_buf_t *worker_buf * chase the near-threshold SEMANTICALLY_RELATED flicker: find the pair that * flips, then compare its score between runs. */ FILE *pair_dump = NULL; + FILE *signals = NULL; { char dump_path[CBM_SZ_1K]; if (cbm_safe_getenv("CBM_SEM_PAIR_DUMP", dump_path, sizeof(dump_path), NULL)) { pair_dump = cbm_fopen(dump_path, "w"); } + if (funcs && cbm_safe_getenv("CBM_SEM_PAIR_SIGNALS", dump_path, sizeof(dump_path), NULL)) { + signals = cbm_fopen(dump_path, "w"); + } } int total_edges = 0; for (int e = 0; e < n; e++) { deferred_edge_t *de = &pairs[e]; + if (de->below) { + if (signals) { + pair_signals_line(signals, gbuf, funcs, de, false); + } + continue; + } if (pair_dump) { fprintf(pair_dump, "%d %d %.9g\n", de->i, de->j, (double)de->score); } - if (edge_counts[de->i] >= max_edges || edge_counts[de->j] >= max_edges) { + bool admitted = edge_counts[de->i] < max_edges && edge_counts[de->j] < max_edges; + if (signals) { + pair_signals_line(signals, gbuf, funcs, de, admitted); + } + if (!admitted) { continue; } char props[PROPS_BUF]; @@ -1202,6 +1233,9 @@ static int phase6b_merge_edges(cbm_gbuf_t *gbuf, deferred_edge_buf_t *worker_buf if (pair_dump) { (void)fclose(pair_dump); } + if (signals) { + (void)fclose(signals); + } cbm_free(CBM_MEM_CLASS_SEMANTIC, pairs); return total_edges; } @@ -1542,13 +1576,14 @@ static void sem_corpus_dump(const cbm_sem_corpus_t *corpus, const char *suffix) /* Phase 6a: score candidate pairs in parallel and collect deferred edges. */ static void phase6a_score_candidates(cbm_sem_func_t *funcs, uint64_t *signatures, int *edge_counts, sem_bucket_t **band_buckets, cbm_sem_config_t cfg, - deferred_edge_buf_t *worker_bufs, int func_count, - int worker_count) { + float record_floor, deferred_edge_buf_t *worker_bufs, + int func_count, int worker_count) { score_ctx_t sc = { .funcs = funcs, .signatures = signatures, .edge_counts = edge_counts, .cfg = cfg, + .record_floor = record_floor, .func_count = func_count, .band_buckets = (void *)band_buckets, .worker_bufs = worker_bufs, @@ -1619,14 +1654,30 @@ static int run_scoring_phase(cbm_gbuf_t *gbuf, cbm_sem_func_t *funcs, uint64_t * deferred_buf_init(&worker_bufs[w]); } + /* The measuring harness may record pairs below the threshold (never + * admitted): CBM_SEM_PAIR_SIGNALS_FLOOR, only with CBM_SEM_PAIR_SIGNALS. */ + float record_floor = cfg.threshold; + { + char path[CBM_SZ_1K]; /* the dump path: a buffer too short reads as unset */ + char buf[CBM_SZ_64]; + if (cbm_safe_getenv("CBM_SEM_PAIR_SIGNALS", path, sizeof(path), NULL) && + cbm_safe_getenv("CBM_SEM_PAIR_SIGNALS_FLOOR", buf, sizeof(buf), NULL)) { + float floor_value = strtof(buf, NULL); + if (floor_value > 0.0F && floor_value < cfg.threshold) { + record_floor = floor_value; + } + } + } + CBM_PROF_START(t_phase6a); sem_state_dump(funcs, func_count); - phase6a_score_candidates(funcs, signatures, edge_counts, band_buckets, cfg, worker_bufs, - func_count, worker_count); + phase6a_score_candidates(funcs, signatures, edge_counts, band_buckets, cfg, record_floor, + worker_bufs, func_count, worker_count); CBM_PROF_END_N("semantic_edges", "6a_score_parallel", t_phase6a, func_count); CBM_PROF_START(t_phase6b); - int total = phase6b_merge_edges(gbuf, worker_bufs, worker_count, edge_counts, cfg.max_edges); + int total = + phase6b_merge_edges(gbuf, worker_bufs, worker_count, edge_counts, cfg.max_edges, funcs); CBM_PROF_END_N("semantic_edges", "6b_edge_merge_seq", t_phase6b, total); cbm_free(CBM_MEM_CLASS_SEMANTIC, worker_bufs); diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 87b9c160b..b00dcd699 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -1274,8 +1274,13 @@ int cbm_pipeline_build_fresh_semantic_manifest(cbm_pipeline_t *p, const char *pr * sections and PDF pages hold MENTIONS edges into the code, ADR records * are nodes, and AsciiDoc files, PDFs and a Sphinx project's extra * source suffixes are indexed. An index built before has none of this - * for its unchanged documents, so it rebuilds once on upgrade. */ -enum { CBM_SEMANTIC_INDEX_VERSION = 6 }; + * for its unchanged documents, so it rebuilds once on upgrade. + * 7: SEMANTICALLY_RELATED changed: TF-IDF compares vocabulary, string + * properties are read with their escapes, admission is best-first and + * every edge carries "p"; doc_link_candidates holds doc section -> code + * candidates. An index built before has the old pairs and no + * candidates, so it rebuilds once on upgrade. */ +enum { CBM_SEMANTIC_INDEX_VERSION = 7 }; typedef struct { cbm_gbuf_t *gbuf; diff --git a/src/semantic/semantic.c b/src/semantic/semantic.c index 2aa1df5a3..c28615d93 100644 --- a/src/semantic/semantic.c +++ b/src/semantic/semantic.c @@ -1619,6 +1619,47 @@ static float small_cosine(const float *a, const float *b, int dims) { return denom < CBM_SEM_DENOM_EPS ? 0.0F : dot / denom; } +static int cmp_int_asc(const void *pa, const void *pb) { + int a = *(const int *)pa; + int b = *(const int *)pb; + return (a > b) - (a < b); +} + +/* Terms are keyed by corpus token, so two functions share a term only when + * they share the word. (Until 2026-10 the key was the token's POSITION in the + * function: any two functions of similar length "shared" their first terms, + * and the TF-IDF signal measured length, not vocabulary.) */ +int cbm_sem_tfidf_terms(const cbm_sem_corpus_t *corpus, char *const *tokens, const int *token_index, + int count, int *indices, float *weights) { + if (!corpus || !indices || !weights || count <= 0) { + return 0; + } + int known = 0; + for (int t = 0; t < count; t++) { + int idx = token_index ? token_index[t] : cbm_sem_corpus_token_index(corpus, tokens[t]); + if (idx >= 0 && cbm_sem_corpus_idf_at(corpus, idx) > 0.0F) { + indices[known++] = idx; + } + } + if (known > 1) { + qsort(indices, (size_t)known, sizeof(indices[0]), cmp_int_asc); + } + /* Each run of one index becomes one term; its weight is the idf added once + * per occurrence (equal indices have equal idf). */ + int terms = 0; + for (int k = 0; k < known; k++) { + float idf = cbm_sem_corpus_idf_at(corpus, indices[k]); + if (terms > 0 && indices[terms - SKIP_ONE] == indices[k]) { + weights[terms - SKIP_ONE] += idf; + } else { + indices[terms] = indices[k]; + weights[terms] = idf; + terms++; + } + } + return terms; +} + /* Sparse cosine over two pre-sorted (index, weight) vectors. Returns 0 when * either side is empty or the magnitude product is below the epsilon guard. */ static float sparse_tfidf_cosine(const cbm_sem_func_t *a, const cbm_sem_func_t *b) { @@ -1651,49 +1692,47 @@ static float sparse_tfidf_cosine(const cbm_sem_func_t *a, const cbm_sem_func_t * return denom > CBM_SEM_DENOM_EPS ? (dot / denom) : 0.0F; } -float cbm_sem_combined_score(const cbm_sem_func_t *a, const cbm_sem_func_t *b, - const cbm_sem_config_t *cfg) { - if (!a || !b || !cfg) { - return 0.0F; +void cbm_sem_signal_values(const cbm_sem_func_t *a, const cbm_sem_func_t *b, + cbm_sem_signals_t *out) { + memset(out, 0, sizeof(*out)); + out->proximity = CBM_SEM_UNIT_POS; + if (!a || !b) { + return; } + if (a->has_minhash && b->has_minhash) { + double j = cbm_minhash_jaccard((const cbm_minhash_t *)a->minhash, + (const cbm_minhash_t *)b->minhash); + out->minhash = (float)j; + out->near_copy = j >= CBM_MINHASH_JACCARD_THRESHOLD; + } + out->tfidf = sparse_tfidf_cosine(a, b); + out->ri = cbm_rsq_ip(&a->ri_code, &b->ri_code); /* signal 2: Random Indexing */ + out->api = cbm_rsq_ip(&a->api_code, &b->api_code); /* signal 4: API signatures */ + out->type = cbm_rsq_ip(&a->type_code, &b->type_code); /* signal 5: type signatures */ + out->decorator = cbm_rsq_ip(&a->deco_code, &b->deco_code); /* signal 7: decorators */ + /* signals 8+9+11: structural profile + data flow + Halstead */ + out->struct_profile = + small_cosine(a->struct_profile, b->struct_profile, CBM_SEM_AST_PROFILE_DIMS); + out->proximity = cbm_sem_proximity(a->file_path, b->file_path); /* signal 6: multiplier */ +} +float cbm_sem_combine(const cbm_sem_signals_t *s, const cbm_sem_config_t *cfg) { /* Short-circuit: if MinHash Jaccard is already above the SIMILAR_TO threshold, * the pass_similarity pipeline already emitted a SIMILAR_TO edge for this pair. * Returning 0 here avoids flooding top-k with cross-service copy-paste boilerplate * (logging_middleware, shared push/pull handlers) that SIMILAR_TO already covers, * freeing the edge budget for true semantic leaps (vocabulary-bridged relations). */ - if (a->has_minhash && b->has_minhash) { - double early_j = cbm_minhash_jaccard((const cbm_minhash_t *)a->minhash, - (const cbm_minhash_t *)b->minhash); - if (early_j >= CBM_MINHASH_JACCARD_THRESHOLD) { - return 0.0F; - } - } - - float score = cfg->w_tfidf * sparse_tfidf_cosine(a, b); - - /* Signal 2: Random Indexing */ - score += cfg->w_ri * cbm_rsq_ip(&a->ri_code, &b->ri_code); - - /* Signal 3: MinHash Jaccard */ - if (a->has_minhash && b->has_minhash) { - double j = cbm_minhash_jaccard((const cbm_minhash_t *)a->minhash, - (const cbm_minhash_t *)b->minhash); - score += cfg->w_minhash * (float)j; + if (!s || !cfg || s->near_copy) { + return 0.0F; } - - /* Signal 4: API Signatures */ - score += cfg->w_api * cbm_rsq_ip(&a->api_code, &b->api_code); - - /* Signal 5: Type Signatures */ - score += cfg->w_type * cbm_rsq_ip(&a->type_code, &b->type_code); - - /* Signal 7: Decorator Pattern */ - score += cfg->w_decorator * cbm_rsq_ip(&a->deco_code, &b->deco_code); - - /* Signal 8+9+11: Structural profile + data flow + Halstead */ - float sp_score = small_cosine(a->struct_profile, b->struct_profile, CBM_SEM_AST_PROFILE_DIMS); - score += cfg->w_struct_profile * sp_score; + /* The terms in the historical order (float addition is not associative). */ + float score = cfg->w_tfidf * s->tfidf; + score += cfg->w_ri * s->ri; + score += cfg->w_minhash * s->minhash; + score += cfg->w_api * s->api; + score += cfg->w_type * s->type; + score += cfg->w_decorator * s->decorator; + score += cfg->w_struct_profile * s->struct_profile; /* Signal 6: Module proximity (multiplier, not additive). * Proximity returns [1.0, 1.10] — a same-file/same-dir boost for ranking. @@ -1701,17 +1740,26 @@ float cbm_sem_combined_score(const cbm_sem_func_t *a, const cbm_sem_func_t *b, * cosine-similarity range. Without the clamp, a 0.95 base × 1.10 proximity * would emit a 1.045 score which violates the semantic of "similarity" and * breaks downstream consumers that expect a normalized value. */ - score *= cbm_sem_proximity(a->file_path, b->file_path); + score *= s->proximity; if (score > CBM_SEM_UNIT_POS) { score = CBM_SEM_UNIT_POS; } if (score < 0.0F) { score = 0.0F; } - return score; } +float cbm_sem_combined_score(const cbm_sem_func_t *a, const cbm_sem_func_t *b, + const cbm_sem_config_t *cfg) { + if (!a || !b || !cfg) { + return 0.0F; + } + cbm_sem_signals_t s; + cbm_sem_signal_values(a, b, &s); + return cbm_sem_combine(&s, cfg); +} + /* ── Graph diffusion ─────────────────────────────────────────────── */ void cbm_sem_diffuse(cbm_sem_vec_t *combined, const cbm_sem_vec_t *neighbors, int neighbor_count, diff --git a/src/semantic/semantic.h b/src/semantic/semantic.h index 1f99a54c0..34d738796 100644 --- a/src/semantic/semantic.h +++ b/src/semantic/semantic.h @@ -206,12 +206,43 @@ const char *cbm_sem_corpus_token_at(const cbm_sem_corpus_t *corpus, int index, /* Free corpus. */ void cbm_sem_corpus_free(cbm_sem_corpus_t *corpus); +/* The TF-IDF terms of one function: one entry per distinct corpus token whose + * idf is above 0, weighted term frequency x idf, in ascending token-index + * order (the order the TF-IDF cosine merges on). `token_index[t]` is the + * corpus index of tokens[t] (-1 when unknown); NULL looks each token up by + * name. `indices` and `weights` hold at least `count` entries. Returns the + * number of terms written. */ +int cbm_sem_tfidf_terms(const cbm_sem_corpus_t *corpus, char *const *tokens, const int *token_index, + int count, int *indices, float *weights); + /* ── Combined scoring ────────────────────────────────────────────── */ /* Compute combined similarity score between two functions. */ float cbm_sem_combined_score(const cbm_sem_func_t *a, const cbm_sem_func_t *b, const cbm_sem_config_t *cfg); +/* The signal values cbm_sem_combined_score weighs, for measuring them one by + * one: each similarity in [0, 1] (minhash 0 when either side has none), + * proximity a multiplier in [1.0, 1.10]. `near_copy`: the pair is a SIMILAR_TO + * copy, which the combined score sets to 0. */ +typedef struct { + float tfidf; + float ri; + float minhash; + float api; + float type; + float decorator; + float struct_profile; + float proximity; + bool near_copy; +} cbm_sem_signals_t; + +void cbm_sem_signal_values(const cbm_sem_func_t *a, const cbm_sem_func_t *b, + cbm_sem_signals_t *out); + +/* cbm_sem_combined_score from the signal values (the same arithmetic). */ +float cbm_sem_combine(const cbm_sem_signals_t *s, const cbm_sem_config_t *cfg); + /* Module proximity multiplier based on file paths. */ float cbm_sem_proximity(const char *path_a, const char *path_b); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index a2861d054..ed7cb7099 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -18810,6 +18810,92 @@ TEST(pipeline_semantic_batched_matches_unbatched) { PASS(); } +/* CBM_SEM_PAIR_SIGNALS (+ _FLOOR) is the semantic engine's measuring + * harness: it records each scored pair with its signal values, from a floor + * below the threshold up. Recording must never change the graph: the same + * SEMANTICALLY_RELATED rows with and without it, one admitted line per edge, + * and every pair under the threshold recorded but never admitted. */ +TEST(pipeline_semantic_pair_signals_never_change_the_graph) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_semsig_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + static const char *const subjects[] = {"User", "Server", "Client", "Cache", "Queue", "Mail"}; + for (size_t i = 0; i < sizeof(subjects) / sizeof(subjects[0]); i++) { + char file[64]; + snprintf(file, sizeof(file), "%s_config.go", subjects[i]); + write_sem_family(tmp, file, subjects[i]); + } + char db_plain[512]; + char db_signals[512]; + char tsv[512]; + snprintf(db_plain, sizeof(db_plain), "%s/plain.db", tmp); + snprintf(db_signals, sizeof(db_signals), "%s/signals.db", tmp); + snprintf(tsv, sizeof(tsv), "%s/pairs.tsv", tmp); + + cbm_setenv("CBM_SEMANTIC_THRESHOLD", "0.3", 1); /* as above: admits the near-duplicates */ + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_plain, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + cbm_pipeline_free(p); + + cbm_setenv("CBM_SEM_PAIR_SIGNALS", tsv, 1); + cbm_setenv("CBM_SEM_PAIR_SIGNALS_FLOOR", "0.05", 1); + p = cbm_pipeline_new(tmp, db_signals, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + int rc = cbm_pipeline_run(p); + cbm_unsetenv("CBM_SEM_PAIR_SIGNALS"); + cbm_unsetenv("CBM_SEM_PAIR_SIGNALS_FLOOR"); + cbm_unsetenv("CBM_SEMANTIC_THRESHOLD"); + cbm_pipeline_free(p); + ASSERT_EQ(rc, 0); + + cbm_store_t *sp = cbm_store_open_path(db_plain); + cbm_store_t *ss = cbm_store_open_path(db_signals); + ASSERT_NOT_NULL(sp); + ASSERT_NOT_NULL(ss); + int edges = 0; + ASSERT_TRUE( + sem_same_rows(cbm_store_get_db(sp), cbm_store_get_db(ss), + "SELECT s.qualified_name, t.qualified_name, e.properties FROM edges e " + "JOIN nodes s ON s.id = e.source_id JOIN nodes t ON t.id = e.target_id " + "WHERE e.type = 'SEMANTICALLY_RELATED' ORDER BY 1, 2", + &edges)); + ASSERT_GT(edges, 0); + cbm_store_close(sp); + cbm_store_close(ss); + + /* qa qb score tfidf ri minhash api type decorator struct proximity + * same_file admitted loc_a loc_b */ + FILE *f = cbm_fopen(tsv, "r"); + ASSERT_NOT_NULL(f); + char line[4096]; + int lines = 0; + int admitted = 0; + int below = 0; + while (fgets(line, sizeof(line), f)) { + char *field[16]; + int n = 0; + for (char *tok = strtok(line, "\t\n"); tok && n < 16; tok = strtok(NULL, "\t\n")) { + field[n++] = tok; + } + ASSERT_EQ(n, 15); + float score = strtof(field[2], NULL); + bool is_admitted = strcmp(field[12], "1") == 0; + if (score < 0.3F) { + below++; + ASSERT_FALSE(is_admitted); + } + admitted += is_admitted ? 1 : 0; + lines++; + } + fclose(f); + ASSERT_EQ(admitted, edges); + ASSERT_GT(below, 0); /* the floor recorded pairs under the threshold */ + ASSERT_GT(lines, admitted); + th_rmtree(tmp); + PASS(); +} + TEST(pipeline_semantic_edges_no_functions) { char tmp[256]; snprintf(tmp, sizeof(tmp), "/tmp/cbm_nofunc_XXXXXX"); @@ -19722,6 +19808,7 @@ SUITE(pipeline) { RUN_TEST(pipeline_markdown_and_config_prose_reaches_fts_body); RUN_TEST(pipeline_semantic_edges_no_functions); RUN_TEST(pipeline_semantic_batched_matches_unbatched); + RUN_TEST(pipeline_semantic_pair_signals_never_change_the_graph); RUN_TEST(pipeline_cpp_static_factory_pointer_receiver_issue1153); RUN_TEST(pipeline_cpp_static_factory_reference_receiver_issue1153); RUN_TEST(pipeline_cpp_static_factory_unique_ptr_receiver_issue1153); diff --git a/tests/test_semantic.c b/tests/test_semantic.c index e56c02c23..87bc4dcec 100644 --- a/tests/test_semantic.c +++ b/tests/test_semantic.c @@ -355,6 +355,111 @@ TEST(sem_corpus_index_accessors_match_name_lookups) { PASS(); } +/* ── TF-IDF terms and the combined score ─────────────────────────── */ + +static int sem_terms_of(const cbm_sem_corpus_t *c, char **tokens, int n, cbm_sem_func_t *f, + int *indices, float *weights) { + memset(f, 0, sizeof(*f)); + f->tfidf_indices = indices; + f->tfidf_weights = weights; + f->tfidf_len = cbm_sem_tfidf_terms(c, tokens, NULL, n, indices, weights); + return f->tfidf_len; +} + +/* Two functions share a TF-IDF term only when they share the word. Until + * 2026-10 the term key was the token's POSITION: any two functions of equal + * length matched on every term, so the signal measured length (a held-out + * judged sample: admitted-edge precision 0.713 -> 0.742 with the fix). */ +TEST(sem_tfidf_terms_compare_vocabulary_not_positions) { + cbm_sem_corpus_t *c = cbm_sem_corpus_new(); + ASSERT_NOT_NULL(c); + static char *parse[] = {"parse", "header", "value"}; + static char *render[] = {"render", "widget", "frame"}; + static char *reparse[] = {"value", "parse", "header", "parse", "absent"}; + static char *other[] = {"queue", "drain"}; + cbm_sem_corpus_add_doc(c, (const char **)parse, 3); + cbm_sem_corpus_add_doc(c, (const char **)render, 3); + cbm_sem_corpus_add_doc(c, (const char **)reparse, 4); /* "absent" stays out */ + cbm_sem_corpus_add_doc(c, (const char **)other, 2); + cbm_sem_corpus_finalize(c); + + cbm_sem_func_t fa; + cbm_sem_func_t fb; + cbm_sem_func_t fc; + int ia[3]; + int ib[3]; + int ic[5]; + float wa[3]; + float wb[3]; + float wc[5]; + ASSERT_EQ(sem_terms_of(c, parse, 3, &fa, ia, wa), 3); + ASSERT_EQ(sem_terms_of(c, render, 3, &fb, ib, wb), 3); + /* A repeated word is one term, its weight twice the idf; an unknown word + * is no term; terms ascend by token index. */ + ASSERT_EQ(sem_terms_of(c, reparse, 5, &fc, ic, wc), 3); + ASSERT_TRUE(ic[0] < ic[1] && ic[1] < ic[2]); + int at_parse = ic[0] == cbm_sem_corpus_token_index(c, "parse") ? 0 + : ic[1] == cbm_sem_corpus_token_index(c, "parse") ? 1 + : 2; + ASSERT_EQ(ic[at_parse], cbm_sem_corpus_token_index(c, "parse")); + ASSERT_FLOAT_EQ(wc[at_parse], 2.0F * cbm_sem_corpus_idf(c, "parse"), 1e-6); + + cbm_sem_signals_t s; + cbm_sem_signal_values(&fa, &fb, &s); + ASSERT_FLOAT_EQ(s.tfidf, 0.0F, 1e-6); /* same length, no shared word */ + cbm_sem_signal_values(&fa, &fc, &s); + ASSERT_TRUE(s.tfidf > 0.9F); /* the same words, other order and length */ + cbm_sem_corpus_free(c); + PASS(); +} + +/* cbm_sem_combine is the weighted sum of the signals, times the proximity + * multiplier, clamped to [0, 1]; a SIMILAR_TO near copy scores 0. And + * cbm_sem_combined_score is exactly that, applied to cbm_sem_signal_values. */ +TEST(sem_combine_weighs_signals) { + cbm_sem_config_t cfg = cbm_sem_get_config(); + cbm_sem_signals_t s = {.tfidf = 0.5F, .ri = 0.25F, .proximity = 1.0F}; + ASSERT_FLOAT_EQ(cbm_sem_combine(&s, &cfg), cfg.w_tfidf * 0.5F + cfg.w_ri * 0.25F, 1e-6); + s.proximity = 1.10F; + ASSERT_FLOAT_EQ(cbm_sem_combine(&s, &cfg), (cfg.w_tfidf * 0.5F + cfg.w_ri * 0.25F) * 1.10F, + 1e-6); + cbm_sem_signals_t all = {.tfidf = 1.0F, + .ri = 1.0F, + .minhash = 1.0F, + .api = 1.0F, + .type = 1.0F, + .decorator = 1.0F, + .struct_profile = 1.0F, + .proximity = 1.10F}; + ASSERT_FLOAT_EQ(cbm_sem_combine(&all, &cfg), 1.0F, 0.0); /* clamped */ + all.near_copy = true; + ASSERT_FLOAT_EQ(cbm_sem_combine(&all, &cfg), 0.0F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_combine(NULL, &cfg), 0.0F, 0.0); + + cbm_sem_corpus_t *c = cbm_sem_corpus_new(); + ASSERT_NOT_NULL(c); + static char *one[] = {"open", "socket", "port"}; + static char *two[] = {"open", "socket", "host", "retry"}; + cbm_sem_corpus_add_doc(c, (const char **)one, 3); + cbm_sem_corpus_add_doc(c, (const char **)two, 4); + cbm_sem_corpus_finalize(c); + cbm_sem_func_t fa; + cbm_sem_func_t fb; + int ia[3]; + int ib[4]; + float wa[3]; + float wb[4]; + sem_terms_of(c, one, 3, &fa, ia, wa); + sem_terms_of(c, two, 4, &fb, ib, wb); + fa.file_path = "net/dial.go"; + fb.file_path = "net/listen.go"; + cbm_sem_signals_t v; + cbm_sem_signal_values(&fa, &fb, &v); + ASSERT_FLOAT_EQ(cbm_sem_combined_score(&fa, &fb, &cfg), cbm_sem_combine(&v, &cfg), 0.0); + cbm_sem_corpus_free(c); + PASS(); +} + TEST(sem_corpus_add_null_doc) { cbm_sem_corpus_t *c = cbm_sem_corpus_new(); ASSERT_NOT_NULL(c); @@ -531,6 +636,8 @@ SUITE(semantic) { RUN_TEST(sem_corpus_add_one_doc); RUN_TEST(sem_corpus_idf); RUN_TEST(sem_corpus_index_accessors_match_name_lookups); + RUN_TEST(sem_tfidf_terms_compare_vocabulary_not_positions); + RUN_TEST(sem_combine_weighs_signals); RUN_TEST(sem_corpus_add_null_doc); RUN_TEST(sem_corpus_free_null); RUN_TEST(sem_get_config_defaults); From ad25b0b64407c530a8d941b05a698c1e061af44f Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Wed, 7 Oct 2026 18:05:58 +0200 Subject: [PATCH 2/7] feat(semantic): best-first admission; each edge carries a calibrated p When a function has more candidate partners than its budget of ten SEMANTICALLY_RELATED edges, admission took the pairs in discovery order: the first ten won, whatever their scores. Admission is now best-first (cbm_sem_admit_best_first: descending score, ties in the canonical order, still a pure function of the sorted inputs). On a blind judged sample from held-out repositories, the pairs first-come admitted over budget were 0 of 17 related; the pairs best-first admits instead, 7 of 20. Where no budget binds nothing changes (django 66 edges identical; fastapi 426 -> 430 edges, 222 changed). Every SEMANTICALLY_RELATED edge now carries "p": the probability that a reader judges the pair related, by score band, from 120 blind-judged admitted pairs (score < 0.778: 0.56, < 0.806: 0.64, < 0.852: 0.73, else 0.86). p is computed from the score as stored (three decimals), so one shown score always has one p. Scores and the 0.75 threshold are unchanged; a scoring change (TF-IDF alone, or weights fitted on the first sample) was judged on the same sample and did not beat the shipped score by the pre-registered margin. Tests: sem_admit_best_first_keeps_the_best, sem_calibrated_p_bands, pipeline_semantic_edges_carry_p; each is RED with its behavior reverted. Signed-off-by: Martin Vogel --- src/pipeline/pass_semantic_edges.c | 43 ++++++++++++++++---- src/semantic/semantic.c | 65 ++++++++++++++++++++++++++++++ src/semantic/semantic.h | 15 +++++++ tests/test_pipeline.c | 53 ++++++++++++++++++++++++ tests/test_semantic.c | 57 ++++++++++++++++++++++++++ 5 files changed, 225 insertions(+), 8 deletions(-) diff --git a/src/pipeline/pass_semantic_edges.c b/src/pipeline/pass_semantic_edges.c index a0bcb636d..17e4f55ad 100644 --- a/src/pipeline/pass_semantic_edges.c +++ b/src/pipeline/pass_semantic_edges.c @@ -1203,8 +1203,29 @@ static int phase6b_merge_edges(cbm_gbuf_t *gbuf, deferred_edge_buf_t *worker_buf } } + /* Admission: best-first by score under the per-function budget + * (cbm_sem_admit_best_first); the dumps and the edge inserts below keep + * the canonical order. */ + float *scores = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)n * sizeof(float)); + int *fa = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)n * sizeof(int)); + int *fb = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)n * sizeof(int)); + bool *eligible = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)n * sizeof(bool)); + bool *admitted = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)n * sizeof(bool)); + bool ranked = scores && fa && fb && eligible && admitted; + for (int e = 0; ranked && e < n; e++) { + scores[e] = pairs[e].score; + fa[e] = pairs[e].i; + fb[e] = pairs[e].j; + eligible[e] = !pairs[e].below; + } + ranked = ranked && cbm_sem_admit_best_first(scores, fa, fb, eligible, n, max_edges, edge_counts, + admitted); + if (!ranked) { + cbm_log_warn("semantic_edges.admission", "reason", "out_of_memory", "pairs", itoa_log(n)); + } + int total_edges = 0; - for (int e = 0; e < n; e++) { + for (int e = 0; ranked && e < n; e++) { deferred_edge_t *de = &pairs[e]; if (de->below) { if (signals) { @@ -1215,21 +1236,27 @@ static int phase6b_merge_edges(cbm_gbuf_t *gbuf, deferred_edge_buf_t *worker_buf if (pair_dump) { fprintf(pair_dump, "%d %d %.9g\n", de->i, de->j, (double)de->score); } - bool admitted = edge_counts[de->i] < max_edges && edge_counts[de->j] < max_edges; if (signals) { - pair_signals_line(signals, gbuf, funcs, de, admitted); + pair_signals_line(signals, gbuf, funcs, de, admitted[e]); } - if (!admitted) { + if (!admitted[e]) { continue; } + /* p from the score as stored, so one shown score carries one p. */ + char score_text[CBM_SZ_32]; + snprintf(score_text, sizeof(score_text), "%.3f", (double)de->score); char props[PROPS_BUF]; - snprintf(props, sizeof(props), "{\"score\":%.3f,\"same_file\":%s}", de->score, - de->same_file ? "true" : "false"); + snprintf(props, sizeof(props), "{\"score\":%s,\"same_file\":%s,\"p\":%.2f}", score_text, + de->same_file ? "true" : "false", + (double)cbm_sem_calibrated_p(strtof(score_text, NULL))); cbm_gbuf_insert_edge(gbuf, de->source_id, de->target_id, "SEMANTICALLY_RELATED", props); - edge_counts[de->i]++; - edge_counts[de->j]++; total_edges++; } + cbm_free(CBM_MEM_CLASS_SEMANTIC, scores); + cbm_free(CBM_MEM_CLASS_SEMANTIC, fa); + cbm_free(CBM_MEM_CLASS_SEMANTIC, fb); + cbm_free(CBM_MEM_CLASS_SEMANTIC, eligible); + cbm_free(CBM_MEM_CLASS_SEMANTIC, admitted); if (pair_dump) { (void)fclose(pair_dump); } diff --git a/src/semantic/semantic.c b/src/semantic/semantic.c index c28615d93..d894d941f 100644 --- a/src/semantic/semantic.c +++ b/src/semantic/semantic.c @@ -1750,6 +1750,71 @@ float cbm_sem_combine(const cbm_sem_signals_t *s, const cbm_sem_config_t *cfg) { return score; } +typedef struct { + float score; + int pos; +} sem_rank_t; + +static int cmp_rank_best_first(const void *pa, const void *pb) { + const sem_rank_t *a = pa; + const sem_rank_t *b = pb; + if (a->score != b->score) { + return a->score > b->score ? -1 : 1; + } + return (a->pos > b->pos) - (a->pos < b->pos); +} + +/* Best-first, not first-come: when a function has more candidate partners + * than its budget, it keeps its best ones. Judged on held-out repositories + * (2026-10-07): the pairs first-come admitted over budget were 0 of 17 + * related, the ones best-first admits instead 7 of 20. */ +bool cbm_sem_admit_best_first(const float *scores, const int *fa, const int *fb, + const bool *eligible, int n, int max_edges, int *counts, + bool *admitted) { + if (n <= 0) { + return true; + } + memset(admitted, 0, (size_t)n * sizeof(admitted[0])); + sem_rank_t *order = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)n * sizeof(sem_rank_t)); + if (!order) { + return false; + } + int m = 0; + for (int k = 0; k < n; k++) { + if (eligible[k]) { + order[m++] = (sem_rank_t){.score = scores[k], .pos = k}; + } + } + qsort(order, (size_t)m, sizeof(order[0]), cmp_rank_best_first); + for (int r = 0; r < m; r++) { + int k = order[r].pos; + if (counts[fa[k]] < max_edges && counts[fb[k]] < max_edges) { + counts[fa[k]]++; + counts[fb[k]]++; + admitted[k] = true; + } + } + cbm_free(CBM_MEM_CLASS_SEMANTIC, order); + return true; +} + +/* Score bands at the quartiles of the admitted scores, each band's p the + * judged share of related pairs (2026-10-07: 120 blind-judged admitted pairs + * from held-out repositories in six languages, three reading rounds; the + * curve is monotone as measured). Ships as the edge property "p". The cuts + * (measured 0.777752, 0.805782, 0.851543) are at the stored score's three + * decimals, so one shown score always carries one p. */ +static const float SEM_P_CUTS[] = {0.778F, 0.806F, 0.852F}; +static const float SEM_P_BANDS[] = {0.556F, 0.640F, 0.731F, 0.857F}; + +float cbm_sem_calibrated_p(float score) { + size_t band = 0; + while (band < sizeof(SEM_P_CUTS) / sizeof(SEM_P_CUTS[0]) && score >= SEM_P_CUTS[band]) { + band++; + } + return SEM_P_BANDS[band]; +} + float cbm_sem_combined_score(const cbm_sem_func_t *a, const cbm_sem_func_t *b, const cbm_sem_config_t *cfg) { if (!a || !b || !cfg) { diff --git a/src/semantic/semantic.h b/src/semantic/semantic.h index 34d738796..34f49c0c5 100644 --- a/src/semantic/semantic.h +++ b/src/semantic/semantic.h @@ -243,6 +243,21 @@ void cbm_sem_signal_values(const cbm_sem_func_t *a, const cbm_sem_func_t *b, /* cbm_sem_combined_score from the signal values (the same arithmetic). */ float cbm_sem_combine(const cbm_sem_signals_t *s, const cbm_sem_config_t *cfg); +/* Best-first admission under a per-function edge budget. Pair k joins + * functions fa[k] and fb[k] with scores[k]; pairs are taken by descending + * score, ties in ascending k (the caller's canonical order), and pair k is + * admitted when eligible[k] and both functions have fewer than max_edges + * admitted pairs. counts holds one zeroed slot per function and is updated. + * Returns false (nothing admitted) when the order cannot be allocated. */ +bool cbm_sem_admit_best_first(const float *scores, const int *fa, const int *fb, + const bool *eligible, int n, int max_edges, int *counts, + bool *admitted); + +/* The probability that an admitted SEMANTICALLY_RELATED pair is related, by + * its score as stored (three decimals): the judged precision of blind-judged + * held-out samples (the edge property "p"). */ +float cbm_sem_calibrated_p(float score); + /* Module proximity multiplier based on file paths. */ float cbm_sem_proximity(const char *path_a, const char *path_b); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index ed7cb7099..206e370a7 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -14,6 +14,7 @@ #include "pipeline/pipeline_internal.h" #include "pipeline/artifact.h" #include "store/store.h" +#include "semantic/semantic.h" #include "git/git_context.h" #include "foundation/dump_verify.h" #include "foundation/sha256.h" @@ -18896,6 +18897,57 @@ TEST(pipeline_semantic_pair_signals_never_change_the_graph) { PASS(); } +/* Every SEMANTICALLY_RELATED edge carries "p": the calibrated probability for + * its score AS STORED (three decimals), so one shown score has one p. */ +TEST(pipeline_semantic_edges_carry_p) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_semp_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + static const char *const subjects[] = {"User", "Server", "Client", "Cache", "Queue", "Mail"}; + for (size_t i = 0; i < sizeof(subjects) / sizeof(subjects[0]); i++) { + char file[64]; + snprintf(file, sizeof(file), "%s_config.go", subjects[i]); + write_sem_family(tmp, file, subjects[i]); + } + char db[512]; + snprintf(db, sizeof(db), "%s/p.db", tmp); + cbm_setenv("CBM_SEMANTIC_THRESHOLD", "0.3", 1); /* as above: admits the near-duplicates */ + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + int rc = cbm_pipeline_run(p); + cbm_unsetenv("CBM_SEMANTIC_THRESHOLD"); + cbm_pipeline_free(p); + ASSERT_EQ(rc, 0); + + cbm_store_t *s = cbm_store_open_path(db); + ASSERT_NOT_NULL(s); + sqlite3_stmt *st = NULL; + ASSERT_EQ(sqlite3_prepare_v2(cbm_store_get_db(s), + "SELECT properties FROM edges WHERE type = 'SEMANTICALLY_RELATED'", + -1, &st, NULL), + SQLITE_OK); + int edges = 0; + while (sqlite3_step(st) == SQLITE_ROW) { + const char *props = (const char *)sqlite3_column_text(st, 0); + ASSERT_NOT_NULL(props); + yyjson_doc *doc = yyjson_read(props, strlen(props), 0); + ASSERT_NOT_NULL(doc); + yyjson_val *score = yyjson_obj_get(yyjson_doc_get_root(doc), "score"); + yyjson_val *pv = yyjson_obj_get(yyjson_doc_get_root(doc), "p"); + ASSERT_TRUE(score && yyjson_is_num(score)); + ASSERT_TRUE(pv && yyjson_is_num(pv)); + double want = cbm_sem_calibrated_p((float)yyjson_get_num(score)); + ASSERT_FLOAT_EQ(yyjson_get_num(pv), want, 0.006); + yyjson_doc_free(doc); + edges++; + } + sqlite3_finalize(st); + cbm_store_close(s); + ASSERT_GT(edges, 0); + th_rmtree(tmp); + PASS(); +} + TEST(pipeline_semantic_edges_no_functions) { char tmp[256]; snprintf(tmp, sizeof(tmp), "/tmp/cbm_nofunc_XXXXXX"); @@ -19809,6 +19861,7 @@ SUITE(pipeline) { RUN_TEST(pipeline_semantic_edges_no_functions); RUN_TEST(pipeline_semantic_batched_matches_unbatched); RUN_TEST(pipeline_semantic_pair_signals_never_change_the_graph); + RUN_TEST(pipeline_semantic_edges_carry_p); RUN_TEST(pipeline_cpp_static_factory_pointer_receiver_issue1153); RUN_TEST(pipeline_cpp_static_factory_reference_receiver_issue1153); RUN_TEST(pipeline_cpp_static_factory_unique_ptr_receiver_issue1153); diff --git a/tests/test_semantic.c b/tests/test_semantic.c index 87bc4dcec..8ae161ed6 100644 --- a/tests/test_semantic.c +++ b/tests/test_semantic.c @@ -460,6 +460,61 @@ TEST(sem_combine_weighs_signals) { PASS(); } +/* Over budget, a function keeps its BEST partners, not its first ones (a + * held-out judged sample: the pairs first-come admitted over budget were 0 of + * 17 related). Equal scores keep the canonical order; an ineligible pair is + * never admitted and takes no budget. */ +TEST(sem_admit_best_first_keeps_the_best) { + /* function 0 has four partners in canonical order, budget 2 */ + const float scores[] = {0.80F, 0.90F, 0.95F, 0.99F, 0.90F}; + const int fa[] = {0, 0, 0, 0, 5}; + const int fb[] = {1, 2, 3, 4, 6}; + const bool eligible[] = {true, true, true, false, true}; + int counts[7] = {0}; + bool admitted[5]; + ASSERT_TRUE(cbm_sem_admit_best_first(scores, fa, fb, eligible, 5, 2, counts, admitted)); + ASSERT_FALSE(admitted[0]); /* first-come would have kept 0.80 */ + ASSERT_TRUE(admitted[1]); + ASSERT_TRUE(admitted[2]); + ASSERT_FALSE(admitted[3]); /* the best score, but recorded only */ + ASSERT_TRUE(admitted[4]); + ASSERT_EQ(counts[0], 2); + ASSERT_EQ(counts[4], 0); + + /* ties: the earlier pair in canonical order wins the last slot */ + const float tied[] = {0.85F, 0.85F}; + const int ta[] = {0, 0}; + const int tb[] = {1, 2}; + const bool all[] = {true, true}; + int tcounts[3] = {0}; + bool tadmitted[2]; + ASSERT_TRUE(cbm_sem_admit_best_first(tied, ta, tb, all, 2, 1, tcounts, tadmitted)); + ASSERT_TRUE(tadmitted[0]); + ASSERT_FALSE(tadmitted[1]); + PASS(); +} + +/* p by the stored (three-decimal) score: the judged bands, monotone. */ +TEST(sem_calibrated_p_bands) { + ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.750F), 0.556F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.777F), 0.556F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.778F), 0.640F, 0.0); + /* the cut sits at the stored precision (0.778), not the raw quartile + * 0.777752: a stored "0.778" and a stored "0.777" never share a band */ + ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.7779F), 0.556F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.805F), 0.640F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.806F), 0.731F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.852F), 0.857F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(1.000F), 0.857F, 0.0); + float last = 0.0F; + for (int milli = 750; milli <= 1000; milli++) { + float p = cbm_sem_calibrated_p((float)milli / 1000.0F); + ASSERT_TRUE(p >= last); + last = p; + } + PASS(); +} + TEST(sem_corpus_add_null_doc) { cbm_sem_corpus_t *c = cbm_sem_corpus_new(); ASSERT_NOT_NULL(c); @@ -638,6 +693,8 @@ SUITE(semantic) { RUN_TEST(sem_corpus_index_accessors_match_name_lookups); RUN_TEST(sem_tfidf_terms_compare_vocabulary_not_positions); RUN_TEST(sem_combine_weighs_signals); + RUN_TEST(sem_admit_best_first_keeps_the_best); + RUN_TEST(sem_calibrated_p_bands); RUN_TEST(sem_corpus_add_null_doc); RUN_TEST(sem_corpus_free_null); RUN_TEST(sem_get_config_defaults); From 3d559b3554acc4a5528daa7adb25277edd7d8b8e Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Wed, 7 Oct 2026 22:27:13 +0200 Subject: [PATCH 3/7] fix(semantic): read JSON strings with their escapes; recalibrate p The semantic pass read every string property of a function (body text, signature, return type, decorators, docstring) with a reader that stopped at the first '"' and kept escapes raw: a value holding an escaped quote lost everything after it, and "\nreturn" became the token "nreturn". cbm_sem_json_str honours the escapes: \" \\ \/ keep their character, \n \t \r \b \f and \uXXXX become a separator. The tokens change, so admissions change: on 13 repositories 1,340 -> 1,387 admitted SEMANTICALLY_RELATED pairs, 614 of them unchanged. Of the dropped pairs that earlier samples had judged, 52 of 89 were related (0.58); of the kept ones, 73 of 92 (0.79). A pre-registered blind sample of 100 newly admitted pairs from 11 held-out repositories (three reading rounds) judged 65 related (0.65, Wilson 0.55-0.74), so the admitted set is 0.72 precise against 0.69 before. The same index run twice gives identical pairs and scores. "p" is refit on the new admitted population (bands at the quartiles of the admitted scores, pooled where a higher band judged lower; added and kept pairs weighted by their share): score < 0.773: 0.63, else 0.76 (was 0.56 / 0.64 / 0.73 / 0.86 in four bands). Tests: sem_json_str_honours_escapes (RED with escapes ignored), sem_calibrated_p_bands. Signed-off-by: Martin Vogel --- src/pipeline/pass_semantic_edges.c | 24 +----------- src/semantic/semantic.c | 62 ++++++++++++++++++++++++++---- src/semantic/semantic.h | 6 +++ tests/test_semantic.c | 37 +++++++++++++----- 4 files changed, 89 insertions(+), 40 deletions(-) diff --git a/src/pipeline/pass_semantic_edges.c b/src/pipeline/pass_semantic_edges.c index 17e4f55ad..da131d3a9 100644 --- a/src/pipeline/pass_semantic_edges.c +++ b/src/pipeline/pass_semantic_edges.c @@ -327,29 +327,9 @@ static const char *file_ext(const char *path) { return dot ? dot : ""; } -/* Extract a JSON string value by key (simple strstr-based, no full parse). */ +/* Extract a JSON string value by key, escapes honoured (cbm_sem_json_str). */ static const char *json_str_value(const char *json, const char *key, char *buf, int bufsize) { - if (!json || !key) { - return NULL; - } - char search[CBM_SZ_64]; - snprintf(search, sizeof(search), "\"%s\":\"", key); - const char *start = strstr(json, search); - if (!start) { - return NULL; - } - start += strlen(search); - const char *end = strchr(start, '"'); - if (!end) { - return NULL; - } - int len = (int)(end - start); - if (len >= bufsize) { - len = bufsize - SKIP_ONE; - } - memcpy(buf, start, (size_t)len); - buf[len] = '\0'; - return buf; + return cbm_sem_json_str(json, key, buf, bufsize); } /* Extract a JSON array of strings by key. Returns count. */ diff --git a/src/semantic/semantic.c b/src/semantic/semantic.c index d894d941f..907b03a30 100644 --- a/src/semantic/semantic.c +++ b/src/semantic/semantic.c @@ -1798,14 +1798,60 @@ bool cbm_sem_admit_best_first(const float *scores, const int *fa, const int *fb, return true; } -/* Score bands at the quartiles of the admitted scores, each band's p the - * judged share of related pairs (2026-10-07: 120 blind-judged admitted pairs - * from held-out repositories in six languages, three reading rounds; the - * curve is monotone as measured). Ships as the edge property "p". The cuts - * (measured 0.777752, 0.805782, 0.851543) are at the stored score's three - * decimals, so one shown score always carries one p. */ -static const float SEM_P_CUTS[] = {0.778F, 0.806F, 0.852F}; -static const float SEM_P_BANDS[] = {0.556F, 0.640F, 0.731F, 0.857F}; +/* Until 2026-10 the reader stopped at the first '"' and kept escapes raw: + * a docstring holding a quote lost everything after it, and "\nreturn" gave + * the token "nreturn". */ +const char *cbm_sem_json_str(const char *json, const char *key, char *buf, int bufsize) { + if (!json || !key || !buf || bufsize <= 0) { + return NULL; + } + char search[CBM_SZ_64]; + snprintf(search, sizeof(search), "\"%s\":\"", key); + const char *p = strstr(json, search); + if (!p) { + return NULL; + } + p += strlen(search); + int n = 0; + for (; *p && *p != '"' && n + SKIP_ONE < bufsize; p++) { + char c = *p; + if (c == '\\' && p[SKIP_ONE]) { + p++; + switch (*p) { + case 'n': + case 't': + case 'r': + case 'b': + case 'f': + c = ' '; + break; + case 'u': + c = ' '; + for (int k = 0; k < 4 && isxdigit((unsigned char)p[SKIP_ONE]); k++) { + p++; + } + break; + default: + c = *p; /* \" \\ \/ */ + break; + } + } + buf[n++] = c; + } + buf[n] = '\0'; + return buf; +} + +/* Score bands at the quartiles of the admitted scores (0.773, 0.802, 0.838 at + * the stored score's three decimals, so one shown score always carries one p), + * each band's p the judged share of related pairs, pooled where a higher band + * judged lower: the upper three pool to one. Judged 2026-10-07 in eleven + * held-out repositories: the pairs this scoring admits that the previous one + * did not (100, blind, three reading rounds) and the earlier samples' pairs it + * keeps (65), each weighted by its share of the admitted set. Ships as the + * edge property "p". */ +static const float SEM_P_CUTS[] = {0.773F}; +static const float SEM_P_BANDS[] = {0.626F, 0.760F}; float cbm_sem_calibrated_p(float score) { size_t band = 0; diff --git a/src/semantic/semantic.h b/src/semantic/semantic.h index 34f49c0c5..143909ef6 100644 --- a/src/semantic/semantic.h +++ b/src/semantic/semantic.h @@ -253,6 +253,12 @@ bool cbm_sem_admit_best_first(const float *scores, const int *fa, const int *fb, const bool *eligible, int n, int max_edges, int *counts, bool *admitted); +/* The string value of "key" in a properties JSON object, unescaped into buf + * (cut at bufsize - 1); NULL when the key is absent. Escapes are honoured: + * \" \\ \/ keep their character, \n \t \r \b \f and \uXXXX become a space (a + * separator for the tokenizer), so a value holding a quote is read whole. */ +const char *cbm_sem_json_str(const char *json, const char *key, char *buf, int bufsize); + /* The probability that an admitted SEMANTICALLY_RELATED pair is related, by * its score as stored (three decimals): the judged precision of blind-judged * held-out samples (the edge property "p"). */ diff --git a/tests/test_semantic.c b/tests/test_semantic.c index 8ae161ed6..98dd0b0ec 100644 --- a/tests/test_semantic.c +++ b/tests/test_semantic.c @@ -496,16 +496,14 @@ TEST(sem_admit_best_first_keeps_the_best) { /* p by the stored (three-decimal) score: the judged bands, monotone. */ TEST(sem_calibrated_p_bands) { - ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.750F), 0.556F, 0.0); - ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.777F), 0.556F, 0.0); - ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.778F), 0.640F, 0.0); - /* the cut sits at the stored precision (0.778), not the raw quartile - * 0.777752: a stored "0.778" and a stored "0.777" never share a band */ - ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.7779F), 0.556F, 0.0); - ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.805F), 0.640F, 0.0); - ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.806F), 0.731F, 0.0); - ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.852F), 0.857F, 0.0); - ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(1.000F), 0.857F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.750F), 0.626F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.772F), 0.626F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.773F), 0.760F, 0.0); + /* the cut sits at the stored precision (0.773), not the raw quartile: + * a stored "0.773" and a stored "0.772" never share a band */ + ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.7729F), 0.626F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.838F), 0.760F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(1.000F), 0.760F, 0.0); float last = 0.0F; for (int milli = 750; milli <= 1000; milli++) { float p = cbm_sem_calibrated_p((float)milli / 1000.0F); @@ -515,6 +513,24 @@ TEST(sem_calibrated_p_bands) { PASS(); } +/* A properties string holding a quote is read whole, and escapes are not + * glued to the next word (until 2026-10 the read stopped at the first quote + * and kept "\n" raw, so "\nreturn" became the token "nreturn"). */ +TEST(sem_json_str_honours_escapes) { + static const char json[] = + "{\"docstring\":\"Say \\\"hi\\\" to\\nthe user \\u00e9 now\",\"bt\":\"x\"}"; + char buf[64]; + ASSERT_NOT_NULL(cbm_sem_json_str(json, "docstring", buf, sizeof(buf))); + ASSERT_STR_EQ(buf, "Say \"hi\" to the user now"); + ASSERT_NOT_NULL(cbm_sem_json_str(json, "bt", buf, sizeof(buf))); + ASSERT_STR_EQ(buf, "x"); + ASSERT_NULL(cbm_sem_json_str(json, "absent", buf, sizeof(buf))); + char small[6]; + ASSERT_NOT_NULL(cbm_sem_json_str(json, "docstring", small, sizeof(small))); + ASSERT_STR_EQ(small, "Say \""); /* cut at the buffer, still terminated */ + PASS(); +} + TEST(sem_corpus_add_null_doc) { cbm_sem_corpus_t *c = cbm_sem_corpus_new(); ASSERT_NOT_NULL(c); @@ -695,6 +711,7 @@ SUITE(semantic) { RUN_TEST(sem_combine_weighs_signals); RUN_TEST(sem_admit_best_first_keeps_the_best); RUN_TEST(sem_calibrated_p_bands); + RUN_TEST(sem_json_str_honours_escapes); RUN_TEST(sem_corpus_add_null_doc); RUN_TEST(sem_corpus_free_null); RUN_TEST(sem_get_config_defaults); From 842535e7aef7bc1064c09a53f4c1ea564fb58bbf Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Wed, 7 Oct 2026 22:27:45 +0200 Subject: [PATCH 4/7] feat(semantic): doc section -> code candidates with a judged p A documentation section that explains a function rarely names it, so the deterministic doc links cannot connect them. The semantic pass now ranks, for every Markdown section that can be about code, the functions whose TF-IDF terms best match the section's heading and body, over the functions' own corpus (function-pair scores are untouched): - gate: no candidates for release-number or project-meta headings (changelog, licence, contributing, links, contents, install, ...), bodies under 20 words, or bodies that are mostly links; - targets: functions and methods, never tests (by path or name); - stored: a section's best five with tfidf >= 0.20, each with p, the probability that the section is about the function, from 274 blind-judged candidates in five held-out repositories (three reading rounds): 0.36 below 0.30, 0.42 below 0.40, else 0.55. Below 0.20 the judged share was 0.06, so nothing is stored there. These are candidates for an agent to verify, not graph edges: at best about half are right, far below the bar for a link. They live in a side table, doc_link_candidates (section and function qualified names, rank, score, p, evidence), created when a generation is published, so older indexes read as "no candidates". Full and legacy-partial reindexes replace a project's rows; the delta route keeps the previous rows (its proxy buffer has no section text or function tokens to recompute them), and a row whose node is gone is skipped when read. get_code_snippet shows them: on a Section, "possibly_about" (its candidate functions); on a function or method, "possibly_described_in" (the sections), at most five, highest p then score first, with a candidates_note saying what they are. Measured on fastapi (6,144 gated sections): 24,293 rows, identical to the generator's dump; index time within noise; the database grows by the table (37 MB there, with a 148-character project prefix on every qualified name). Tests: semantic:sem_doc_calibrated_p_bands, store_arch:doc_candidates_replace_and_get (RED when a replace keeps old rows), pipeline:pipeline_doc_candidates_published_and_kept_by_delta (RED with test functions admitted, the heading gate off, the publish write skipped, or the delta route replacing rows), mcp:snippet_doc_candidates_both_directions (RED with stale rows shown). Signed-off-by: Martin Vogel --- src/mcp/mcp.c | 64 ++++ src/pipeline/pass_semantic_edges.c | 572 ++++++++++++++++++++++++++++ src/pipeline/pipeline.c | 35 ++ src/pipeline/pipeline_incremental.c | 16 +- src/pipeline/pipeline_internal.h | 16 + src/semantic/semantic.c | 17 + src/semantic/semantic.h | 8 + src/store/store.c | 131 +++++++ src/store/store.h | 28 ++ tests/test_mcp.c | 84 ++++ tests/test_pipeline.c | 112 ++++++ tests/test_semantic.c | 20 + tests/test_store_arch.c | 83 ++++ 13 files changed, 1182 insertions(+), 4 deletions(-) diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index bd4196eea..3321eb1f6 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -12859,6 +12859,68 @@ static char *snippet_budget_floor(bool json_format, int max_output_tokens) { return json; } +/* Doc -> code candidates (doc_link_candidates): on a Section, the functions it + * is possibly about; on a function or method, the sections that possibly + * describe it -- highest p first. Candidates with a judged probability for the + * agent to verify, never links. A row whose other end no longer resolves (a + * delta update renamed or removed it) is skipped. */ +enum { MCP_DOC_CANDIDATES_MAX = 5 }; +static void add_doc_candidates(yyjson_mut_doc *doc, yyjson_mut_val *root, cbm_store_t *store, + const cbm_node_t *node) { + if (!store || !node->label || !node->qualified_name || !node->project) { + return; + } + bool section = strcmp(node->label, "Section") == 0; + if (!section && strcmp(node->label, "Function") != 0 && strcmp(node->label, "Method") != 0) { + return; + } + cbm_doc_candidate_t *rows = NULL; + int n = 0; + if (cbm_store_doc_candidates_get(store, node->project, section ? node->qualified_name : NULL, + section ? NULL : node->qualified_name, MCP_DOC_CANDIDATES_MAX, + &rows, &n) != CBM_STORE_OK) { + return; + } + yyjson_mut_val *arr = yyjson_mut_arr(doc); + int added = 0; + for (int i = 0; i < n; i++) { + cbm_node_t other = {0}; + if (cbm_store_find_node_by_qn(store, node->project, + section ? rows[i].target_qn : rows[i].section_qn, + &other) != CBM_STORE_OK) { + continue; + } + yyjson_mut_val *o = yyjson_mut_obj(doc); + yyjson_mut_obj_add_strcpy(doc, o, section ? "qualified_name" : "section", + other.qualified_name ? other.qualified_name : ""); + yyjson_mut_obj_add_strcpy(doc, o, "file_path", other.file_path ? other.file_path : ""); + yyjson_mut_obj_add_int(doc, o, "start_line", other.start_line); + if (section) { + yyjson_mut_obj_add_int(doc, o, "end_line", other.end_line); + } + yyjson_mut_obj_add_real(doc, o, "p", (double)(int)(rows[i].p * 1000.0 + 0.5) / 1000.0); + int shared = 0; + int name_tokens = 0; + if (section && rows[i].evidence && + sscanf(rows[i].evidence, "{\"shared_terms\":%d,\"name_tokens\":%d}", &shared, + &name_tokens) == 2) { + yyjson_mut_obj_add_int(doc, o, "shared_terms", shared); + yyjson_mut_obj_add_int(doc, o, "name_tokens", name_tokens); + } + yyjson_mut_arr_append(arr, o); + added++; + free_node_contents(&other); + } + cbm_store_doc_candidates_free(rows, n); + if (added > 0) { + yyjson_mut_obj_add_val(doc, root, section ? "possibly_about" : "possibly_described_in", + arr); + yyjson_mut_obj_add_str(doc, root, "candidates_note", + "doc<->code candidates by shared terms, not links: p is the " + "judged probability the section is about the function; verify"); + } +} + static char *build_snippet_response(cbm_mcp_server_t *srv, cbm_node_t *node, const char *match_method, bool include_neighbors, cbm_node_t *alternatives, int alt_count, const char *args) { @@ -13075,6 +13137,8 @@ static char *build_snippet_response(cbm_mcp_server_t *srv, cbm_node_t *node, yyjson_mut_obj_add_val(doc, root_obj, "alternatives", arr); } + add_doc_candidates(doc, root_obj, store, node); + bool bounded_default = max_output_tokens > 0; size_t snippet_byte_budget = bounded_default ? (size_t)max_output_tokens * (size_t)MCP_OUTPUT_BYTES_PER_TOKEN_ESTIMATE diff --git a/src/pipeline/pass_semantic_edges.c b/src/pipeline/pass_semantic_edges.c index da131d3a9..c5caf3a83 100644 --- a/src/pipeline/pass_semantic_edges.c +++ b/src/pipeline/pass_semantic_edges.c @@ -30,6 +30,8 @@ #include "foundation/platform.h" #include "foundation/profile.h" +#include +#include #include #include #include @@ -1870,6 +1872,573 @@ static cbm_sem_corpus_t *run_token_phases_batched(cbm_gbuf_t *gbuf, cbm_sem_func /* ── Pass entry point ────────────────────────────────────────────── */ +/* ── Phase 6c: doc sections -> functions (candidate generator) ───── */ + +/* Doc -> code candidates: for every Section node that can be about code, the + * functions whose TF-IDF terms best match the section's heading + body, over + * the FUNCTIONS' corpus (sections never change a function-pair score). + * Gate: no candidates for sections whose heading names a release (v1.2), a + * changelog, licence, contributing, links, contents, install or support + * kind of section, whose body has fewer than SEM_DOC_MIN_WORDS words, or + * whose body is mostly links. + * Targets: never a test function (by path or name); candidates come from + * the section's key terms (held by at most one in SEM_DOC_KEY_DF_DIV + * functions, or SEM_DOC_KEY_DF_MIN) and are scored on every shared term. + * Stored (doc_link_candidates, never graph edges): a section's best + * CBM_SEM_DOC_TOP_K with tfidf >= CBM_SEM_DOC_MIN_SCORE, each with its judged + * probability (cbm_sem_doc_calibrated_p) -- candidates for an agent to + * verify, not links. + * CBM_SEM_DOC_SIGNALS= also dumps every section's best SEM_DOC_TOP_K: + * section and function qualified names, tfidf, rank, shared terms, both + * locations, and the name evidence (the function's name tokens present in the + * section, and the highest idf among them) -- the population the judged + * calibration samples are drawn from. */ +enum { + SEM_DOC_TOP_K = 10, + SEM_DOC_KEY_DF_MIN = 50, + SEM_DOC_KEY_DF_DIV = 20, + SEM_DOC_MAX_TOKENS = 1024, + SEM_DOC_TEXT_MAX = 8192, + SEM_DOC_NAME_TOKENS = 16, + SEM_DOC_MIN_WORDS = 20, + SEM_DOC_LINK_PCT_MAX = 20, /* links per 100 body words */ + SEM_DOC_WORD_MIN_LEN = 3, +}; + +typedef struct { + float score; + int func; + int shared; +} doc_cand_t; + +typedef struct { + int func; + float tfidf; + int rank; + int shared; + int name_shared; /* the function's NAME tokens present in the section */ + float name_idf; /* the highest idf among them (0: none) */ +} doc_hit_t; + +typedef struct { + doc_hit_t hits[SEM_DOC_TOP_K]; + int n; +} doc_result_t; + +static int cmp_doc_cand(const void *pa, const void *pb) { + const doc_cand_t *a = pa; + const doc_cand_t *b = pb; + if (a->score != b->score) { + return a->score > b->score ? -1 : 1; + } + return (a->func > b->func) - (a->func < b->func); +} + +static int cmp_section_node(const void *pa, const void *pb) { + const cbm_gbuf_node_t *a = *(const cbm_gbuf_node_t *const *)pa; + const cbm_gbuf_node_t *b = *(const cbm_gbuf_node_t *const *)pb; + int c = strcmp(a->qualified_name ? a->qualified_name : "", + b->qualified_name ? b->qualified_name : ""); + if (c) { + return c; + } + return (a->start_line > b->start_line) - (a->start_line < b->start_line); +} + +/* Heading kinds that are about the project, not about code: a release + * number, or one of these words (a trailing '*' matches any ending). */ +static const char *const DOC_SKIP_HEADING[] = {"changelog", + "change log", + "release note*", + "what's new", + "license", + "licence", + "contribut*", + "acknowledg*", + "credits", + "sponsor*", + "author*", + "code of conduct", + "security policy", + "links", + "references", + "resources", + "further reading", + "table of contents", + "contents", + "toc", + "installation", + "install", + "getting started", + "support", + "community", + "faq", + "roadmap", + "todo", + NULL}; + +static bool doc_word_char(char c) { + return isalnum((unsigned char)c) || c == '\''; +} + +/* Does lowercase heading lc contain word (or word* as a prefix) at a word start? */ +static bool doc_heading_has(const char *lc, const char *word) { + size_t wl = strlen(word); + bool prefix = wl > 0 && word[wl - SKIP_ONE] == '*'; + size_t ml = prefix ? wl - SKIP_ONE : wl; + for (const char *p = lc; *p; p++) { + if (strncmp(p, word, ml) != 0) { + continue; + } + if (p > lc && doc_word_char(p[-SKIP_ONE])) { + continue; + } + if (!prefix && doc_word_char(p[ml])) { + continue; + } + return true; + } + return false; +} + +/* A release-number heading: "v1.2", "1.2.3", "[0.12.15]", "Gin v1.11.0". */ +static bool doc_heading_is_release(const char *lc) { + for (const char *p = lc; *p; p++) { + if (p > lc && doc_word_char(p[-SKIP_ONE])) { + continue; + } + const char *q = *p == 'v' ? p + SKIP_ONE : p; + if (!isdigit((unsigned char)*q)) { + continue; + } + while (isdigit((unsigned char)*q)) { + q++; + } + if (*q == '.' && isdigit((unsigned char)q[SKIP_ONE])) { + return true; + } + } + return false; +} + +/* Can this section be about code? (heading kind, body length, link share) */ +static bool doc_section_gated(const cbm_gbuf_node_t *sec, char *body) { + char lc[CBM_SZ_256]; + size_t n = 0; + for (const char *p = sec->name ? sec->name : ""; *p && n + SKIP_ONE < sizeof(lc); p++) { + lc[n++] = (char)tolower((unsigned char)*p); + } + lc[n] = '\0'; + if (doc_heading_is_release(lc)) { + return false; + } + for (int k = 0; DOC_SKIP_HEADING[k]; k++) { + if (doc_heading_has(lc, DOC_SKIP_HEADING[k])) { + return false; + } + } + if (!json_str_value(sec->properties_json, "docstring", body, SEM_DOC_TEXT_MAX)) { + return false; + } + int words = 0; + int links = 0; + int run = 0; + for (const char *p = body;; p++) { + if (isalpha((unsigned char)*p)) { + run++; + continue; + } + words += run >= SEM_DOC_WORD_MIN_LEN ? SKIP_ONE : 0; + run = 0; + if (!*p) { + break; + } + if (strncmp(p, "http://", sizeof("http://") - SKIP_ONE) == 0 || + strncmp(p, "https://", sizeof("https://") - SKIP_ONE) == 0 || + strncmp(p, "](", sizeof("](") - SKIP_ONE) == 0) { + links++; + } + } + return words >= SEM_DOC_MIN_WORDS && links * 100 < SEM_DOC_LINK_PCT_MAX * words; +} + +/* A test function, by its file (test directories, test file names) or name. */ +static bool doc_is_test_function(const cbm_gbuf_node_t *fn) { + static const char *const DIRS[] = {"test/", "tests/", "__tests__/", "testing/", "spec/", + "specs/", "testdata/", "fixture/", "fixtures/", NULL}; + const char *path = fn && fn->file_path ? fn->file_path : ""; + for (int k = 0; DIRS[k]; k++) { + for (const char *p = path; (p = strstr(p, DIRS[k])) != NULL; p++) { + if (p == path || p[-SKIP_ONE] == '/') { + return true; + } + } + } + const char *base = strrchr(path, '/'); + base = base ? base + SKIP_ONE : path; + if (strncmp(base, "test_", sizeof("test_") - SKIP_ONE) == 0 || strstr(base, "_test.") || + strstr(base, ".test.") || strstr(base, ".spec.") || strstr(base, "Test.") || + strstr(base, "Tests.")) { + return true; + } + const char *name = fn && fn->name ? fn->name : ""; + return strncmp(name, "test_", sizeof("test_") - SKIP_ONE) == 0 || + (strncmp(name, "Test", sizeof("Test") - SKIP_ONE) == 0 && + (isupper((unsigned char)name[4]) || name[4] == '_')); +} + +/* The section's tokens (heading + body); the caller frees each token. */ +static int doc_section_tokens(const cbm_gbuf_node_t *n, const char *body, char **tokens) { + int count = cbm_sem_tokenize(n->name, tokens, SEM_DOC_MAX_TOKENS); + if (body && count < SEM_DOC_MAX_TOKENS) { + count += cbm_sem_tokenize(body, tokens + count, SEM_DOC_MAX_TOKENS - count); + } + return count; +} + +/* Dot product and shared-term count of two ascending term lists. */ +static float doc_terms_dot(const int *ia, const float *wa, int na, const int *ib, const float *wb, + int nb, int *shared) { + float dot = 0.0F; + int a = 0; + int b = 0; + *shared = 0; + while (a < na && b < nb) { + if (ia[a] == ib[b]) { + dot += wa[a] * wb[b]; + (*shared)++; + a++; + b++; + } else if (ia[a] < ib[b]) { + a++; + } else { + b++; + } + } + return dot; +} + +static float terms_norm(const float *w, int n) { + float s = 0.0F; + for (int k = 0; k < n; k++) { + s += w[k] * w[k]; + } + return sqrtf(s); +} + +typedef struct { + const cbm_sem_func_t *funcs; + int func_count; + const cbm_sem_corpus_t *corpus; + const cbm_gbuf_node_t **secs; + int nsec; + const int *post_off; + const int *post_f; + int key_df; + const float *fnorm; + const bool *is_test; + const int *name_off; /* per function: its name tokens' corpus indices */ + const int *name_idx; + doc_result_t *res; + _Atomic int next; +} doc_ctx_t; + +/* Name-field evidence of one hit: the function's name tokens that the + * section holds (sidx ascending), and the highest idf among them. */ +static void doc_name_evidence(const doc_ctx_t *dc, const int *sidx, int ns, doc_hit_t *hit) { + hit->name_shared = 0; + hit->name_idf = 0.0F; + for (int k = dc->name_off[hit->func]; k < dc->name_off[hit->func + SKIP_ONE]; k++) { + int t = dc->name_idx[k]; + int lo = 0; + int hi = ns - SKIP_ONE; + while (lo <= hi) { + int mid = (lo + hi) / 2; + if (sidx[mid] == t) { + float idf = cbm_sem_corpus_idf_at(dc->corpus, t); + hit->name_shared++; + hit->name_idf = idf > hit->name_idf ? idf : hit->name_idf; + break; + } + if (sidx[mid] < t) { + lo = mid + SKIP_ONE; + } else { + hi = mid - SKIP_ONE; + } + } + } +} + +static void doc_section_one(doc_ctx_t *dc, int s, int *stamp, doc_cand_t *cands, int *sidx, + float *sw, char *body) { + doc_result_t *r = &dc->res[s]; + r->n = 0; + if (!doc_section_gated(dc->secs[s], body)) { + return; + } + char *tokens[SEM_DOC_MAX_TOKENS]; + int count = doc_section_tokens(dc->secs[s], body, tokens); + int ns = cbm_sem_tfidf_terms(dc->corpus, tokens, NULL, count, sidx, sw); + for (int t = 0; t < count; t++) { + cbm_free(CBM_MEM_CLASS_SEMANTIC, tokens[t]); + } + float snorm = terms_norm(sw, ns); + int nc = 0; + for (int k = 0; k < ns && snorm > 0.0F; k++) { + int t = sidx[k]; + if (dc->post_off[t + SKIP_ONE] - dc->post_off[t] > dc->key_df) { + continue; + } + for (int p = dc->post_off[t]; p < dc->post_off[t + SKIP_ONE]; p++) { + int f = dc->post_f[p]; + if (stamp[f] != s && !dc->is_test[f]) { + stamp[f] = s; + cands[nc++] = (doc_cand_t){.func = f}; + } + } + } + for (int c = 0; c < nc; c++) { + const cbm_sem_func_t *fn = &dc->funcs[cands[c].func]; + float dot = doc_terms_dot(sidx, sw, ns, fn->tfidf_indices, fn->tfidf_weights, fn->tfidf_len, + &cands[c].shared); + float denom = snorm * dc->fnorm[cands[c].func]; + cands[c].score = denom > 0.0F ? dot / denom : 0.0F; + } + if (nc > 1) { + qsort(cands, (size_t)nc, sizeof(cands[0]), cmp_doc_cand); + } + for (int c = 0; c < nc && c < SEM_DOC_TOP_K; c++) { + doc_hit_t *hit = &r->hits[r->n++]; + *hit = (doc_hit_t){.func = cands[c].func, + .tfidf = cands[c].score, + .rank = c + SKIP_ONE, + .shared = cands[c].shared}; + doc_name_evidence(dc, sidx, ns, hit); + } +} + +static void doc_section_worker(int worker_id, void *ctx_ptr) { + (void)worker_id; + doc_ctx_t *dc = ctx_ptr; + int *stamp = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)dc->func_count * sizeof(int)); + doc_cand_t *cands = + cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)dc->func_count * sizeof(doc_cand_t)); + int *sidx = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)SEM_DOC_MAX_TOKENS * sizeof(int)); + float *sw = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)SEM_DOC_MAX_TOKENS * sizeof(float)); + char *body = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, SEM_DOC_TEXT_MAX); + if (stamp && cands && sidx && sw && body) { + for (int f = 0; f < dc->func_count; f++) { + stamp[f] = -1; + } + while (true) { + int s = atomic_fetch_add_explicit(&dc->next, SKIP_ONE, memory_order_relaxed); + if (s >= dc->nsec) { + break; + } + doc_section_one(dc, s, stamp, cands, sidx, sw, body); + } + } + cbm_free(CBM_MEM_CLASS_SEMANTIC, stamp); + cbm_free(CBM_MEM_CLASS_SEMANTIC, cands); + cbm_free(CBM_MEM_CLASS_SEMANTIC, sidx); + cbm_free(CBM_MEM_CLASS_SEMANTIC, sw); + cbm_free(CBM_MEM_CLASS_SEMANTIC, body); +} + +static void doc_section_lines(FILE *out, const cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *sec, + const cbm_sem_func_t *funcs, const doc_result_t *r) { + for (int h = 0; h < r->n; h++) { + const doc_hit_t *hit = &r->hits[h]; + const cbm_gbuf_node_t *fn = cbm_gbuf_find_by_id(gbuf, funcs[hit->func].node_id); + (void)fprintf(out, "%s\t%s\t%.6f\t%d\t%d\t%s:%d-%d\t%s:%d-%d\t%d\t%.6f\n", + sec->qualified_name ? sec->qualified_name : "?", + fn && fn->qualified_name ? fn->qualified_name : "?", (double)hit->tfidf, + hit->rank, hit->shared, sec->file_path ? sec->file_path : "?", + sec->start_line, sec->end_line, fn && fn->file_path ? fn->file_path : "?", + fn ? fn->start_line : 0, fn ? fn->end_line : 0, hit->name_shared, + (double)hit->name_idf); + } +} + +/* The stored candidates of every section (rank <= CBM_SEM_DOC_TOP_K, tfidf >= + * CBM_SEM_DOC_MIN_SCORE), malloc'd for cbm_store_doc_candidates_free; p at two + * decimals, as on SEMANTICALLY_RELATED edges. */ +static cbm_doc_candidate_t *doc_candidate_rows(const cbm_gbuf_t *gbuf, const cbm_gbuf_node_t **secs, + int nsec, const cbm_sem_func_t *funcs, + const doc_result_t *res, int *out_count) { + int n = 0; + for (int s = 0; s < nsec; s++) { + for (int h = 0; h < res[s].n; h++) { + n += res[s].hits[h].rank <= CBM_SEM_DOC_TOP_K && + res[s].hits[h].tfidf >= CBM_SEM_DOC_MIN_SCORE; + } + } + *out_count = 0; + cbm_doc_candidate_t *rows = n > 0 ? calloc((size_t)n, sizeof(*rows)) : NULL; + if (!rows) { + return NULL; + } + int k = 0; + for (int s = 0; s < nsec; s++) { + for (int h = 0; h < res[s].n; h++) { + const doc_hit_t *hit = &res[s].hits[h]; + if (hit->rank > CBM_SEM_DOC_TOP_K || hit->tfidf < CBM_SEM_DOC_MIN_SCORE) { + continue; + } + const cbm_gbuf_node_t *fn = cbm_gbuf_find_by_id(gbuf, funcs[hit->func].node_id); + if (!fn || !fn->qualified_name || !secs[s]->qualified_name) { + continue; + } + char ev[CBM_SZ_64]; + snprintf(ev, sizeof(ev), "{\"shared_terms\":%d,\"name_tokens\":%d}", hit->shared, + hit->name_shared); + rows[k] = (cbm_doc_candidate_t){ + .section_qn = strdup(secs[s]->qualified_name), + .target_qn = strdup(fn->qualified_name), + .rank = hit->rank, + .score = hit->tfidf, + .p = (double)(int)(cbm_sem_doc_calibrated_p(hit->tfidf) * 100.0F + 0.5F) / 100.0, + .evidence = strdup(ev)}; + if (!rows[k].section_qn || !rows[k].target_qn || !rows[k].evidence) { + cbm_store_doc_candidates_free(rows, k + SKIP_ONE); + return NULL; + } + k++; + } + } + *out_count = k; + return rows; +} + +static void phase6c_doc_sections(cbm_pipeline_ctx_t *ctx, const cbm_sem_func_t *funcs, + int func_count, const cbm_sem_corpus_t *corpus, int worker_count) { + const cbm_gbuf_t *gbuf = ctx->gbuf; + if (!corpus || func_count <= 0) { + return; + } + const cbm_gbuf_node_t **found = NULL; + int nsec = 0; + if (cbm_gbuf_find_by_label(gbuf, "Section", &found, &nsec) != 0 || nsec == 0) { + return; + } + int nterms = cbm_sem_corpus_token_count(corpus); + const cbm_gbuf_node_t **secs = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)nsec * sizeof(*secs)); + int *post_off = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)nterms + SKIP_ONE) * sizeof(int)); + float *fnorm = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)func_count * sizeof(float)); + bool *is_test = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)func_count * sizeof(bool)); + int *name_off = + cbm_calloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)func_count + SKIP_ONE) * sizeof(int)); + int *name_idx = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, + ((size_t)func_count * SEM_DOC_NAME_TOKENS + SKIP_ONE) * sizeof(int)); + doc_result_t *res = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)nsec * sizeof(doc_result_t)); + int *post_f = NULL; + int *cursor = NULL; + FILE *out = NULL; + if (secs && post_off && fnorm && is_test && name_off && name_idx && res) { + for (int f = 0; f < func_count; f++) { + for (int k = 0; k < funcs[f].tfidf_len; k++) { + post_off[funcs[f].tfidf_indices[k] + SKIP_ONE]++; + } + fnorm[f] = terms_norm(funcs[f].tfidf_weights, funcs[f].tfidf_len); + const cbm_gbuf_node_t *fn = cbm_gbuf_find_by_id(gbuf, funcs[f].node_id); + is_test[f] = doc_is_test_function(fn); + char *tok[SEM_DOC_NAME_TOKENS]; + int nt = fn && fn->name ? cbm_sem_tokenize(fn->name, tok, SEM_DOC_NAME_TOKENS) : 0; + int base = name_off[f]; + int m = 0; + for (int t = 0; t < nt; t++) { + int idx = cbm_sem_corpus_token_index(corpus, tok[t]); + bool dup = false; + for (int u = 0; u < m; u++) { + dup = dup || name_idx[base + u] == idx; + } + if (idx >= 0 && !dup) { + name_idx[base + m++] = idx; + } + cbm_free(CBM_MEM_CLASS_SEMANTIC, tok[t]); + } + name_off[f + SKIP_ONE] = base + m; + } + for (int t = 0; t < nterms; t++) { + post_off[t + SKIP_ONE] += post_off[t]; + } + post_f = + cbm_alloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)post_off[nterms] + SKIP_ONE) * sizeof(int)); + cursor = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)nterms + SKIP_ONE) * sizeof(int)); + } + if (post_f && cursor) { + for (int f = 0; f < func_count; f++) { + for (int k = 0; k < funcs[f].tfidf_len; k++) { + int t = funcs[f].tfidf_indices[k]; + post_f[post_off[t] + cursor[t]++] = f; + } + } + int key_df = func_count / SEM_DOC_KEY_DF_DIV; + if (key_df < SEM_DOC_KEY_DF_MIN) { + key_df = SEM_DOC_KEY_DF_MIN; + } + memcpy(secs, found, (size_t)nsec * sizeof(*secs)); + qsort(secs, (size_t)nsec, sizeof(*secs), cmp_section_node); + doc_ctx_t dc = {.funcs = funcs, + .func_count = func_count, + .corpus = corpus, + .secs = secs, + .nsec = nsec, + .post_off = post_off, + .post_f = post_f, + .key_df = key_df, + .fnorm = fnorm, + .is_test = is_test, + .name_off = name_off, + .name_idx = name_idx, + .res = res}; + atomic_init(&dc.next, 0); + cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false}; + cbm_parallel_for(worker_count, doc_section_worker, &dc, opts); + int row_count = 0; + cbm_doc_candidate_t *rows = doc_candidate_rows(gbuf, secs, nsec, funcs, res, &row_count); + cbm_pipeline_set_doc_candidates(ctx->pipeline, rows, row_count); + cbm_log_info("pass.semantic.doc_candidates", "sections", itoa_log(nsec), "rows", + itoa_log(row_count)); + char path[CBM_SZ_1K]; + out = cbm_safe_getenv("CBM_SEM_DOC_SIGNALS", path, sizeof(path), NULL) + ? cbm_fopen(path, "w") + : NULL; + for (int s = 0; out && s < nsec; s++) { + doc_section_lines(out, gbuf, secs[s], funcs, &res[s]); + } + /* CBM_SEM_DOC_FUNCS=: every function with its location and the + * test flag, for measuring candidates the TF-IDF generator does not + * produce (judged samples only). */ + char fpath[CBM_SZ_1K]; + FILE *fout = cbm_safe_getenv("CBM_SEM_DOC_FUNCS", fpath, sizeof(fpath), NULL) + ? cbm_fopen(fpath, "w") + : NULL; + for (int f = 0; fout && f < func_count; f++) { + const cbm_gbuf_node_t *fn = cbm_gbuf_find_by_id(gbuf, funcs[f].node_id); + (void)fprintf(fout, "%s\t%s:%d-%d\t%d\n", + fn && fn->qualified_name ? fn->qualified_name : "?", + fn && fn->file_path ? fn->file_path : "?", fn ? fn->start_line : 0, + fn ? fn->end_line : 0, is_test[f] ? 1 : 0); + } + if (fout) { + (void)fclose(fout); + } + } + if (out) { + (void)fclose(out); + } + cbm_free(CBM_MEM_CLASS_SEMANTIC, cursor); + cbm_free(CBM_MEM_CLASS_SEMANTIC, post_f); + cbm_free(CBM_MEM_CLASS_SEMANTIC, secs); + cbm_free(CBM_MEM_CLASS_SEMANTIC, post_off); + cbm_free(CBM_MEM_CLASS_SEMANTIC, fnorm); + cbm_free(CBM_MEM_CLASS_SEMANTIC, is_test); + cbm_free(CBM_MEM_CLASS_SEMANTIC, name_off); + cbm_free(CBM_MEM_CLASS_SEMANTIC, name_idx); + cbm_free(CBM_MEM_CLASS_SEMANTIC, res); +} + int cbm_pipeline_pass_semantic_edges(cbm_pipeline_ctx_t *ctx) { /* Controlled by pipeline mode (moderate/full), not env var */ cbm_log_info("pass.start", "pass", "semantic_edges"); @@ -1958,6 +2527,9 @@ int cbm_pipeline_pass_semantic_edges(cbm_pipeline_ctx_t *ctx) { sem_mem_mark("6_score"); + /* Phase 6c: doc sections -> function candidates (doc_link_candidates). */ + phase6c_doc_sections(ctx, funcs, func_count, corpus, worker_count); + /* Phase 7: Cleanup */ CBM_PROF_START(t_phase7); free_lsh_buckets(band_buckets); diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 38e1cb4b2..436a85de4 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -273,6 +273,11 @@ struct cbm_pipeline { int doc_link_row_count; bool doc_links_failed; bool doc_links_ran; + /* Doc -> code candidates from pass_semantic_edges, published with the + * generation (heap rows, cbm_store_doc_candidates_free in + * cbm_pipeline_free). NULL when the pass found none or did not run. */ + cbm_doc_candidate_t *doc_candidates; + int doc_candidate_count; /* Deterministic test-only seam at the final publication boundary. Kept * per pipeline so concurrent test/process activity cannot cross-trigger. */ @@ -915,6 +920,23 @@ void cbm_pipeline_set_lsp_surfaces(cbm_pipeline_t *p, cbm_lsp_surface_row_t *row p->surface_row_count = count; } +void cbm_pipeline_set_doc_candidates(cbm_pipeline_t *p, cbm_doc_candidate_t *rows, int count) { + if (!p) { + cbm_store_doc_candidates_free(rows, count); + return; + } + cbm_store_doc_candidates_free(p->doc_candidates, p->doc_candidate_count); + p->doc_candidates = rows; + p->doc_candidate_count = rows ? count : 0; +} + +const cbm_doc_candidate_t *cbm_pipeline_doc_candidates(const cbm_pipeline_t *p, int *count) { + if (count) { + *count = p ? p->doc_candidate_count : 0; + } + return p ? p->doc_candidates : NULL; +} + static void pipeline_release_test_config(cbm_pipeline_t *p) { if (!p) { return; @@ -995,6 +1017,9 @@ void cbm_pipeline_free(cbm_pipeline_t *p) { p->surface_rows = NULL; p->surface_row_count = 0; pipeline_reset_doc_links(p); + cbm_store_doc_candidates_free(p->doc_candidates, p->doc_candidate_count); + p->doc_candidates = NULL; + p->doc_candidate_count = 0; cbm_git_context_free(&p->git_ctx); /* gbuf, store, registry freed during/after run */ /* Defensive owner cleanup; normal runs release after publication. */ @@ -3075,6 +3100,14 @@ static int publish_staged_impl(char *stage_path, const cbm_pipeline_generation_t ok = cbm_store_adr_store(store, generation->project, generation->adr_content) == CBM_STORE_OK; } + /* Doc -> code candidates belong to the generation too: replaced inside + * the staging store (the table is created here), so a reader never sees + * candidates of another graph. */ + if (ok && !generation->doc_candidates_in_place) { + ok = + cbm_store_doc_candidates_replace(store, generation->project, generation->doc_candidates, + generation->doc_candidate_count) == CBM_STORE_OK; + } if (ok) { ok = cbm_store_set_format_version(store, CBM_INDEX_FORMAT_VERSION) == CBM_STORE_OK; @@ -3366,6 +3399,8 @@ static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_hash_t *bas .doc_link_row_count = p->doc_link_row_count, /* every full route runs a doc-link phase; none having run is a fault */ .doc_links_failed = p->doc_links_failed || !p->doc_links_ran, + .doc_candidates = p->doc_candidates, + .doc_candidate_count = p->doc_candidate_count, }; free(db_dir); diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index bc3beb7e6..330d36d89 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -2082,7 +2082,8 @@ static int dump_and_persist(cbm_gbuf_t *gbuf, const char *db_path, const char *p const cbm_coverage_row_t *cov, int cov_count, const cbm_coverage_meta_t *meta_template, const cbm_lsp_surface_row_t *surface_rows, int surface_row_count, - legacy_doc_rows_t doc) { + legacy_doc_rows_t doc, const cbm_doc_candidate_t *doc_candidates, + int doc_candidate_count) { struct timespec t; cbm_clock_gettime(CLOCK_MONOTONIC, &t); cbm_pipeline_generation_t generation = { @@ -2101,6 +2102,8 @@ static int dump_and_persist(cbm_gbuf_t *gbuf, const char *db_path, const char *p .doc_link_rows = doc.rows, .doc_link_row_count = doc.count, .doc_links_failed = doc.failed, + .doc_candidates = doc_candidates, + .doc_candidate_count = doc_candidate_count, }; int rc = cbm_pipeline_publish_generation(&generation); cbm_log_info("incremental.dump", "rc", itoa_buf(rc), "elapsed_ms", @@ -3169,6 +3172,7 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char .doc_link_rows = doc_rows, .doc_link_row_count = doc_row_count, .doc_links_failed = doc_failed, + .doc_candidates_in_place = true, /* the clone keeps the previous rows */ }; cbm_store_close(staging); staging = NULL; @@ -3796,9 +3800,13 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil * would satisfy a future closure plan with yesterday's surface; an * empty table just routes the next incremental to a full rebuild. */ legacy_doc_rows_t doc_out = {.rows = doc_rows, .count = doc_row_count, .failed = doc_failed}; - int persist_rc = - dump_and_persist(existing, db_path, project, cbm_pipeline_cancelled_ptr(p), manifest, - manifest_count, saved_adr, cov, cov_n, &coverage_meta, NULL, 0, doc_out); + /* Doc candidates: the post-passes ran the semantic pass over the whole + * rehydrated project, so this run's rows are complete. */ + int doc_cand_n = 0; + const cbm_doc_candidate_t *doc_cands = cbm_pipeline_doc_candidates(p, &doc_cand_n); + int persist_rc = dump_and_persist(existing, db_path, project, cbm_pipeline_cancelled_ptr(p), + manifest, manifest_count, saved_adr, cov, cov_n, + &coverage_meta, NULL, 0, doc_out, doc_cands, doc_cand_n); cbm_doclinks_free_rows(doc_rows, doc_row_count); cbm_pipeline_free_semantic_manifest(manifest, manifest_count); free(saved_adr); diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index b00dcd699..6bcbb6361 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -1310,6 +1310,16 @@ typedef struct { const cbm_doc_link_row_t *doc_link_rows; int doc_link_row_count; bool doc_links_failed; + /* Doc -> code candidates of this generation (pass_semantic_edges; may be + * NULL: none found, or a mode without the semantic pass), written into + * the staging store with the graph. */ + const cbm_doc_candidate_t *doc_candidates; + int doc_candidate_count; + /* True when the staging store already holds this generation's + * candidates: the delta clone keeps the previous generation's rows (its + * proxy buffer has no section text or function tokens to recompute + * them); a row whose node is gone resolves to nothing when read. */ + bool doc_candidates_in_place; } cbm_pipeline_generation_t; /* Serialize and fully populate a sibling staging database, then atomically @@ -1382,6 +1392,12 @@ void cbm_pipeline_set_doc_link_rows(cbm_pipeline_t *p, cbm_doc_link_row_t *rows, void cbm_pipeline_take_doc_link_rows(cbm_pipeline_t *p, cbm_doc_link_row_t **rows, int *count, bool *failed, bool *ran); +/* Hand the run's doc -> code candidates (pass_semantic_edges) to the + * pipeline. Takes ownership (rows and strings malloc'd, released with + * cbm_store_doc_candidates_free); a NULL p frees them. Passing NULL/0 clears. */ +void cbm_pipeline_set_doc_candidates(cbm_pipeline_t *p, cbm_doc_candidate_t *rows, int count); +const cbm_doc_candidate_t *cbm_pipeline_doc_candidates(const cbm_pipeline_t *p, int *count); + /* Pipeline accessors for incremental use */ const char *cbm_pipeline_repo_path(const cbm_pipeline_t *p); const cbm_test_declarations_t *cbm_pipeline_test_declarations(const cbm_pipeline_t *p); diff --git a/src/semantic/semantic.c b/src/semantic/semantic.c index 907b03a30..bbdd57b56 100644 --- a/src/semantic/semantic.c +++ b/src/semantic/semantic.c @@ -1861,6 +1861,23 @@ float cbm_sem_calibrated_p(float score) { return SEM_P_BANDS[band]; } +/* Doc -> code candidates by their tfidf: each band's p the judged share of + * sections that are about the function (2026-10-07: 274 blind-judged + * candidates -- a section's best five non-test functions -- from five held-out + * repositories, three reading rounds; the top three measured bands pooled to + * stay monotone). Below the first cut (judged 0.06) nothing is stored. */ +static const float SEM_DOC_P_CUTS[] = {CBM_SEM_DOC_MIN_SCORE, 0.30F, 0.40F}; +static const float SEM_DOC_P_BANDS[] = {0.06F, 0.36F, 0.42F, 0.548F}; + +float cbm_sem_doc_calibrated_p(float score) { + size_t band = 0; + while (band < sizeof(SEM_DOC_P_CUTS) / sizeof(SEM_DOC_P_CUTS[0]) && + score >= SEM_DOC_P_CUTS[band]) { + band++; + } + return SEM_DOC_P_BANDS[band]; +} + float cbm_sem_combined_score(const cbm_sem_func_t *a, const cbm_sem_func_t *b, const cbm_sem_config_t *cfg) { if (!a || !b || !cfg) { diff --git a/src/semantic/semantic.h b/src/semantic/semantic.h index 143909ef6..e641e2f29 100644 --- a/src/semantic/semantic.h +++ b/src/semantic/semantic.h @@ -264,6 +264,14 @@ const char *cbm_sem_json_str(const char *json, const char *key, char *buf, int b * held-out samples (the edge property "p"). */ float cbm_sem_calibrated_p(float score); +/* Doc -> code candidates: a section's best CBM_SEM_DOC_TOP_K non-test + * functions by TF-IDF, stored when their tfidf is at least + * CBM_SEM_DOC_MIN_SCORE, each with the judged probability that the section is + * about the function. */ +#define CBM_SEM_DOC_TOP_K 5 +#define CBM_SEM_DOC_MIN_SCORE 0.20F +float cbm_sem_doc_calibrated_p(float score); + /* Module proximity multiplier based on file paths. */ float cbm_sem_proximity(const char *path_a, const char *path_b); diff --git a/src/store/store.c b/src/store/store.c index 4f76915d5..aad3f79e6 100644 --- a/src/store/store.c +++ b/src/store/store.c @@ -10746,6 +10746,137 @@ int cbm_adr_validate_section_keys(const char **keys, int count, char *errbuf, in return CBM_STORE_OK; } +/* ── Doc -> code candidates ─────────────────────────────────────── */ + +static const char DOC_CANDIDATES_DDL[] = + "CREATE TABLE IF NOT EXISTS doc_link_candidates (" + " project TEXT NOT NULL," + " section_qn TEXT NOT NULL," + " target_qn TEXT NOT NULL," + " rank INTEGER NOT NULL," + " score REAL NOT NULL," + " p REAL NOT NULL," + " evidence TEXT NOT NULL DEFAULT ''" + ");" + "CREATE INDEX IF NOT EXISTS idx_doc_link_candidates_section " + "ON doc_link_candidates(project, section_qn);" + "CREATE INDEX IF NOT EXISTS idx_doc_link_candidates_target " + "ON doc_link_candidates(project, target_qn);"; + +int cbm_store_doc_candidates_replace(cbm_store_t *s, const char *project, + const cbm_doc_candidate_t *rows, int count) { + if (!s || !s->db || !project || count < 0 || (count > 0 && !rows)) { + return CBM_STORE_ERR; + } + if (exec_sql(s, DOC_CANDIDATES_DDL) != CBM_STORE_OK || exec_sql(s, "BEGIN;") != CBM_STORE_OK) { + return CBM_STORE_ERR; + } + sqlite3_stmt *del = NULL; + sqlite3_stmt *ins = NULL; + bool ok = sqlite3_prepare_v2(s->db, "DELETE FROM doc_link_candidates WHERE project = ?1;", + CBM_NOT_FOUND, &del, NULL) == SQLITE_OK && + sqlite3_prepare_v2(s->db, + "INSERT INTO doc_link_candidates " + "(project, section_qn, target_qn, rank, score, p, evidence) " + "VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7);", + CBM_NOT_FOUND, &ins, NULL) == SQLITE_OK; + if (ok) { + bind_text(del, SKIP_ONE, project); + ok = sqlite3_step(del) == SQLITE_DONE; + } + for (int i = 0; ok && i < count; i++) { + if (!rows[i].section_qn || !rows[i].target_qn) { + continue; + } + bind_text(ins, SKIP_ONE, project); + bind_text(ins, ST_COL_2, rows[i].section_qn); + bind_text(ins, ST_COL_3, rows[i].target_qn); + sqlite3_bind_int(ins, ST_COL_4, rows[i].rank); + sqlite3_bind_double(ins, ST_COL_5, rows[i].score); + sqlite3_bind_double(ins, ST_COL_6, rows[i].p); + bind_text(ins, CBM_SZ_7, rows[i].evidence ? rows[i].evidence : ""); + ok = sqlite3_step(ins) == SQLITE_DONE; + sqlite3_reset(ins); + } + sqlite3_finalize(del); + sqlite3_finalize(ins); + if (!ok) { + store_set_error_sqlite(s, "doc_candidates replace"); + (void)exec_sql(s, "ROLLBACK;"); + return CBM_STORE_ERR; + } + return exec_sql(s, "COMMIT;"); +} + +int cbm_store_doc_candidates_get(cbm_store_t *s, const char *project, const char *section_qn, + const char *target_qn, int limit, cbm_doc_candidate_t **out, + int *count) { + if (!s || !s->db || !project || !out || !count || (!section_qn == !target_qn) || limit <= 0) { + return CBM_STORE_ERR; + } + *out = NULL; + *count = 0; + sqlite3_stmt *probe = NULL; + if (sqlite3_prepare_v2(s->db, + "SELECT 1 FROM sqlite_master WHERE type='table' AND " + "name='doc_link_candidates' LIMIT 1;", + CBM_NOT_FOUND, &probe, NULL) != SQLITE_OK) { + store_set_error_sqlite(s, "doc_candidates probe"); + return CBM_STORE_ERR; + } + int rc = sqlite3_step(probe); + sqlite3_finalize(probe); + if (rc == SQLITE_DONE) { + return CBM_STORE_NOT_FOUND; + } + if (rc != SQLITE_ROW) { + store_set_error_sqlite(s, "doc_candidates probe step"); + return CBM_STORE_ERR; + } + const char *sql = section_qn ? "SELECT section_qn, target_qn, rank, score, p, evidence FROM " + "doc_link_candidates WHERE project=?1 AND section_qn=?2 " + "ORDER BY p DESC, score DESC, target_qn LIMIT ?3;" + : "SELECT section_qn, target_qn, rank, score, p, evidence FROM " + "doc_link_candidates WHERE project=?1 AND target_qn=?2 " + "ORDER BY p DESC, score DESC, section_qn LIMIT ?3;"; + sqlite3_stmt *stmt = NULL; + if (sqlite3_prepare_v2(s->db, sql, CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) { + store_set_error_sqlite(s, "doc_candidates get"); + return CBM_STORE_ERR; + } + bind_text(stmt, SKIP_ONE, project); + bind_text(stmt, ST_COL_2, section_qn ? section_qn : target_qn); + sqlite3_bind_int(stmt, ST_COL_3, limit); + cbm_doc_candidate_t *rows = calloc((size_t)limit, sizeof(*rows)); + if (!rows) { + sqlite3_finalize(stmt); + return CBM_STORE_ERR; + } + int n = 0; + while (n < limit && sqlite3_step(stmt) == SQLITE_ROW) { + rows[n].section_qn = heap_strdup((const char *)sqlite3_column_text(stmt, 0)); + rows[n].target_qn = heap_strdup((const char *)sqlite3_column_text(stmt, SKIP_ONE)); + rows[n].rank = sqlite3_column_int(stmt, ST_COL_2); + rows[n].score = sqlite3_column_double(stmt, ST_COL_3); + rows[n].p = sqlite3_column_double(stmt, ST_COL_4); + rows[n].evidence = heap_strdup((const char *)sqlite3_column_text(stmt, ST_COL_5)); + n++; + } + sqlite3_finalize(stmt); + *out = rows; + *count = n; + return CBM_STORE_OK; +} + +void cbm_store_doc_candidates_free(cbm_doc_candidate_t *rows, int count) { + for (int i = 0; rows && i < count; i++) { + free((void *)rows[i].section_qn); + free((void *)rows[i].target_qn); + free((void *)rows[i].evidence); + } + free(rows); +} + void cbm_adr_sections_free(cbm_adr_sections_t *s) { if (!s) { return; diff --git a/src/store/store.h b/src/store/store.h index 8dc67ee48..881ba3dc9 100644 --- a/src/store/store.h +++ b/src/store/store.h @@ -1083,6 +1083,34 @@ int cbm_store_adr_update_sections(cbm_store_t *s, const char *project, const cha const char **values, int count, cbm_adr_t *out); void cbm_store_adr_free(cbm_adr_t *adr); +/* ── Doc -> code candidates (for the agent to verify) ───────────── */ + +/* One documentation section -> function candidate, with the calibrated + * probability that the section is about the function. Candidates, not links: + * never graph edges; a short ranked list per section for the agent to judge. */ +typedef struct { + const char *section_qn; + const char *target_qn; + int rank; + double score; + double p; + const char *evidence; /* JSON: {"terms":n,"name_tokens":m} */ +} cbm_doc_candidate_t; + +/* Replace the project's candidates. The table is created here, at publish + * (not in the schema): a database written by an older build has none, and + * readers treat that as "no candidates". */ +int cbm_store_doc_candidates_replace(cbm_store_t *s, const char *project, + const cbm_doc_candidate_t *rows, int count); + +/* The candidates of one section (section_qn) or for one function (target_qn; + * pass the other NULL), highest p first, then score, at most limit. *out is + * allocated (cbm_store_doc_candidates_free). CBM_STORE_NOT_FOUND: no table. */ +int cbm_store_doc_candidates_get(cbm_store_t *s, const char *project, const char *section_qn, + const char *target_qn, int limit, cbm_doc_candidate_t **out, + int *count); +void cbm_store_doc_candidates_free(cbm_doc_candidate_t *rows, int count); + /* ADR section parsing/rendering (pure functions, no store needed) */ enum { PROPS_MAX = 16 }; diff --git a/tests/test_mcp.c b/tests/test_mcp.c index 5d107a8b3..6ae9a0814 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -18508,6 +18508,89 @@ TEST(snippet_exact_qn) { PASS(); } +/* Doc -> code candidates on get_code_snippet: a Section lists the functions + * it is possibly about (highest p first, evidence included), a function the + * sections that possibly describe it; a row whose other end no longer exists + * is skipped, and a node without rows gets no candidate keys at all. */ +TEST(snippet_doc_candidates_both_directions) { + char tmp[256]; + cbm_mcp_server_t *srv = setup_snippet_server(tmp, sizeof(tmp)); + ASSERT_NOT_NULL(srv); + cbm_store_t *st = cbm_mcp_server_store(srv); + cbm_node_t sec = {.project = "test-project", + .label = "Section", + .name = "Orders", + .qualified_name = "test-project.README.Orders", + .file_path = "README.md", + .start_line = 1, + .end_line = 4}; + ASSERT_TRUE(cbm_store_upsert_node(st, &sec) > 0); + const cbm_doc_candidate_t rows[] = { + {.section_qn = "test-project.README.Orders", + .target_qn = "test-project.cmd.server.main.HandleRequest", + .rank = 2, + .score = 0.25, + .p = 0.36, + .evidence = "{\"shared_terms\":2,\"name_tokens\":0}"}, + {.section_qn = "test-project.README.Orders", + .target_qn = "test-project.cmd.server.main.ProcessOrder", + .rank = 1, + .score = 0.45, + .p = 0.548, + .evidence = "{\"shared_terms\":5,\"name_tokens\":2}"}, + {.section_qn = "test-project.README.Orders", + .target_qn = "test-project.gone.Missing", + .rank = 3, + .score = 0.31, + .p = 0.42, + .evidence = "{\"shared_terms\":1,\"name_tokens\":0}"}, + }; + ASSERT_EQ(cbm_store_doc_candidates_replace(st, "test-project", rows, 3), CBM_STORE_OK); + + char *resp = call_snippet(srv, "{\"qualified_name\":\"test-project.README.Orders\"," + "\"project\":\"test-project\"}"); + ASSERT_NOT_NULL(resp); + yyjson_doc *doc = yyjson_read(resp, strlen(resp), 0); + ASSERT_NOT_NULL(doc); + yyjson_val *about = yyjson_obj_get(yyjson_doc_get_root(doc), "possibly_about"); + ASSERT_EQ((int)yyjson_arr_size(about), 2); + yyjson_val *first = yyjson_arr_get(about, 0); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(first, "qualified_name")), + "test-project.cmd.server.main.ProcessOrder"); + ASSERT_FLOAT_EQ(yyjson_get_num(yyjson_obj_get(first, "p")), 0.548, 1e-9); + ASSERT_EQ((int)yyjson_get_int(yyjson_obj_get(first, "shared_terms")), 5); + ASSERT_EQ((int)yyjson_get_int(yyjson_obj_get(first, "start_line")), 7); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(yyjson_arr_get(about, 1), "qualified_name")), + "test-project.cmd.server.main.HandleRequest"); + ASSERT_NOT_NULL(yyjson_obj_get(yyjson_doc_get_root(doc), "candidates_note")); + yyjson_doc_free(doc); + free(resp); + + resp = call_snippet(srv, "{\"qualified_name\":\"test-project.cmd.server.main.ProcessOrder\"," + "\"project\":\"test-project\"}"); + ASSERT_NOT_NULL(resp); + doc = yyjson_read(resp, strlen(resp), 0); + ASSERT_NOT_NULL(doc); + yyjson_val *in = yyjson_obj_get(yyjson_doc_get_root(doc), "possibly_described_in"); + ASSERT_EQ((int)yyjson_arr_size(in), 1); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(yyjson_arr_get(in, 0), "section")), + "test-project.README.Orders"); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(yyjson_arr_get(in, 0), "file_path")), "README.md"); + yyjson_doc_free(doc); + free(resp); + + resp = call_snippet(srv, "{\"qualified_name\":\"test-project.cmd.server.Run\"," + "\"project\":\"test-project\"}"); + ASSERT_NOT_NULL(resp); + ASSERT_NULL(strstr(resp, "possibly_")); + ASSERT_NULL(strstr(resp, "candidates_note")); + free(resp); + + cbm_mcp_server_free(srv); + cleanup_snippet_dir(tmp); + PASS(); +} + /* ── TestSnippet_QNSuffix ─────────────────────────────────────── */ TEST(snippet_qn_suffix) { @@ -23952,6 +24035,7 @@ SUITE(mcp) { /* Snippet resolution (port of snippet_test.go) */ RUN_TEST(snippet_exact_qn); + RUN_TEST(snippet_doc_candidates_both_directions); RUN_TEST(snippet_qn_suffix); RUN_TEST(snippet_unique_short_name); RUN_TEST(snippet_c_macro_namespace_c1); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 206e370a7..d724f0213 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -18948,6 +18948,117 @@ TEST(pipeline_semantic_edges_carry_p) { PASS(); } +/* Count a section's stored doc -> code candidates (section QN containing + * heading) and check each row's shape: rank within the top + * CBM_SEM_DOC_TOP_K, score at or above the floor, p the doc curve's band at + * two decimals, never a test function. Returns -1 on a bad row. */ +static int doc_candidate_rows_for(cbm_store_t *s, const char *heading, char *top, size_t top_sz) { + sqlite3_stmt *st = NULL; + if (sqlite3_prepare_v2(cbm_store_get_db(s), + "SELECT target_qn, rank, score, p FROM doc_link_candidates " + "WHERE instr(section_qn, ?1) > 0 ORDER BY rank", + -1, &st, NULL) != SQLITE_OK) { + return -1; + } + sqlite3_bind_text(st, 1, heading, -1, SQLITE_TRANSIENT); + int n = 0; + bool ok = true; + while (ok && sqlite3_step(st) == SQLITE_ROW) { + const char *target = (const char *)sqlite3_column_text(st, 0); + int rank = sqlite3_column_int(st, 1); + double score = sqlite3_column_double(st, 2); + double p = sqlite3_column_double(st, 3); + double want = (double)(int)(cbm_sem_doc_calibrated_p((float)score) * 100.0F + 0.5F) / 100.0; + ok = target && rank >= 1 && rank <= CBM_SEM_DOC_TOP_K && + score >= (double)CBM_SEM_DOC_MIN_SCORE && p == want && !strstr(target, ".Test"); + if (ok && n == 0 && top) { + snprintf(top, top_sz, "%s", target); + } + n++; + } + sqlite3_finalize(st); + return ok ? n : -1; +} + +/* Doc -> code candidates end to end: a full index publishes a section's + * calibrated candidates (the best match first, a test function never, a + * changelog section none), and a delta-route reindex keeps them -- its proxy + * buffer cannot recompute them, so the clone's rows must survive. */ +TEST(pipeline_doc_candidates_published_and_kept_by_delta) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_doccand_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + static const char *const subjects[] = {"User", "Server", "Client", "Cache", "Queue", "Mail"}; + for (size_t i = 0; i < sizeof(subjects) / sizeof(subjects[0]); i++) { + char file[64]; + snprintf(file, sizeof(file), "%s_config.go", subjects[i]); + write_sem_family(tmp, file, subjects[i]); + } + /* A test whose comment is the section's own text: the best match of all, + * and still never a candidate. */ + write_temp_file(tmp, "mail_config_test.go", + "package main\n\n" + "// TestParseMailConfig: the mail config file is parsed by ParseMailConfig,\n" + "// which loads the mail config from disk and validates the mail config\n" + "// fields before anything reads them. Missing fields fail the validation,\n" + "// and the parsed mail config can be written back to disk afterwards.\n" + "func TestParseMailConfig(t *testing.T) {\n" + "\tParseMailConfig(\"mail.conf\")\n}\n"); + write_temp_file(tmp, "README.md", + "# Mail configuration\n\n" + "The mail config file is parsed by ParseMailConfig, which loads the mail " + "config from disk and validates the mail config fields before anything " + "reads them. Missing fields fail the validation, and the parsed mail " + "config can be written back to disk afterwards.\n\n" + "## Changelog\n\n" + "The mail config loader now validates missing fields and writes the mail " + "config back to disk after parsing, so a parsed mail config file always " + "carries every field it needs.\n"); + char db[512]; + snprintf(db, sizeof(db), "%s/doccand.db", tmp); + + cbm_pipeline_incremental_test_reset_faults(); + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + cbm_pipeline_free(p); + + cbm_store_t *s = cbm_store_open_path(db); + ASSERT_NOT_NULL(s); + char top[512] = ""; + int mail = doc_candidate_rows_for(s, "Mail-configuration", top, sizeof(top)); + int changelog = doc_candidate_rows_for(s, "Changelog", NULL, 0); + cbm_store_close(s); + ASSERT_GT(mail, 0); + ASSERT_NOT_NULL(strstr(top, "Mail")); + ASSERT_EQ(changelog, 0); + + /* Delta route: an edit that keeps every definition name (closure repair + * declines added names); the candidates are kept as they were. */ + char edited[512]; + snprintf(edited, sizeof(edited), "%s/user_config.go", tmp); + FILE *ef = cbm_fopen(edited, "a"); + ASSERT_NOT_NULL(ef); + (void)fputs("\n// edited after the first index\n", ef); + (void)fclose(ef); + cbm_pipeline_incremental_test_reset_faults(); + p = cbm_pipeline_new(tmp, db, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + ASSERT_EQ(cbm_pipeline_incremental_test_last_route(), CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR); + cbm_pipeline_free(p); + s = cbm_store_open_path(db); + ASSERT_NOT_NULL(s); + char top_after[512] = ""; + int mail_after = doc_candidate_rows_for(s, "Mail-configuration", top_after, sizeof(top_after)); + cbm_store_close(s); + ASSERT_EQ(mail_after, mail); + ASSERT_STR_EQ(top_after, top); + th_rmtree(tmp); + cbm_pipeline_incremental_test_reset_faults(); + PASS(); +} + TEST(pipeline_semantic_edges_no_functions) { char tmp[256]; snprintf(tmp, sizeof(tmp), "/tmp/cbm_nofunc_XXXXXX"); @@ -19862,6 +19973,7 @@ SUITE(pipeline) { RUN_TEST(pipeline_semantic_batched_matches_unbatched); RUN_TEST(pipeline_semantic_pair_signals_never_change_the_graph); RUN_TEST(pipeline_semantic_edges_carry_p); + RUN_TEST(pipeline_doc_candidates_published_and_kept_by_delta); RUN_TEST(pipeline_cpp_static_factory_pointer_receiver_issue1153); RUN_TEST(pipeline_cpp_static_factory_reference_receiver_issue1153); RUN_TEST(pipeline_cpp_static_factory_unique_ptr_receiver_issue1153); diff --git a/tests/test_semantic.c b/tests/test_semantic.c index 98dd0b0ec..006cad45c 100644 --- a/tests/test_semantic.c +++ b/tests/test_semantic.c @@ -513,6 +513,25 @@ TEST(sem_calibrated_p_bands) { PASS(); } +/* Doc -> code candidates: the judged bands (sample 4), monotone, with nothing + * stored below CBM_SEM_DOC_MIN_SCORE (that band judged 0.06). */ +TEST(sem_doc_calibrated_p_bands) { + ASSERT_FLOAT_EQ(cbm_sem_doc_calibrated_p(0.10F), 0.06F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_doc_calibrated_p(0.199F), 0.06F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_doc_calibrated_p(CBM_SEM_DOC_MIN_SCORE), 0.36F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_doc_calibrated_p(0.299F), 0.36F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_doc_calibrated_p(0.30F), 0.42F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_doc_calibrated_p(0.40F), 0.548F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_doc_calibrated_p(1.00F), 0.548F, 0.0); + float last = 0.0F; + for (int milli = 0; milli <= 1000; milli++) { + float p = cbm_sem_doc_calibrated_p((float)milli / 1000.0F); + ASSERT_TRUE(p >= last); + last = p; + } + PASS(); +} + /* A properties string holding a quote is read whole, and escapes are not * glued to the next word (until 2026-10 the read stopped at the first quote * and kept "\n" raw, so "\nreturn" became the token "nreturn"). */ @@ -711,6 +730,7 @@ SUITE(semantic) { RUN_TEST(sem_combine_weighs_signals); RUN_TEST(sem_admit_best_first_keeps_the_best); RUN_TEST(sem_calibrated_p_bands); + RUN_TEST(sem_doc_calibrated_p_bands); RUN_TEST(sem_json_str_honours_escapes); RUN_TEST(sem_corpus_add_null_doc); RUN_TEST(sem_corpus_free_null); diff --git a/tests/test_store_arch.c b/tests/test_store_arch.c index 4e9f59068..6c6037770 100644 --- a/tests/test_store_arch.c +++ b/tests/test_store_arch.c @@ -621,6 +621,88 @@ TEST(adr_delete_not_found) { PASS(); } +/* Doc -> code candidates: a database without the table reads as "none"; + * a replace swaps exactly one project's rows; reads go highest p first (then + * score) in both directions. */ +TEST(doc_candidates_replace_and_get) { + cbm_store_t *s = cbm_store_open_memory(); + ASSERT_NOT_NULL(s); + cbm_doc_candidate_t *got = NULL; + int n = -1; + ASSERT_EQ(cbm_store_doc_candidates_get(s, "p", "p.secA", NULL, 5, &got, &n), + CBM_STORE_NOT_FOUND); + + const cbm_doc_candidate_t rows[] = { + {.section_qn = "p.secA", + .target_qn = "p.f2", + .rank = 2, + .score = 0.25, + .p = 0.36, + .evidence = "{\"shared_terms\":3,\"name_tokens\":0}"}, + {.section_qn = "p.secA", + .target_qn = "p.f1", + .rank = 1, + .score = 0.45, + .p = 0.548, + .evidence = "{\"shared_terms\":6,\"name_tokens\":1}"}, + {.section_qn = "p.secB", + .target_qn = "p.f1", + .rank = 1, + .score = 0.32, + .p = 0.42, + .evidence = ""}, + }; + ASSERT_EQ(cbm_store_doc_candidates_replace(s, "p", rows, 3), CBM_STORE_OK); + const cbm_doc_candidate_t other = {.section_qn = "q.secA", + .target_qn = "q.f9", + .rank = 1, + .score = 0.5, + .p = 0.548, + .evidence = ""}; + ASSERT_EQ(cbm_store_doc_candidates_replace(s, "q", &other, 1), CBM_STORE_OK); + + ASSERT_EQ(cbm_store_doc_candidates_get(s, "p", "p.secA", NULL, 5, &got, &n), CBM_STORE_OK); + ASSERT_EQ(n, 2); + ASSERT_STR_EQ(got[0].target_qn, "p.f1"); + ASSERT_EQ(got[0].rank, 1); + ASSERT_STR_EQ(got[0].evidence, "{\"shared_terms\":6,\"name_tokens\":1}"); + ASSERT_STR_EQ(got[1].target_qn, "p.f2"); + cbm_store_doc_candidates_free(got, n); + + ASSERT_EQ(cbm_store_doc_candidates_get(s, "p", NULL, "p.f1", 5, &got, &n), CBM_STORE_OK); + ASSERT_EQ(n, 2); + ASSERT_STR_EQ(got[0].section_qn, "p.secA"); + ASSERT_STR_EQ(got[1].section_qn, "p.secB"); + cbm_store_doc_candidates_free(got, n); + + ASSERT_EQ(cbm_store_doc_candidates_get(s, "p", NULL, "p.f1", 1, &got, &n), CBM_STORE_OK); + ASSERT_EQ(n, 1); + cbm_store_doc_candidates_free(got, n); + + /* a replace drops every earlier row of the project, and only of it */ + ASSERT_EQ(cbm_store_doc_candidates_replace(s, "p", &rows[2], 1), CBM_STORE_OK); + ASSERT_EQ(cbm_store_doc_candidates_get(s, "p", "p.secA", NULL, 5, &got, &n), CBM_STORE_OK); + ASSERT_EQ(n, 0); + cbm_store_doc_candidates_free(got, n); + ASSERT_EQ(cbm_store_doc_candidates_get(s, "p", "p.secB", NULL, 5, &got, &n), CBM_STORE_OK); + ASSERT_EQ(n, 1); + cbm_store_doc_candidates_free(got, n); + ASSERT_EQ(cbm_store_doc_candidates_get(s, "q", "q.secA", NULL, 5, &got, &n), CBM_STORE_OK); + ASSERT_EQ(n, 1); + cbm_store_doc_candidates_free(got, n); + + ASSERT_EQ(cbm_store_doc_candidates_replace(s, "p", NULL, 0), CBM_STORE_OK); + ASSERT_EQ(cbm_store_doc_candidates_get(s, "p", NULL, "p.f1", 5, &got, &n), CBM_STORE_OK); + ASSERT_EQ(n, 0); + cbm_store_doc_candidates_free(got, n); + + /* exactly one of section / target */ + ASSERT_EQ(cbm_store_doc_candidates_get(s, "p", NULL, NULL, 5, &got, &n), CBM_STORE_ERR); + ASSERT_EQ(cbm_store_doc_candidates_get(s, "p", "p.secA", "p.f1", 5, &got, &n), CBM_STORE_ERR); + cbm_store_close(s); + PASS(); +} + TEST(adr_parse_sections_basic) { cbm_adr_sections_t sec = cbm_adr_parse_sections("## PURPOSE\nFoo\n\n## STACK\nBar"); ASSERT_EQ(sec.count, 2); @@ -1695,6 +1777,7 @@ SUITE(store_arch) { RUN_TEST(adr_upsert); RUN_TEST(adr_delete); RUN_TEST(adr_delete_not_found); + RUN_TEST(doc_candidates_replace_and_get); RUN_TEST(adr_parse_sections_basic); RUN_TEST(adr_parse_sections_all_six); RUN_TEST(adr_parse_sections_non_canonical); From 483fefedbc0a9fc06e9409fabd793655a31b6459 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Wed, 7 Oct 2026 23:19:46 +0200 Subject: [PATCH 5/7] fix(semantic): allocate doc candidates through the memory core The candidate rows and their strings were allocated with calloc/strdup in the semantic pass and in cbm_store_doc_candidates_get, and released with free. They now go through the memory core in CBM_MEM_CLASS_STORE (cbm_calloc, cbm_mem_strdup, cbm_free) on both sides, so the rows the pass hands to the pipeline and the rows a read returns are accounted like every other store buffer. make lint-memory-core: no file grew. Signed-off-by: Martin Vogel --- src/pipeline/pass_semantic_edges.c | 11 ++++++----- src/pipeline/pipeline_internal.h | 4 ++-- src/store/store.c | 19 +++++++++++-------- src/store/store.h | 3 ++- 4 files changed, 21 insertions(+), 16 deletions(-) diff --git a/src/pipeline/pass_semantic_edges.c b/src/pipeline/pass_semantic_edges.c index c5caf3a83..b0cd4dd3f 100644 --- a/src/pipeline/pass_semantic_edges.c +++ b/src/pipeline/pass_semantic_edges.c @@ -2261,7 +2261,7 @@ static void doc_section_lines(FILE *out, const cbm_gbuf_t *gbuf, const cbm_gbuf_ } /* The stored candidates of every section (rank <= CBM_SEM_DOC_TOP_K, tfidf >= - * CBM_SEM_DOC_MIN_SCORE), malloc'd for cbm_store_doc_candidates_free; p at two + * CBM_SEM_DOC_MIN_SCORE), in CBM_MEM_CLASS_STORE for cbm_store_doc_candidates_free; p at two * decimals, as on SEMANTICALLY_RELATED edges. */ static cbm_doc_candidate_t *doc_candidate_rows(const cbm_gbuf_t *gbuf, const cbm_gbuf_node_t **secs, int nsec, const cbm_sem_func_t *funcs, @@ -2274,7 +2274,8 @@ static cbm_doc_candidate_t *doc_candidate_rows(const cbm_gbuf_t *gbuf, const cbm } } *out_count = 0; - cbm_doc_candidate_t *rows = n > 0 ? calloc((size_t)n, sizeof(*rows)) : NULL; + cbm_doc_candidate_t *rows = + n > 0 ? cbm_calloc(CBM_MEM_CLASS_STORE, (size_t)n * sizeof(*rows)) : NULL; if (!rows) { return NULL; } @@ -2293,12 +2294,12 @@ static cbm_doc_candidate_t *doc_candidate_rows(const cbm_gbuf_t *gbuf, const cbm snprintf(ev, sizeof(ev), "{\"shared_terms\":%d,\"name_tokens\":%d}", hit->shared, hit->name_shared); rows[k] = (cbm_doc_candidate_t){ - .section_qn = strdup(secs[s]->qualified_name), - .target_qn = strdup(fn->qualified_name), + .section_qn = cbm_mem_strdup(CBM_MEM_CLASS_STORE, secs[s]->qualified_name), + .target_qn = cbm_mem_strdup(CBM_MEM_CLASS_STORE, fn->qualified_name), .rank = hit->rank, .score = hit->tfidf, .p = (double)(int)(cbm_sem_doc_calibrated_p(hit->tfidf) * 100.0F + 0.5F) / 100.0, - .evidence = strdup(ev)}; + .evidence = cbm_mem_strdup(CBM_MEM_CLASS_STORE, ev)}; if (!rows[k].section_qn || !rows[k].target_qn || !rows[k].evidence) { cbm_store_doc_candidates_free(rows, k + SKIP_ONE); return NULL; diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 6bcbb6361..7ecddbb4e 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -1393,8 +1393,8 @@ void cbm_pipeline_take_doc_link_rows(cbm_pipeline_t *p, cbm_doc_link_row_t **row bool *failed, bool *ran); /* Hand the run's doc -> code candidates (pass_semantic_edges) to the - * pipeline. Takes ownership (rows and strings malloc'd, released with - * cbm_store_doc_candidates_free); a NULL p frees them. Passing NULL/0 clears. */ + * pipeline. Takes ownership (rows and strings in CBM_MEM_CLASS_STORE, released + * with cbm_store_doc_candidates_free); a NULL p frees them. NULL/0 clears. */ void cbm_pipeline_set_doc_candidates(cbm_pipeline_t *p, cbm_doc_candidate_t *rows, int count); const cbm_doc_candidate_t *cbm_pipeline_doc_candidates(const cbm_pipeline_t *p, int *count); diff --git a/src/store/store.c b/src/store/store.c index aad3f79e6..7a64fd17b 100644 --- a/src/store/store.c +++ b/src/store/store.c @@ -10847,19 +10847,22 @@ int cbm_store_doc_candidates_get(cbm_store_t *s, const char *project, const char bind_text(stmt, SKIP_ONE, project); bind_text(stmt, ST_COL_2, section_qn ? section_qn : target_qn); sqlite3_bind_int(stmt, ST_COL_3, limit); - cbm_doc_candidate_t *rows = calloc((size_t)limit, sizeof(*rows)); + cbm_doc_candidate_t *rows = cbm_calloc(CBM_MEM_CLASS_STORE, (size_t)limit * sizeof(*rows)); if (!rows) { sqlite3_finalize(stmt); return CBM_STORE_ERR; } int n = 0; while (n < limit && sqlite3_step(stmt) == SQLITE_ROW) { - rows[n].section_qn = heap_strdup((const char *)sqlite3_column_text(stmt, 0)); - rows[n].target_qn = heap_strdup((const char *)sqlite3_column_text(stmt, SKIP_ONE)); + rows[n].section_qn = + cbm_mem_strdup(CBM_MEM_CLASS_STORE, (const char *)sqlite3_column_text(stmt, 0)); + rows[n].target_qn = + cbm_mem_strdup(CBM_MEM_CLASS_STORE, (const char *)sqlite3_column_text(stmt, SKIP_ONE)); rows[n].rank = sqlite3_column_int(stmt, ST_COL_2); rows[n].score = sqlite3_column_double(stmt, ST_COL_3); rows[n].p = sqlite3_column_double(stmt, ST_COL_4); - rows[n].evidence = heap_strdup((const char *)sqlite3_column_text(stmt, ST_COL_5)); + rows[n].evidence = + cbm_mem_strdup(CBM_MEM_CLASS_STORE, (const char *)sqlite3_column_text(stmt, ST_COL_5)); n++; } sqlite3_finalize(stmt); @@ -10870,11 +10873,11 @@ int cbm_store_doc_candidates_get(cbm_store_t *s, const char *project, const char void cbm_store_doc_candidates_free(cbm_doc_candidate_t *rows, int count) { for (int i = 0; rows && i < count; i++) { - free((void *)rows[i].section_qn); - free((void *)rows[i].target_qn); - free((void *)rows[i].evidence); + cbm_free(CBM_MEM_CLASS_STORE, (void *)rows[i].section_qn); + cbm_free(CBM_MEM_CLASS_STORE, (void *)rows[i].target_qn); + cbm_free(CBM_MEM_CLASS_STORE, (void *)rows[i].evidence); } - free(rows); + cbm_free(CBM_MEM_CLASS_STORE, rows); } void cbm_adr_sections_free(cbm_adr_sections_t *s) { diff --git a/src/store/store.h b/src/store/store.h index 881ba3dc9..e7fb3142b 100644 --- a/src/store/store.h +++ b/src/store/store.h @@ -1105,7 +1105,8 @@ int cbm_store_doc_candidates_replace(cbm_store_t *s, const char *project, /* The candidates of one section (section_qn) or for one function (target_qn; * pass the other NULL), highest p first, then score, at most limit. *out is - * allocated (cbm_store_doc_candidates_free). CBM_STORE_NOT_FOUND: no table. */ + * allocated in CBM_MEM_CLASS_STORE (cbm_store_doc_candidates_free). + * CBM_STORE_NOT_FOUND: no table. */ int cbm_store_doc_candidates_get(cbm_store_t *s, const char *project, const char *section_qn, const char *target_qn, int limit, cbm_doc_candidate_t **out, int *count); From aa89a9c6257071f722b0f25fd6af95d0bc0992b1 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Thu, 8 Oct 2026 00:52:00 +0200 Subject: [PATCH 6/7] test(pipeline): edit the doc-candidate fixture by its exact name pipeline_doc_candidates_published_and_kept_by_delta appended its delta-route edit to "user_config.go" while the fixture writes "User_config.go". On a case-insensitive filesystem (macOS, Windows) that is the same file; on Linux it is a new file, closure repair declines new files (correctly), the run falls back to a full rebuild, the route assertion fails and the early return leaks the pipeline (LeakSanitizer on the arm64 GCC leg). The edit now names the fixture file exactly. Signed-off-by: Martin Vogel --- tests/test_pipeline.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index d724f0213..c11c2842c 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -19034,9 +19034,11 @@ TEST(pipeline_doc_candidates_published_and_kept_by_delta) { ASSERT_EQ(changelog, 0); /* Delta route: an edit that keeps every definition name (closure repair - * declines added names); the candidates are kept as they were. */ + * declines added names and new files -- the name must match the fixture's + * case exactly, or a case-sensitive filesystem gets a new file); the + * candidates are kept as they were. */ char edited[512]; - snprintf(edited, sizeof(edited), "%s/user_config.go", tmp); + snprintf(edited, sizeof(edited), "%s/User_config.go", tmp); FILE *ef = cbm_fopen(edited, "a"); ASSERT_NOT_NULL(ef); (void)fputs("\n// edited after the first index\n", ef); From 75588a2b6915ce4572f1acfa057d6a5b32c0db6b Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Fri, 9 Oct 2026 02:00:27 +0200 Subject: [PATCH 7/7] feat(semantic): doc candidates by format and position; whole files and home folders Doc -> code candidates now carry a p judged for their kind, their document format and where the document sits. Format (sample 6, 234 blind-judged candidates): reStructuredText, AsciiDoc and PDF sections each get their own curve (reST stored from tfidf 0.30, AsciiDoc from 0.20, PDF not measured and not stored). A candidate that repeats an exact MENTIONS link of the same section is not stored. Position (sample 7, 490 blind-judged candidates from four monorepos): every document gets a home folder -- its folder; a docs folder documents its parent; a folder without code hands over to the nearest one with code; the repository root means the whole project. Markdown candidates outside the document's home were right 6 of 90 and are never stored; inside it they are stored from tfidf 0.30 (0.33 / 0.53); for documents of the whole project samples 4 and 7 pooled give 0.22 / 0.51 from 0.30. New kinds, all in doc_link_candidates (evidence gains "kind" and "position"; no new edge type, no schema change): - local: up to two home functions ranked below the top five (0.32); - file: a whole file, its vector the sum of its functions' terms and its definitions' names, config keys included, cut to 64 terms (0.30, from tfidf 0.40); - folder: a README or index document -> its home folder (0.87). get_code_snippet on a section lists every kind with its label; get_file_outline shows the sections about the file or a folder holding it (cbm_store_doc_candidates_for_path). cbm_sem_p_2dp rounded 0.325 and 0.525 down (their float values sit 1.2e-6 below the half); its epsilon is 1e-4 on the x100 scale. Replayed on the sample repositories, the build stores exactly the rows the judged curves select (950 rows, 0 differences). Signed-off-by: Martin Vogel --- src/mcp/mcp.c | 171 +++++-- src/pipeline/pass_semantic_edges.c | 797 ++++++++++++++++++++++++++--- src/semantic/semantic.c | 107 +++- src/semantic/semantic.h | 53 +- src/store/store.c | 78 ++- src/store/store.h | 6 + tests/test_mcp.c | 106 +++- tests/test_pipeline.c | 350 ++++++++++++- tests/test_semantic.c | 138 ++++- tests/test_store_arch.c | 77 ++- 10 files changed, 1716 insertions(+), 167 deletions(-) diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 3321eb1f6..441b3d0e3 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -12859,68 +12859,154 @@ static char *snippet_budget_floor(bool json_format, int max_output_tokens) { return json; } -/* Doc -> code candidates (doc_link_candidates): on a Section, the functions it - * is possibly about; on a function or method, the sections that possibly - * describe it -- highest p first. Candidates with a judged probability for the - * agent to verify, never links. A row whose other end no longer resolves (a - * delta update renamed or removed it) is skipped. */ -enum { MCP_DOC_CANDIDATES_MAX = 5 }; -static void add_doc_candidates(yyjson_mut_doc *doc, yyjson_mut_val *root, cbm_store_t *store, - const cbm_node_t *node) { - if (!store || !node->label || !node->qualified_name || !node->project) { - return; - } - bool section = strcmp(node->label, "Section") == 0; - if (!section && strcmp(node->label, "Function") != 0 && strcmp(node->label, "Method") != 0) { - return; - } - cbm_doc_candidate_t *rows = NULL; - int n = 0; - if (cbm_store_doc_candidates_get(store, node->project, section ? node->qualified_name : NULL, - section ? NULL : node->qualified_name, MCP_DOC_CANDIDATES_MAX, - &rows, &n) != CBM_STORE_OK) { - return; - } - yyjson_mut_val *arr = yyjson_mut_arr(doc); +/* Doc -> code candidates (doc_link_candidates): on a Section, the functions, + * whole files and the home folder it is possibly about; on a function or + * method, the sections that possibly describe it; on a file (get_file_outline), + * the sections that possibly describe the file or a folder holding it -- + * highest p first. Candidates with a judged probability for the agent to + * verify, never links. A row whose other end no longer resolves (a delta + * update renamed or removed it) is skipped. */ +enum { MCP_DOC_CANDIDATES_MAX = 5, MCP_DOC_CANDIDATES_SECTION_MAX = 10 }; +static const char MCP_DOC_CANDIDATES_NOTE[] = + "doc<->code candidates by shared terms or the doc's folder, not links: p is the judged " + "probability the section is about the target; verify"; + +/* The evidence fields of a candidate row as shown: kind, position, and (on + * the section side) the shared terms and name tokens. */ +static void doc_candidate_evidence(yyjson_mut_doc *doc, yyjson_mut_val *o, const char *evidence, + bool counts) { + yyjson_doc *ev = evidence ? yyjson_read(evidence, strlen(evidence), 0) : NULL; + yyjson_val *root = ev ? yyjson_doc_get_root(ev) : NULL; + static const char *const STRS[] = {"kind", "position", NULL}; + for (int k = 0; root && STRS[k]; k++) { + yyjson_val *v = yyjson_obj_get(root, STRS[k]); + if (v && yyjson_is_str(v)) { + yyjson_mut_obj_add_strcpy(doc, o, STRS[k], yyjson_get_str(v)); + } + } + static const char *const INTS[] = {"shared_terms", "name_tokens", NULL}; + for (int k = 0; counts && root && INTS[k]; k++) { + yyjson_val *v = yyjson_obj_get(root, INTS[k]); + if (v && yyjson_is_int(v)) { + yyjson_mut_obj_add_int(doc, o, INTS[k], (int)yyjson_get_int(v)); + } + } + yyjson_doc_free(ev); +} + +/* rows -> an array of shown candidates; the other end of each row is its + * target (by_section) or its section. Returns how many resolved. */ +static int doc_candidates_array(yyjson_mut_doc *doc, yyjson_mut_val *arr, cbm_store_t *store, + const char *project, const cbm_doc_candidate_t *rows, int n, + bool by_section) { int added = 0; for (int i = 0; i < n; i++) { cbm_node_t other = {0}; - if (cbm_store_find_node_by_qn(store, node->project, - section ? rows[i].target_qn : rows[i].section_qn, + if (cbm_store_find_node_by_qn(store, project, + by_section ? rows[i].target_qn : rows[i].section_qn, &other) != CBM_STORE_OK) { continue; } yyjson_mut_val *o = yyjson_mut_obj(doc); - yyjson_mut_obj_add_strcpy(doc, o, section ? "qualified_name" : "section", + yyjson_mut_obj_add_strcpy(doc, o, by_section ? "qualified_name" : "section", other.qualified_name ? other.qualified_name : ""); + if (by_section) { + yyjson_mut_obj_add_strcpy(doc, o, "label", other.label ? other.label : ""); + } yyjson_mut_obj_add_strcpy(doc, o, "file_path", other.file_path ? other.file_path : ""); yyjson_mut_obj_add_int(doc, o, "start_line", other.start_line); - if (section) { + if (by_section) { yyjson_mut_obj_add_int(doc, o, "end_line", other.end_line); } yyjson_mut_obj_add_real(doc, o, "p", (double)(int)(rows[i].p * 1000.0 + 0.5) / 1000.0); - int shared = 0; - int name_tokens = 0; - if (section && rows[i].evidence && - sscanf(rows[i].evidence, "{\"shared_terms\":%d,\"name_tokens\":%d}", &shared, - &name_tokens) == 2) { - yyjson_mut_obj_add_int(doc, o, "shared_terms", shared); - yyjson_mut_obj_add_int(doc, o, "name_tokens", name_tokens); - } + doc_candidate_evidence(doc, o, rows[i].evidence, by_section); yyjson_mut_arr_append(arr, o); added++; free_node_contents(&other); } + return added; +} + +static void add_doc_candidates(yyjson_mut_doc *doc, yyjson_mut_val *root, cbm_store_t *store, + const cbm_node_t *node) { + if (!store || !node->label || !node->qualified_name || !node->project) { + return; + } + bool section = strcmp(node->label, "Section") == 0; + if (!section && strcmp(node->label, "Function") != 0 && strcmp(node->label, "Method") != 0) { + return; + } + cbm_doc_candidate_t *rows = NULL; + int n = 0; + if (cbm_store_doc_candidates_get(store, node->project, section ? node->qualified_name : NULL, + section ? NULL : node->qualified_name, + section ? MCP_DOC_CANDIDATES_SECTION_MAX + : MCP_DOC_CANDIDATES_MAX, + &rows, &n) != CBM_STORE_OK) { + return; + } + yyjson_mut_val *arr = yyjson_mut_arr(doc); + int added = doc_candidates_array(doc, arr, store, node->project, rows, n, section); cbm_store_doc_candidates_free(rows, n); if (added > 0) { yyjson_mut_obj_add_val(doc, root, section ? "possibly_about" : "possibly_described_in", arr); - yyjson_mut_obj_add_str(doc, root, "candidates_note", - "doc<->code candidates by shared terms, not links: p is the " - "judged probability the section is about the function; verify"); + yyjson_mut_obj_add_str(doc, root, "candidates_note", MCP_DOC_CANDIDATES_NOTE); } } +/* get_file_outline: the sections possibly about the file as a whole or about + * a folder holding it (NULL: none). */ +static yyjson_mut_val *file_doc_candidates(yyjson_mut_doc *doc, cbm_store_t *store, + const char *project, const char *path) { + cbm_doc_candidate_t *rows = NULL; + int n = 0; + if (cbm_store_doc_candidates_for_path(store, project, path, MCP_DOC_CANDIDATES_MAX, &rows, + &n) != CBM_STORE_OK) { + return NULL; + } + yyjson_mut_val *arr = yyjson_mut_arr(doc); + int added = doc_candidates_array(doc, arr, store, project, rows, n, false); + cbm_store_doc_candidates_free(rows, n); + return added > 0 ? arr : NULL; +} + +static const char *mut_str(yyjson_mut_val *o, const char *key) { + yyjson_mut_val *v = yyjson_mut_obj_get(o, key); + return v && yyjson_mut_is_str(v) ? yyjson_mut_get_str(v) : ""; +} + +/* The tree form of file_doc_candidates. */ +static void file_doc_candidates_tree(cbm_sb_t *sb, cbm_store_t *store, const char *project, + const char *path) { + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + yyjson_mut_val *arr = doc ? file_doc_candidates(doc, store, project, path) : NULL; + if (arr) { + static const char *const columns[] = {"section", "file_path", "line", "p", "kind"}; + cbm_tree_table_header(sb, "possibly_described_in", (int)yyjson_mut_arr_size(arr), columns, + (int)(sizeof(columns) / sizeof(columns[0]))); + size_t i = 0; + size_t max = 0; + yyjson_mut_val *o = NULL; + yyjson_mut_arr_foreach(arr, i, max, o) { + char line[CBM_SZ_32]; + char p[CBM_SZ_32]; + snprintf(line, sizeof(line), "%d", + (int)yyjson_mut_get_int(yyjson_mut_obj_get(o, "start_line"))); + snprintf(p, sizeof(p), "%.2f", yyjson_mut_get_real(yyjson_mut_obj_get(o, "p"))); + cbm_tree_row_begin(sb); + cbm_tree_cell_str(sb, mut_str(o, "section"), true); + cbm_tree_cell_str(sb, mut_str(o, "file_path"), false); + cbm_tree_cell_str(sb, line, false); + cbm_tree_cell_str(sb, p, false); + cbm_tree_cell_str(sb, mut_str(o, "kind"), false); + cbm_tree_row_end(sb); + } + cbm_tree_scalar_str(sb, "candidates_note", MCP_DOC_CANDIDATES_NOTE); + } + yyjson_mut_doc_free(doc); +} + static char *build_snippet_response(cbm_mcp_server_t *srv, cbm_node_t *node, const char *match_method, bool include_neighbors, cbm_node_t *alternatives, int alt_count, const char *args) { @@ -13404,6 +13490,9 @@ static char *handle_get_file_outline(cbm_mcp_server_t *srv, const char *args) { cbm_tree_scalar_int(&sb, "limit", limit); cbm_tree_scalar_int(&sb, "returned", row_count); cbm_tree_scalar_bool(&sb, "has_more", (int64_t)offset + row_count < total); + if (offset == 0) { + file_doc_candidates_tree(&sb, store, project, normalized_path); + } payload = cbm_sb_finish(&sb); } else { yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); @@ -13439,6 +13528,12 @@ static char *handle_get_file_outline(cbm_mcp_server_t *srv, const char *args) { yyjson_mut_obj_add_int(doc, object, "limit", limit); yyjson_mut_obj_add_int(doc, object, "returned", row_count); yyjson_mut_obj_add_bool(doc, object, "has_more", (int64_t)offset + row_count < total); + yyjson_mut_val *described = + offset == 0 ? file_doc_candidates(doc, store, project, normalized_path) : NULL; + if (described) { + yyjson_mut_obj_add_val(doc, object, "possibly_described_in", described); + yyjson_mut_obj_add_str(doc, object, "candidates_note", MCP_DOC_CANDIDATES_NOTE); + } payload = yy_doc_to_str(doc); yyjson_mut_doc_free(doc); } diff --git a/src/pipeline/pass_semantic_edges.c b/src/pipeline/pass_semantic_edges.c index b0cd4dd3f..d406306e3 100644 --- a/src/pipeline/pass_semantic_edges.c +++ b/src/pipeline/pass_semantic_edges.c @@ -1920,9 +1920,20 @@ typedef struct { float name_idf; /* the highest idf among them (0: none) */ } doc_hit_t; +typedef struct { + int file; + float tfidf; + int rank; + int shared; +} doc_file_hit_t; + typedef struct { doc_hit_t hits[SEM_DOC_TOP_K]; int n; + doc_hit_t local[CBM_SEM_DOC_LOCAL_K]; /* inside the home, below the top CBM_SEM_DOC_TOP_K */ + int nlocal; + doc_file_hit_t files[CBM_SEM_DOC_FILE_K]; + int nfiles; } doc_result_t; static int cmp_doc_cand(const void *pa, const void *pb) { @@ -2062,11 +2073,11 @@ static bool doc_section_gated(const cbm_gbuf_node_t *sec, char *body) { return words >= SEM_DOC_MIN_WORDS && links * 100 < SEM_DOC_LINK_PCT_MAX * words; } -/* A test function, by its file (test directories, test file names) or name. */ -static bool doc_is_test_function(const cbm_gbuf_node_t *fn) { +/* A test file: in a test directory, or a test file name. */ +static bool doc_is_test_path(const char *path) { static const char *const DIRS[] = {"test/", "tests/", "__tests__/", "testing/", "spec/", "specs/", "testdata/", "fixture/", "fixtures/", NULL}; - const char *path = fn && fn->file_path ? fn->file_path : ""; + path = path ? path : ""; for (int k = 0; DIRS[k]; k++) { for (const char *p = path; (p = strstr(p, DIRS[k])) != NULL; p++) { if (p == path || p[-SKIP_ONE] == '/') { @@ -2076,9 +2087,14 @@ static bool doc_is_test_function(const cbm_gbuf_node_t *fn) { } const char *base = strrchr(path, '/'); base = base ? base + SKIP_ONE : path; - if (strncmp(base, "test_", sizeof("test_") - SKIP_ONE) == 0 || strstr(base, "_test.") || - strstr(base, ".test.") || strstr(base, ".spec.") || strstr(base, "Test.") || - strstr(base, "Tests.")) { + return strncmp(base, "test_", sizeof("test_") - SKIP_ONE) == 0 || strstr(base, "_test.") || + strstr(base, ".test.") || strstr(base, ".spec.") || strstr(base, "Test.") || + strstr(base, "Tests."); +} + +/* A test function, by its file or name. */ +static bool doc_is_test_function(const cbm_gbuf_node_t *fn) { + if (doc_is_test_path(fn ? fn->file_path : NULL)) { return true; } const char *name = fn && fn->name ? fn->name : ""; @@ -2126,6 +2142,449 @@ static float terms_norm(const float *w, int n) { return sqrtf(s); } +/* ── Phase 6c: whole files and the doc's home folder ── */ + +/* Whole-file candidates: a file's terms are the sum of its non-test + * functions' TF-IDF vectors and the TF-IDF vectors of its other definitions' + * names (a config file's keys are Variable nodes), cut to its + * SEM_DOC_FILE_TERMS heaviest terms (ties by token index), so a file's vector + * costs at most what one long function's does. */ +enum { SEM_DOC_FILE_TERMS = 64 }; +static const char *const DOC_FILE_DEF_LABELS[] = {"Class", "Struct", "Interface", "Enum", + "Type", "Trait", "Field", "Variable", + "Constant", "Macro", NULL}; + +typedef struct { + int count; + const cbm_gbuf_node_t **node; /* per file: its File node */ + int *off; /* count + 1: each file's terms in idx / w */ + int *idx; + float *w; + float *norm; + int *post_off; /* nterms + 1: the files holding each term */ + int *post_f; + CBMHashTable *by_path; /* file path (graph-owned) -> file index + 1 */ +} doc_files_t; + +typedef struct { + int idx; + float w; +} doc_term_t; + +static int cmp_term_idx(const void *pa, const void *pb) { + const doc_term_t *a = pa; + const doc_term_t *b = pb; + return (a->idx > b->idx) - (a->idx < b->idx); +} + +static int cmp_term_heavy(const void *pa, const void *pb) { + const doc_term_t *a = pa; + const doc_term_t *b = pb; + if (a->w != b->w) { + return a->w > b->w ? -1 : 1; + } + return (a->idx > b->idx) - (a->idx < b->idx); +} + +static void doc_files_free(doc_files_t *df) { + cbm_free(CBM_MEM_CLASS_SEMANTIC, df->node); + cbm_free(CBM_MEM_CLASS_SEMANTIC, df->off); + cbm_free(CBM_MEM_CLASS_SEMANTIC, df->idx); + cbm_free(CBM_MEM_CLASS_SEMANTIC, df->w); + cbm_free(CBM_MEM_CLASS_SEMANTIC, df->norm); + cbm_free(CBM_MEM_CLASS_SEMANTIC, df->post_off); + cbm_free(CBM_MEM_CLASS_SEMANTIC, df->post_f); + if (df->by_path) { + cbm_ht_free(df->by_path); + } + memset(df, 0, sizeof(*df)); +} + +static int doc_file_of(const doc_files_t *df, const char *path) { + return path ? (int)(intptr_t)cbm_ht_get(df->by_path, path) - SKIP_ONE : -SKIP_ONE; +} + +/* The scratch terms of one file: append n terms, growing the buffer. */ +typedef struct { + doc_term_t *t; + int n; + int cap; +} doc_scratch_t; + +static bool doc_scratch_add(doc_scratch_t *sc, const int *idx, const float *w, int n) { + if (sc->n + n > sc->cap) { + int cap = sc->cap ? sc->cap : CBM_SZ_256; + while (cap < sc->n + n) { + cap *= 2; + } + doc_term_t *t = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, sc->t, (size_t)cap * sizeof(*t)); + if (!t) { + return false; + } + sc->t = t; + sc->cap = cap; + } + for (int k = 0; k < n; k++) { + sc->t[sc->n++] = (doc_term_t){.idx = idx[k], .w = w[k]}; + } + return true; +} + +/* Sum the scratch terms per token and keep the heaviest SEM_DOC_FILE_TERMS, + * written at df->idx/w + off in ascending token order; returns how many. */ +static int doc_scratch_reduce(doc_scratch_t *sc, doc_files_t *df, int off) { + if (sc->n == 0) { + return 0; + } + qsort(sc->t, (size_t)sc->n, sizeof(*sc->t), cmp_term_idx); + int m = 0; + for (int k = 0; k < sc->n; k++) { + if (m > 0 && sc->t[m - SKIP_ONE].idx == sc->t[k].idx) { + sc->t[m - SKIP_ONE].w += sc->t[k].w; + } else { + sc->t[m++] = sc->t[k]; + } + } + if (m > SEM_DOC_FILE_TERMS) { + qsort(sc->t, (size_t)m, sizeof(*sc->t), cmp_term_heavy); + m = SEM_DOC_FILE_TERMS; + qsort(sc->t, (size_t)m, sizeof(*sc->t), cmp_term_idx); + } + for (int k = 0; k < m; k++) { + df->idx[off + k] = sc->t[k].idx; + df->w[off + k] = sc->t[k].w; + } + return m; +} + +/* The definitions (non-function) of every indexed non-test file, grouped by + * file: def_off (count + 1) into def_node. */ +static bool doc_files_defs(const cbm_gbuf_t *gbuf, const doc_files_t *df, int **def_off, + const cbm_gbuf_node_t ***def_node) { + *def_off = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)df->count + SKIP_ONE) * sizeof(int)); + if (!*def_off) { + return false; + } + for (int pass = 0; pass < 2; pass++) { + int *cursor = + pass ? cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)df->count * sizeof(int)) : NULL; + if (pass && !cursor) { + return false; + } + for (int l = 0; DOC_FILE_DEF_LABELS[l]; l++) { + const cbm_gbuf_node_t **nodes = NULL; + int n = 0; + if (cbm_gbuf_find_by_label(gbuf, DOC_FILE_DEF_LABELS[l], &nodes, &n) != 0) { + continue; + } + for (int i = 0; i < n; i++) { + int f = nodes[i]->name ? doc_file_of(df, nodes[i]->file_path) : -SKIP_ONE; + if (f >= 0 && !pass) { + (*def_off)[f + SKIP_ONE]++; + } else if (f >= 0) { + (*def_node)[(*def_off)[f] + cursor[f]++] = nodes[i]; + } + } + } + if (!pass) { + for (int f = 0; f < df->count; f++) { + (*def_off)[f + SKIP_ONE] += (*def_off)[f]; + } + *def_node = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, + ((size_t)(*def_off)[df->count] + SKIP_ONE) * sizeof(**def_node)); + if (!*def_node) { + return false; + } + } + cbm_free(CBM_MEM_CLASS_SEMANTIC, cursor); + } + return true; +} + +/* The scratch terms of one definition's name. */ +static bool doc_scratch_name(doc_scratch_t *sc, const cbm_sem_corpus_t *corpus, const char *name) { + char *tok[SEM_DOC_NAME_TOKENS]; + int idx[SEM_DOC_NAME_TOKENS]; + float w[SEM_DOC_NAME_TOKENS]; + int nt = cbm_sem_tokenize(name, tok, SEM_DOC_NAME_TOKENS); + int nw = nt > 0 ? cbm_sem_tfidf_terms(corpus, tok, NULL, nt, idx, w) : 0; + for (int t = 0; t < nt; t++) { + cbm_free(CBM_MEM_CLASS_SEMANTIC, tok[t]); + } + return doc_scratch_add(sc, idx, w, nw); +} + +/* Each file's term vector, norm, and the term -> files postings. */ +static bool doc_files_vectors(const cbm_gbuf_t *gbuf, doc_files_t *df, const cbm_sem_func_t *funcs, + const int *ffile, int func_count, const cbm_sem_corpus_t *corpus, + int nterms) { + int *fn_off = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)df->count + SKIP_ONE) * sizeof(int)); + int *fn_list = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)func_count + SKIP_ONE) * sizeof(int)); + int *def_off = NULL; + const cbm_gbuf_node_t **def_node = NULL; + doc_scratch_t sc = {0}; + bool ok = fn_off && fn_list && doc_files_defs(gbuf, df, &def_off, &def_node); + for (int f = 0; ok && f < func_count; f++) { + fn_off[ffile[f] + SKIP_ONE] += ffile[f] >= 0; + } + for (int i = 0; ok && i < df->count; i++) { + fn_off[i + SKIP_ONE] += fn_off[i]; + } + for (int f = 0; ok && f < func_count; f++) { + if (ffile[f] >= 0) { + fn_list[fn_off[ffile[f]]++] = f; /* fn_off[i] ends at file i's end: shifted below */ + } + } + for (int i = df->count; ok && i > 0; i--) { + fn_off[i] = fn_off[i - SKIP_ONE]; + } + if (ok) { + fn_off[0] = 0; + } + for (int i = 0; ok && i < df->count; i++) { + sc.n = 0; + for (int k = fn_off[i]; ok && k < fn_off[i + SKIP_ONE]; k++) { + const cbm_sem_func_t *fn = &funcs[fn_list[k]]; + ok = doc_scratch_add(&sc, fn->tfidf_indices, fn->tfidf_weights, fn->tfidf_len); + } + for (int k = def_off[i]; ok && k < def_off[i + SKIP_ONE]; k++) { + ok = doc_scratch_name(&sc, corpus, def_node[k]->name); + } + int m = ok ? doc_scratch_reduce(&sc, df, df->off[i]) : 0; + df->off[i + SKIP_ONE] = df->off[i] + m; + df->norm[i] = terms_norm(df->w + df->off[i], m); + for (int k = 0; k < m; k++) { + df->post_off[df->idx[df->off[i] + k] + SKIP_ONE]++; + } + } + cbm_free(CBM_MEM_CLASS_SEMANTIC, sc.t); + cbm_free(CBM_MEM_CLASS_SEMANTIC, fn_off); + cbm_free(CBM_MEM_CLASS_SEMANTIC, fn_list); + cbm_free(CBM_MEM_CLASS_SEMANTIC, def_off); + cbm_free(CBM_MEM_CLASS_SEMANTIC, def_node); + if (!ok) { + return false; + } + for (int t = 0; t < nterms; t++) { + df->post_off[t + SKIP_ONE] += df->post_off[t]; + } + df->post_f = + cbm_alloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)df->post_off[nterms] + SKIP_ONE) * sizeof(int)); + int *cursor = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)nterms + SKIP_ONE) * sizeof(int)); + ok = df->post_f && cursor; + for (int i = 0; ok && i < df->count; i++) { + for (int k = df->off[i]; k < df->off[i + SKIP_ONE]; k++) { + int t = df->idx[k]; + df->post_f[df->post_off[t] + cursor[t]++] = i; + } + } + cbm_free(CBM_MEM_CLASS_SEMANTIC, cursor); + return ok; +} + +/* The indexed non-test files (their File nodes) with their term vectors; + * ffile[f] = function f's file (-1: a test function or no File node). */ +static bool doc_files_build(const cbm_gbuf_t *gbuf, const cbm_sem_func_t *funcs, int func_count, + const bool *is_test, const cbm_sem_corpus_t *corpus, int nterms, + doc_files_t *df, int *ffile) { + memset(df, 0, sizeof(*df)); + const cbm_gbuf_node_t **files = NULL; + int nfiles = 0; + if (cbm_gbuf_find_by_label(gbuf, "File", &files, &nfiles) != 0) { + nfiles = 0; + } + df->by_path = cbm_ht_create_in(CBM_MEM_CLASS_SEMANTIC, (uint32_t)nfiles + SKIP_ONE); + df->node = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)nfiles + SKIP_ONE) * sizeof(*df->node)); + if (!df->by_path || !df->node) { + return false; + } + for (int i = 0; i < nfiles; i++) { + const char *path = files[i]->file_path; + if (path && path[0] && !doc_is_test_path(path) && !cbm_ht_has(df->by_path, path)) { + df->node[df->count] = files[i]; + cbm_ht_set(df->by_path, path, (void *)(intptr_t)(df->count + SKIP_ONE)); + df->count++; + } + } + for (int f = 0; f < func_count; f++) { + const cbm_gbuf_node_t *fn = cbm_gbuf_find_by_id(gbuf, funcs[f].node_id); + ffile[f] = is_test[f] || !fn ? -SKIP_ONE : doc_file_of(df, fn->file_path); + } + size_t cap = (size_t)df->count * SEM_DOC_FILE_TERMS + SKIP_ONE; + df->off = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)df->count + SKIP_ONE) * sizeof(int)); + df->idx = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, cap * sizeof(int)); + df->w = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, cap * sizeof(float)); + df->norm = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)df->count + SKIP_ONE) * sizeof(float)); + df->post_off = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)nterms + SKIP_ONE) * sizeof(int)); + return df->off && df->idx && df->w && df->norm && df->post_off && + doc_files_vectors(gbuf, df, funcs, ffile, func_count, corpus, nterms); +} + +/* A doc's home folder: the folder whose code it documents. From the doc's + * directory, a docs folder (doc, docs, documentation: the deepest one) + * documents its parent; then climb while the folder holds no indexed code + * (a non-test file with terms). "" = the repository root: the doc documents + * the whole project. Returns an allocated string (NULL: out of memory). */ +static bool doc_dir_is_docs(const char *name, size_t n) { + static const char *const DOCS[] = {"doc", "docs", "documentation", NULL}; + for (int k = 0; DOCS[k]; k++) { + if (strlen(DOCS[k]) == n && strncasecmp(name, DOCS[k], n) == 0) { + return true; + } + } + return false; +} + +static char *doc_home(const char *doc_path, const CBMHashTable *code_dirs) { + const char *slash = doc_path ? strrchr(doc_path, '/') : NULL; + size_t len = slash ? (size_t)(slash - doc_path) : 0; + char *dir = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, len + SKIP_ONE); + if (!dir) { + return NULL; + } + memcpy(dir, doc_path ? doc_path : "", len); + dir[len] = '\0'; + size_t cut = len + SKIP_ONE; /* the deepest docs folder's start (none) */ + for (size_t a = 0; a < len;) { + const char *end = strchr(dir + a, '/'); + size_t b = end ? (size_t)(end - dir) : len; + if (doc_dir_is_docs(dir + a, b - a)) { + cut = a; + } + a = b + SKIP_ONE; + } + if (cut <= len) { + len = cut > 0 ? cut - SKIP_ONE : 0; /* the docs folder's parent */ + dir[len] = '\0'; + } + while (len > 0 && !cbm_ht_has(code_dirs, dir)) { + const char *up = strrchr(dir, '/'); + len = up ? (size_t)(up - dir) : 0; + dir[len] = '\0'; + } + return dir; +} + +/* Every folder holding indexed code: the directories (and their ancestors) + * of the files with terms; keys owned by the table's caller (freed by + * doc_code_dirs_free). */ +static void doc_free_key(const char *key, void *value, void *userdata) { + (void)value; + (void)userdata; + cbm_free(CBM_MEM_CLASS_SEMANTIC, (void *)key); +} + +static void doc_code_dirs_free(CBMHashTable *dirs) { + if (dirs) { + cbm_ht_foreach(dirs, doc_free_key, NULL); + cbm_ht_free(dirs); + } +} + +static CBMHashTable *doc_code_dirs(const doc_files_t *df) { + CBMHashTable *dirs = cbm_ht_create_in(CBM_MEM_CLASS_SEMANTIC, CBM_SZ_256); + for (int i = 0; dirs && i < df->count; i++) { + const char *path = df->node[i]->file_path; + if (df->off[i + SKIP_ONE] == df->off[i]) { + continue; + } + for (const char *p = strchr(path, '/'); p; p = strchr(p + SKIP_ONE, '/')) { + size_t n = (size_t)(p - path); + char *key = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, n + SKIP_ONE); + if (!key) { + doc_code_dirs_free(dirs); + return NULL; + } + memcpy(key, path, n); + key[n] = '\0'; + if (cbm_ht_has(dirs, key)) { + cbm_free(CBM_MEM_CLASS_SEMANTIC, key); + } else { + cbm_ht_set(dirs, key, (void *)(intptr_t)SKIP_ONE); + } + } + } + return dirs; +} + +/* A candidate's position: inside the home (or the home folder itself), + * outside it, or no position (the home is the repository root). */ +static cbm_sem_doc_pos_t doc_position(const char *home, const char *path) { + if (!home || !home[0]) { + return CBM_SEM_DOC_POS_GLOBAL; + } + size_t n = strlen(home); + return path && strncmp(path, home, n) == 0 && (path[n] == '/' || path[n] == '\0') + ? CBM_SEM_DOC_POS_LOCAL + : CBM_SEM_DOC_POS_OUTSIDE; +} + +static const char *doc_position_name(cbm_sem_doc_pos_t pos) { + return pos == CBM_SEM_DOC_POS_LOCAL ? "local" + : pos == CBM_SEM_DOC_POS_OUTSIDE ? "outside" + : "global"; +} + +/* Per section: its doc's home, whether it is its doc's first section (the + * one a folder candidate starts from), and the Folder nodes by path. */ +typedef struct { + char **home; + bool *first; + CBMHashTable *folders; /* folder path (graph-owned) -> Folder node */ +} doc_place_t; + +static void doc_place_free(doc_place_t *pl, int nsec) { + for (int s = 0; pl->home && s < nsec; s++) { + cbm_free(CBM_MEM_CLASS_SEMANTIC, pl->home[s]); + } + cbm_free(CBM_MEM_CLASS_SEMANTIC, pl->home); + cbm_free(CBM_MEM_CLASS_SEMANTIC, pl->first); + if (pl->folders) { + cbm_ht_free(pl->folders); + } + memset(pl, 0, sizeof(*pl)); +} + +static bool doc_place_build(const cbm_gbuf_t *gbuf, const cbm_gbuf_node_t **secs, int nsec, + const doc_files_t *df, doc_place_t *pl) { + memset(pl, 0, sizeof(*pl)); + pl->home = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)nsec * sizeof(*pl->home)); + pl->first = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)nsec * sizeof(*pl->first)); + pl->folders = cbm_ht_create_in(CBM_MEM_CLASS_SEMANTIC, CBM_SZ_256); + CBMHashTable *firsts = cbm_ht_create_in(CBM_MEM_CLASS_SEMANTIC, CBM_SZ_256); + CBMHashTable *dirs = doc_code_dirs(df); + bool ok = pl->home && pl->first && pl->folders && firsts && dirs; + for (int s = 0; ok && s < nsec; s++) { + pl->home[s] = doc_home(secs[s]->file_path, dirs); + ok = pl->home[s] != NULL; + const char *path = secs[s]->file_path; + if (ok && path) { + int prev = (int)(intptr_t)cbm_ht_get(firsts, path) - SKIP_ONE; + if (prev < 0 || secs[s]->start_line < secs[prev]->start_line) { + cbm_ht_set(firsts, path, (void *)(intptr_t)(s + SKIP_ONE)); + } + } + } + for (int s = 0; ok && s < nsec; s++) { + const char *path = secs[s]->file_path; + pl->first[s] = path && (int)(intptr_t)cbm_ht_get(firsts, path) == s + SKIP_ONE; + } + const cbm_gbuf_node_t **folders = NULL; + int nfold = 0; + if (ok && cbm_gbuf_find_by_label(gbuf, "Folder", &folders, &nfold) == 0) { + for (int i = 0; i < nfold; i++) { + if (folders[i]->file_path && folders[i]->file_path[0]) { + cbm_ht_set(pl->folders, folders[i]->file_path, (void *)folders[i]); + } + } + } + if (firsts) { + cbm_ht_free(firsts); + } + doc_code_dirs_free(dirs); + return ok; +} + typedef struct { const cbm_sem_func_t *funcs; int func_count; @@ -2139,6 +2598,10 @@ typedef struct { const bool *is_test; const int *name_off; /* per function: its name tokens' corpus indices */ const int *name_idx; + const char *const *fpath; /* per function: its file */ + const char *const *home; /* per section: its doc's home ("" = the root) */ + const doc_files_t *files; + int file_key_df; doc_result_t *res; _Atomic int next; } doc_ctx_t; @@ -2169,10 +2632,73 @@ static void doc_name_evidence(const doc_ctx_t *dc, const int *sidx, int ns, doc_ } } +/* The section's best CBM_SEM_DOC_FILE_K whole files, retrieved by the + * section's key terms over the files' postings, scored by cosine. */ +static void doc_section_files(doc_ctx_t *dc, int s, const int *sidx, const float *sw, int ns, + float snorm, int *fstamp, doc_cand_t *fc) { + const doc_files_t *df = dc->files; + doc_result_t *r = &dc->res[s]; + int nc = 0; + for (int k = 0; k < ns && snorm > 0.0F; k++) { + int t = sidx[k]; + if (df->post_off[t + SKIP_ONE] - df->post_off[t] > dc->file_key_df) { + continue; + } + for (int p = df->post_off[t]; p < df->post_off[t + SKIP_ONE]; p++) { + int f = df->post_f[p]; + if (fstamp[f] != s) { + fstamp[f] = s; + fc[nc++] = (doc_cand_t){.func = f}; + } + } + } + for (int c = 0; c < nc; c++) { + int f = fc[c].func; + int len = df->off[f + SKIP_ONE] - df->off[f]; + float dot = doc_terms_dot(sidx, sw, ns, df->idx + df->off[f], df->w + df->off[f], len, + &fc[c].shared); + float denom = snorm * df->norm[f]; + fc[c].score = denom > 0.0F ? dot / denom : 0.0F; + } + if (nc > 1) { + qsort(fc, (size_t)nc, sizeof(fc[0]), cmp_doc_cand); + } + for (int c = 0; c < nc && c < CBM_SEM_DOC_FILE_K; c++) { + r->files[r->nfiles++] = (doc_file_hit_t){ + .file = fc[c].func, .tfidf = fc[c].score, .rank = c + SKIP_ONE, .shared = fc[c].shared}; + } +} + +/* Up to CBM_SEM_DOC_LOCAL_K functions inside the section's home that rank + * below the top CBM_SEM_DOC_TOP_K, at CBM_SEM_DOC_MIN_SCORE or more (cands + * sorted best first). */ +static void doc_section_local(doc_ctx_t *dc, int s, const doc_cand_t *cands, int nc, + const int *sidx, int ns) { + doc_result_t *r = &dc->res[s]; + const char *home = dc->home[s]; + for (int c = CBM_SEM_DOC_TOP_K; home && home[0] && c < nc && r->nlocal < CBM_SEM_DOC_LOCAL_K; + c++) { + if (cands[c].score < CBM_SEM_DOC_MIN_SCORE) { + break; + } + if (doc_position(home, dc->fpath[cands[c].func]) != CBM_SEM_DOC_POS_LOCAL) { + continue; + } + doc_hit_t *hit = &r->local[r->nlocal++]; + *hit = (doc_hit_t){.func = cands[c].func, + .tfidf = cands[c].score, + .rank = c + SKIP_ONE, + .shared = cands[c].shared}; + doc_name_evidence(dc, sidx, ns, hit); + } +} + static void doc_section_one(doc_ctx_t *dc, int s, int *stamp, doc_cand_t *cands, int *sidx, - float *sw, char *body) { + float *sw, char *body, int *fstamp, doc_cand_t *fc) { doc_result_t *r = &dc->res[s]; r->n = 0; + r->nlocal = 0; + r->nfiles = 0; if (!doc_section_gated(dc->secs[s], body)) { return; } @@ -2215,100 +2741,209 @@ static void doc_section_one(doc_ctx_t *dc, int s, int *stamp, doc_cand_t *cands, .shared = cands[c].shared}; doc_name_evidence(dc, sidx, ns, hit); } + doc_section_local(dc, s, cands, nc, sidx, ns); + doc_section_files(dc, s, sidx, sw, ns, snorm, fstamp, fc); } static void doc_section_worker(int worker_id, void *ctx_ptr) { (void)worker_id; doc_ctx_t *dc = ctx_ptr; + int nf = dc->files->count; int *stamp = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)dc->func_count * sizeof(int)); doc_cand_t *cands = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)dc->func_count * sizeof(doc_cand_t)); + int *fstamp = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)nf + SKIP_ONE) * sizeof(int)); + doc_cand_t *fc = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)nf + SKIP_ONE) * sizeof(*fc)); int *sidx = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)SEM_DOC_MAX_TOKENS * sizeof(int)); float *sw = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)SEM_DOC_MAX_TOKENS * sizeof(float)); char *body = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, SEM_DOC_TEXT_MAX); - if (stamp && cands && sidx && sw && body) { + if (stamp && cands && fstamp && fc && sidx && sw && body) { for (int f = 0; f < dc->func_count; f++) { stamp[f] = -1; } + for (int f = 0; f < nf; f++) { + fstamp[f] = -1; + } while (true) { int s = atomic_fetch_add_explicit(&dc->next, SKIP_ONE, memory_order_relaxed); if (s >= dc->nsec) { break; } - doc_section_one(dc, s, stamp, cands, sidx, sw, body); + doc_section_one(dc, s, stamp, cands, sidx, sw, body, fstamp, fc); } } cbm_free(CBM_MEM_CLASS_SEMANTIC, stamp); cbm_free(CBM_MEM_CLASS_SEMANTIC, cands); + cbm_free(CBM_MEM_CLASS_SEMANTIC, fstamp); + cbm_free(CBM_MEM_CLASS_SEMANTIC, fc); cbm_free(CBM_MEM_CLASS_SEMANTIC, sidx); cbm_free(CBM_MEM_CLASS_SEMANTIC, sw); cbm_free(CBM_MEM_CLASS_SEMANTIC, body); } -static void doc_section_lines(FILE *out, const cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *sec, - const cbm_sem_func_t *funcs, const doc_result_t *r) { - for (int h = 0; h < r->n; h++) { - const doc_hit_t *hit = &r->hits[h]; - const cbm_gbuf_node_t *fn = cbm_gbuf_find_by_id(gbuf, funcs[hit->func].node_id); - (void)fprintf(out, "%s\t%s\t%.6f\t%d\t%d\t%s:%d-%d\t%s:%d-%d\t%d\t%.6f\n", - sec->qualified_name ? sec->qualified_name : "?", - fn && fn->qualified_name ? fn->qualified_name : "?", (double)hit->tfidf, - hit->rank, hit->shared, sec->file_path ? sec->file_path : "?", - sec->start_line, sec->end_line, fn && fn->file_path ? fn->file_path : "?", - fn ? fn->start_line : 0, fn ? fn->end_line : 0, hit->name_shared, - (double)hit->name_idf); +static const char *const DOC_KIND_NAME[CBM_SEM_DOC_KIND_COUNT] = {"function", "local", "file", + "folder"}; + +/* The doc's home Folder node, for the doc's first section (NULL: another + * section, or the home is the repository root). */ +static const cbm_gbuf_node_t *doc_home_folder(const doc_place_t *pl, int s) { + return pl->first[s] && pl->home[s][0] ? cbm_ht_get(pl->folders, pl->home[s]) : NULL; +} + +static cbm_sem_doc_pos_t doc_target_position(const char *home, const cbm_gbuf_node_t *target, + cbm_sem_doc_kind_t kind) { + return kind == CBM_SEM_DOC_KIND_FOLDER ? CBM_SEM_DOC_POS_LOCAL + : doc_position(home, target->file_path); +} + +/* One CBM_SEM_DOC_SIGNALS line: section and target qualified names, tfidf, + * rank, shared terms, both locations, the name evidence, then the kind, the + * position and the doc's home ("." = the repository root). */ +static void doc_signal_line(FILE *out, const cbm_gbuf_node_t *sec, const cbm_gbuf_node_t *t, + const doc_hit_t *hit, cbm_sem_doc_kind_t kind, const char *home) { + (void)fprintf(out, "%s\t%s\t%.6f\t%d\t%d\t%s:%d-%d\t%s:%d-%d\t%d\t%.6f\t%s\t%s\t%s\n", + sec->qualified_name ? sec->qualified_name : "?", + t && t->qualified_name ? t->qualified_name : "?", (double)hit->tfidf, hit->rank, + hit->shared, sec->file_path ? sec->file_path : "?", sec->start_line, + sec->end_line, t && t->file_path ? t->file_path : "?", t ? t->start_line : 0, + t ? t->end_line : 0, hit->name_shared, (double)hit->name_idf, DOC_KIND_NAME[kind], + t ? doc_position_name(doc_target_position(home, t, kind)) : "?", + home[0] ? home : "."); +} + +static void doc_section_lines(FILE *out, const cbm_gbuf_t *gbuf, const doc_ctx_t *dc, + const doc_place_t *pl, int s) { + const cbm_gbuf_node_t *sec = dc->secs[s]; + const doc_result_t *r = &dc->res[s]; + for (int h = 0; h < r->n + r->nlocal; h++) { + const doc_hit_t *hit = h < r->n ? &r->hits[h] : &r->local[h - r->n]; + const cbm_gbuf_node_t *fn = cbm_gbuf_find_by_id(gbuf, dc->funcs[hit->func].node_id); + doc_signal_line(out, sec, fn, hit, + h < r->n ? CBM_SEM_DOC_KIND_FUNCTION : CBM_SEM_DOC_KIND_LOCAL, pl->home[s]); + } + for (int h = 0; h < r->nfiles; h++) { + const doc_file_hit_t *fh = &r->files[h]; + doc_hit_t hit = {.tfidf = fh->tfidf, .rank = fh->rank, .shared = fh->shared}; + doc_signal_line(out, sec, dc->files->node[fh->file], &hit, CBM_SEM_DOC_KIND_FILE, + pl->home[s]); + } + const cbm_gbuf_node_t *folder = doc_home_folder(pl, s); + if (folder) { + doc_hit_t hit = {.rank = SKIP_ONE}; + doc_signal_line(out, sec, folder, &hit, CBM_SEM_DOC_KIND_FOLDER, pl->home[s]); } } -/* The stored candidates of every section (rank <= CBM_SEM_DOC_TOP_K, tfidf >= - * CBM_SEM_DOC_MIN_SCORE), in CBM_MEM_CLASS_STORE for cbm_store_doc_candidates_free; p at two - * decimals, as on SEMANTICALLY_RELATED edges. */ -static cbm_doc_candidate_t *doc_candidate_rows(const cbm_gbuf_t *gbuf, const cbm_gbuf_node_t **secs, - int nsec, const cbm_sem_func_t *funcs, - const doc_result_t *res, int *out_count) { +/* Does the section already link the target exactly (a MENTIONS edge)? A + * candidate for that pair would only repeat a fact the graph holds. */ +static bool doc_pair_linked(const cbm_gbuf_t *gbuf, int64_t sec_id, int64_t fn_id) { + const cbm_gbuf_edge_t **edges = NULL; int n = 0; - for (int s = 0; s < nsec; s++) { - for (int h = 0; h < res[s].n; h++) { - n += res[s].hits[h].rank <= CBM_SEM_DOC_TOP_K && - res[s].hits[h].tfidf >= CBM_SEM_DOC_MIN_SCORE; + if (cbm_gbuf_find_edges_by_source_type(gbuf, sec_id, "MENTIONS", &edges, &n) != 0) { + return false; + } + for (int i = 0; i < n; i++) { + if (edges[i]->target_id == fn_id) { + return true; + } + } + return false; +} + +typedef struct { + cbm_doc_candidate_t *rows; + int n; +} doc_rows_t; + +/* Append the candidate (section -> target) when its judged p is above 0 and + * the graph does not link the pair exactly; false: out of memory (the row at + * rw->n may hold partial strings). */ +static bool doc_row_add(doc_rows_t *rw, const cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *sec, + const cbm_gbuf_node_t *target, const doc_hit_t *hit, + cbm_sem_doc_kind_t kind, const char *home) { + if (!target || !target->qualified_name || !sec->qualified_name) { + return true; + } + cbm_sem_doc_pos_t pos = doc_target_position(home, target, kind); + cbm_sem_doc_format_t fmt = cbm_sem_doc_format(sec->file_path); + float p = kind == CBM_SEM_DOC_KIND_FOLDER + ? cbm_sem_doc_folder_p(fmt, sec->file_path) + : cbm_sem_doc_calibrated_p(fmt, kind, pos, hit->tfidf); + if (p <= 0.0F || doc_pair_linked(gbuf, sec->id, target->id)) { + return true; /* not stored for this band, kind or format; or linked exactly */ + } + char ev[CBM_SZ_128]; + snprintf(ev, sizeof(ev), + "{\"shared_terms\":%d,\"name_tokens\":%d,\"kind\":\"%s\",\"position\":\"%s\"}", + hit->shared, hit->name_shared, DOC_KIND_NAME[kind], doc_position_name(pos)); + rw->rows[rw->n] = (cbm_doc_candidate_t){ + .section_qn = cbm_mem_strdup(CBM_MEM_CLASS_STORE, sec->qualified_name), + .target_qn = cbm_mem_strdup(CBM_MEM_CLASS_STORE, target->qualified_name), + .rank = hit->rank, + .score = hit->tfidf, + .p = cbm_sem_p_2dp(p), + .evidence = cbm_mem_strdup(CBM_MEM_CLASS_STORE, ev)}; + const cbm_doc_candidate_t *row = &rw->rows[rw->n]; + if (!row->section_qn || !row->target_qn || !row->evidence) { + return false; + } + rw->n++; + return true; +} + +static bool doc_section_rows(doc_rows_t *rw, const cbm_gbuf_t *gbuf, const doc_ctx_t *dc, + const doc_place_t *pl, int s) { + const cbm_gbuf_node_t *sec = dc->secs[s]; + const doc_result_t *r = &dc->res[s]; + bool ok = true; + for (int h = 0; ok && h < r->n + r->nlocal; h++) { + const doc_hit_t *hit = h < r->n ? &r->hits[h] : &r->local[h - r->n]; + if (h < r->n && (hit->rank > CBM_SEM_DOC_TOP_K || hit->tfidf < CBM_SEM_DOC_MIN_SCORE)) { + continue; } + ok = + doc_row_add(rw, gbuf, sec, cbm_gbuf_find_by_id(gbuf, dc->funcs[hit->func].node_id), hit, + h < r->n ? CBM_SEM_DOC_KIND_FUNCTION : CBM_SEM_DOC_KIND_LOCAL, pl->home[s]); + } + for (int h = 0; ok && h < r->nfiles; h++) { + const doc_file_hit_t *fh = &r->files[h]; + doc_hit_t hit = {.tfidf = fh->tfidf, .rank = fh->rank, .shared = fh->shared}; + ok = doc_row_add(rw, gbuf, sec, dc->files->node[fh->file], &hit, CBM_SEM_DOC_KIND_FILE, + pl->home[s]); + } + doc_hit_t first = {.rank = SKIP_ONE}; + return ok && doc_row_add(rw, gbuf, sec, doc_home_folder(pl, s), &first, CBM_SEM_DOC_KIND_FOLDER, + pl->home[s]); +} + +/* The stored candidates of every section -- its best CBM_SEM_DOC_TOP_K + * functions at CBM_SEM_DOC_MIN_SCORE or more, its local functions below them, + * its best files, and (for a doc's first section) the doc's home folder -- + * each with its judged p, in CBM_MEM_CLASS_STORE for + * cbm_store_doc_candidates_free; p at two decimals, as on + * SEMANTICALLY_RELATED edges. */ +static cbm_doc_candidate_t *doc_candidate_rows(const cbm_gbuf_t *gbuf, const doc_ctx_t *dc, + const doc_place_t *pl, int *out_count) { + int n = 0; + for (int s = 0; s < dc->nsec; s++) { + const doc_result_t *r = &dc->res[s]; + n += (r->n < CBM_SEM_DOC_TOP_K ? r->n : CBM_SEM_DOC_TOP_K) + r->nlocal + r->nfiles + + SKIP_ONE; } *out_count = 0; - cbm_doc_candidate_t *rows = - n > 0 ? cbm_calloc(CBM_MEM_CLASS_STORE, (size_t)n * sizeof(*rows)) : NULL; - if (!rows) { + doc_rows_t rw = {.rows = n > 0 ? cbm_calloc(CBM_MEM_CLASS_STORE, (size_t)n * sizeof(*rw.rows)) + : NULL}; + if (!rw.rows) { return NULL; } - int k = 0; - for (int s = 0; s < nsec; s++) { - for (int h = 0; h < res[s].n; h++) { - const doc_hit_t *hit = &res[s].hits[h]; - if (hit->rank > CBM_SEM_DOC_TOP_K || hit->tfidf < CBM_SEM_DOC_MIN_SCORE) { - continue; - } - const cbm_gbuf_node_t *fn = cbm_gbuf_find_by_id(gbuf, funcs[hit->func].node_id); - if (!fn || !fn->qualified_name || !secs[s]->qualified_name) { - continue; - } - char ev[CBM_SZ_64]; - snprintf(ev, sizeof(ev), "{\"shared_terms\":%d,\"name_tokens\":%d}", hit->shared, - hit->name_shared); - rows[k] = (cbm_doc_candidate_t){ - .section_qn = cbm_mem_strdup(CBM_MEM_CLASS_STORE, secs[s]->qualified_name), - .target_qn = cbm_mem_strdup(CBM_MEM_CLASS_STORE, fn->qualified_name), - .rank = hit->rank, - .score = hit->tfidf, - .p = (double)(int)(cbm_sem_doc_calibrated_p(hit->tfidf) * 100.0F + 0.5F) / 100.0, - .evidence = cbm_mem_strdup(CBM_MEM_CLASS_STORE, ev)}; - if (!rows[k].section_qn || !rows[k].target_qn || !rows[k].evidence) { - cbm_store_doc_candidates_free(rows, k + SKIP_ONE); - return NULL; - } - k++; + for (int s = 0; s < dc->nsec; s++) { + if (!doc_section_rows(&rw, gbuf, dc, pl, s)) { + cbm_store_doc_candidates_free(rw.rows, rw.n + SKIP_ONE); + return NULL; } } - *out_count = k; - return rows; + *out_count = rw.n; + return rw.rows; } static void phase6c_doc_sections(cbm_pipeline_ctx_t *ctx, const cbm_sem_func_t *funcs, @@ -2332,10 +2967,15 @@ static void phase6c_doc_sections(cbm_pipeline_ctx_t *ctx, const cbm_sem_func_t * int *name_idx = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)func_count * SEM_DOC_NAME_TOKENS + SKIP_ONE) * sizeof(int)); doc_result_t *res = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)nsec * sizeof(doc_result_t)); + const char **fn_paths = + cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)func_count * sizeof(*fn_paths)); + int *ffile = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)func_count * sizeof(int)); + doc_files_t files = {0}; + doc_place_t place = {0}; int *post_f = NULL; int *cursor = NULL; FILE *out = NULL; - if (secs && post_off && fnorm && is_test && name_off && name_idx && res) { + if (secs && post_off && fnorm && is_test && name_off && name_idx && res && fn_paths && ffile) { for (int f = 0; f < func_count; f++) { for (int k = 0; k < funcs[f].tfidf_len; k++) { post_off[funcs[f].tfidf_indices[k] + SKIP_ONE]++; @@ -2343,6 +2983,7 @@ static void phase6c_doc_sections(cbm_pipeline_ctx_t *ctx, const cbm_sem_func_t * fnorm[f] = terms_norm(funcs[f].tfidf_weights, funcs[f].tfidf_len); const cbm_gbuf_node_t *fn = cbm_gbuf_find_by_id(gbuf, funcs[f].node_id); is_test[f] = doc_is_test_function(fn); + fn_paths[f] = fn && fn->file_path ? fn->file_path : ""; char *tok[SEM_DOC_NAME_TOKENS]; int nt = fn && fn->name ? cbm_sem_tokenize(fn->name, tok, SEM_DOC_NAME_TOKENS) : 0; int base = name_off[f]; @@ -2374,12 +3015,22 @@ static void phase6c_doc_sections(cbm_pipeline_ctx_t *ctx, const cbm_sem_func_t * post_f[post_off[t] + cursor[t]++] = f; } } + memcpy(secs, found, (size_t)nsec * sizeof(*secs)); + qsort(secs, (size_t)nsec, sizeof(*secs), cmp_section_node); + } + bool placed = + post_f && cursor && + doc_files_build(gbuf, funcs, func_count, is_test, corpus, nterms, &files, ffile) && + doc_place_build(gbuf, secs, nsec, &files, &place); + if (placed) { int key_df = func_count / SEM_DOC_KEY_DF_DIV; if (key_df < SEM_DOC_KEY_DF_MIN) { key_df = SEM_DOC_KEY_DF_MIN; } - memcpy(secs, found, (size_t)nsec * sizeof(*secs)); - qsort(secs, (size_t)nsec, sizeof(*secs), cmp_section_node); + int file_key_df = files.count / SEM_DOC_KEY_DF_DIV; + if (file_key_df < SEM_DOC_KEY_DF_MIN) { + file_key_df = SEM_DOC_KEY_DF_MIN; + } doc_ctx_t dc = {.funcs = funcs, .func_count = func_count, .corpus = corpus, @@ -2392,21 +3043,25 @@ static void phase6c_doc_sections(cbm_pipeline_ctx_t *ctx, const cbm_sem_func_t * .is_test = is_test, .name_off = name_off, .name_idx = name_idx, + .fpath = fn_paths, + .home = (const char *const *)place.home, + .files = &files, + .file_key_df = file_key_df, .res = res}; atomic_init(&dc.next, 0); cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false}; cbm_parallel_for(worker_count, doc_section_worker, &dc, opts); int row_count = 0; - cbm_doc_candidate_t *rows = doc_candidate_rows(gbuf, secs, nsec, funcs, res, &row_count); + cbm_doc_candidate_t *rows = doc_candidate_rows(gbuf, &dc, &place, &row_count); cbm_pipeline_set_doc_candidates(ctx->pipeline, rows, row_count); - cbm_log_info("pass.semantic.doc_candidates", "sections", itoa_log(nsec), "rows", - itoa_log(row_count)); + cbm_log_info("pass.semantic.doc_candidates", "sections", itoa_log(nsec), "files", + itoa_log(files.count), "rows", itoa_log(row_count)); char path[CBM_SZ_1K]; out = cbm_safe_getenv("CBM_SEM_DOC_SIGNALS", path, sizeof(path), NULL) ? cbm_fopen(path, "w") : NULL; for (int s = 0; out && s < nsec; s++) { - doc_section_lines(out, gbuf, secs[s], funcs, &res[s]); + doc_section_lines(out, gbuf, &dc, &place, s); } /* CBM_SEM_DOC_FUNCS=: every function with its location and the * test flag, for measuring candidates the TF-IDF generator does not @@ -2438,6 +3093,10 @@ static void phase6c_doc_sections(cbm_pipeline_ctx_t *ctx, const cbm_sem_func_t * cbm_free(CBM_MEM_CLASS_SEMANTIC, name_off); cbm_free(CBM_MEM_CLASS_SEMANTIC, name_idx); cbm_free(CBM_MEM_CLASS_SEMANTIC, res); + cbm_free(CBM_MEM_CLASS_SEMANTIC, fn_paths); + cbm_free(CBM_MEM_CLASS_SEMANTIC, ffile); + doc_place_free(&place, nsec); + doc_files_free(&files); } int cbm_pipeline_pass_semantic_edges(cbm_pipeline_ctx_t *ctx) { diff --git a/src/semantic/semantic.c b/src/semantic/semantic.c index bbdd57b56..af738151e 100644 --- a/src/semantic/semantic.c +++ b/src/semantic/semantic.c @@ -1866,16 +1866,111 @@ float cbm_sem_calibrated_p(float score) { * candidates -- a section's best five non-test functions -- from five held-out * repositories, three reading rounds; the top three measured bands pooled to * stay monotone). Below the first cut (judged 0.06) nothing is stored. */ -static const float SEM_DOC_P_CUTS[] = {CBM_SEM_DOC_MIN_SCORE, 0.30F, 0.40F}; -static const float SEM_DOC_P_BANDS[] = {0.06F, 0.36F, 0.42F, 0.548F}; +enum { SEM_DOC_P_NBANDS = 4 }; +static const float SEM_DOC_P_CUTS[SEM_DOC_P_NBANDS - 1] = {CBM_SEM_DOC_MIN_SCORE, 0.30F, 0.40F}; +/* Per format, p below the first cut and in each band; 0 = not stored (a + * band judged below 0.20). Blind-judged, three reading rounds, pooled where + * a higher band judged lower: + * Markdown by kind and position since SEM7: SEM_DOC_P_MD_* below (its + * first curve, 2026-10-07: 274 candidates of five repositories, + * 0.06 / 0.36 / 0.42 / 0.55, did not separate positions). + * reST 2026-10-08, 120 candidates, three Sphinx projects; 0.20-0.30 + * judged 0.125, so reST starts at 0.30. + * AsciiDoc 2026-10-08, 114 candidates, one repository (the only one of + * its search that qualified). + * PDF no qualifying repository: not measured, nothing stored. */ +static const float SEM_DOC_P_BANDS[CBM_SEM_DOC_FMT_COUNT][SEM_DOC_P_NBANDS] = { + [CBM_SEM_DOC_FMT_OTHER] = {0.0F, 0.0F, 0.0F, 0.0F}, + [CBM_SEM_DOC_FMT_MARKDOWN] = {0.0F, 0.0F, 0.0F, 0.0F}, /* SEM_DOC_P_MD_* */ + [CBM_SEM_DOC_FMT_RST] = {0.0F, 0.0F, 0.35F, 0.625F}, + [CBM_SEM_DOC_FMT_ADOC] = {0.0F, 0.225F, 0.25F, 0.50F}, + [CBM_SEM_DOC_FMT_PDF] = {0.0F, 0.0F, 0.0F, 0.0F}, +}; + +cbm_sem_doc_format_t cbm_sem_doc_format(const char *path) { + static const struct { + const char *ext; + cbm_sem_doc_format_t fmt; + } exts[] = {{".md", CBM_SEM_DOC_FMT_MARKDOWN}, {".mdx", CBM_SEM_DOC_FMT_MARKDOWN}, + {".markdown", CBM_SEM_DOC_FMT_MARKDOWN}, {".rst", CBM_SEM_DOC_FMT_RST}, + {".rest", CBM_SEM_DOC_FMT_RST}, {".txt", CBM_SEM_DOC_FMT_RST}, + {".adoc", CBM_SEM_DOC_FMT_ADOC}, {".asciidoc", CBM_SEM_DOC_FMT_ADOC}, + {".asc", CBM_SEM_DOC_FMT_ADOC}, {".pdf", CBM_SEM_DOC_FMT_PDF}}; + const char *dot = path ? strrchr(path, '.') : NULL; + if (!dot || strchr(dot, '/')) { + return CBM_SEM_DOC_FMT_OTHER; + } + for (size_t i = 0; i < sizeof(exts) / sizeof(exts[0]); i++) { + if (strcasecmp(dot, exts[i].ext) == 0) { + return exts[i].fmt; + } + } + return CBM_SEM_DOC_FMT_OTHER; +} + +double cbm_sem_p_2dp(float p) { + /* 1e-4 on the x100 scale: a float holds 0.325 as 0.32499998..., 1.2e-6 + * below the half; judged values have at most three decimals, so the + * nearest one that must round down (x.xx4) is 0.1 away */ + return (double)(long)((double)p * 100.0 + 0.5 + 1e-4) / 100.0; +} + +/* Markdown by kind and position (SEM7, 2026-10-08: 490 blind-judged + * candidates from four monorepos with per-component docs, three reading + * rounds, pooled where a higher band judged lower). A doc's position decides + * most: functions outside the doc's home were right 6 of 90 (none stored); + * inside it 0.18 / 0.33 / 0.53. A doc of the whole project: SEM7's and + * sample 4's judged items pooled (the user's choice; 158 items, nine + * repositories): 0.17 / 0.22 / 0.51. Local extras (home functions below the + * top 5) 19 of 60 = 0.32 at every band; whole files 0.12 below 0.40, then + * 0.30. Below 0.20 nothing. */ +static const float SEM_DOC_P_MD_FUNCTION[CBM_SEM_DOC_POS_COUNT][SEM_DOC_P_NBANDS] = { + [CBM_SEM_DOC_POS_GLOBAL] = {0.0F, 0.0F, 0.217F, 0.509F}, + [CBM_SEM_DOC_POS_LOCAL] = {0.0F, 0.0F, 0.325F, 0.525F}, + [CBM_SEM_DOC_POS_OUTSIDE] = {0.0F, 0.0F, 0.0F, 0.0F}, +}; +static const float SEM_DOC_P_MD_LOCAL[SEM_DOC_P_NBANDS] = {0.0F, 0.317F, 0.317F, 0.317F}; +static const float SEM_DOC_P_MD_FILE[SEM_DOC_P_NBANDS] = {0.0F, 0.0F, 0.0F, 0.30F}; -float cbm_sem_doc_calibrated_p(float score) { +float cbm_sem_doc_calibrated_p(cbm_sem_doc_format_t fmt, cbm_sem_doc_kind_t kind, + cbm_sem_doc_pos_t pos, float score) { + if ((int)fmt < 0 || fmt >= CBM_SEM_DOC_FMT_COUNT || (int)pos < 0 || + pos >= CBM_SEM_DOC_POS_COUNT) { + return 0.0F; + } size_t band = 0; - while (band < sizeof(SEM_DOC_P_CUTS) / sizeof(SEM_DOC_P_CUTS[0]) && - score >= SEM_DOC_P_CUTS[band]) { + while (band < SEM_DOC_P_NBANDS - 1 && score >= SEM_DOC_P_CUTS[band]) { band++; } - return SEM_DOC_P_BANDS[band]; + if (fmt != CBM_SEM_DOC_FMT_MARKDOWN) { + /* measured for function candidates only, without position (SEM6) */ + return kind == CBM_SEM_DOC_KIND_FUNCTION ? SEM_DOC_P_BANDS[fmt][band] : 0.0F; + } + switch (kind) { + case CBM_SEM_DOC_KIND_FUNCTION: + return SEM_DOC_P_MD_FUNCTION[pos][band]; + case CBM_SEM_DOC_KIND_LOCAL: + return pos == CBM_SEM_DOC_POS_LOCAL ? SEM_DOC_P_MD_LOCAL[band] : 0.0F; + case CBM_SEM_DOC_KIND_FILE: + return SEM_DOC_P_MD_FILE[band]; + default: + return 0.0F; /* a folder candidate: cbm_sem_doc_folder_p */ + } +} + +/* A doc's home folder as its candidate (SEM7): a README or index doc was + * right 26 of 30 (0.87); any other doc 5 of 30 (not stored). Markdown only. */ +float cbm_sem_doc_folder_p(cbm_sem_doc_format_t fmt, const char *doc_path) { + if (fmt != CBM_SEM_DOC_FMT_MARKDOWN || !doc_path) { + return 0.0F; + } + const char *base = strrchr(doc_path, '/'); + base = base ? base + 1 : doc_path; + const char *dot = strrchr(base, '.'); + size_t n = dot ? (size_t)(dot - base) : strlen(base); + bool readme = (n == sizeof("readme") - 1 && strncasecmp(base, "readme", n) == 0) || + (n == sizeof("index") - 1 && strncasecmp(base, "index", n) == 0); + return readme ? 0.867F : 0.0F; } float cbm_sem_combined_score(const cbm_sem_func_t *a, const cbm_sem_func_t *b, diff --git a/src/semantic/semantic.h b/src/semantic/semantic.h index e641e2f29..66f4cc0a1 100644 --- a/src/semantic/semantic.h +++ b/src/semantic/semantic.h @@ -270,7 +270,58 @@ float cbm_sem_calibrated_p(float score); * about the function. */ #define CBM_SEM_DOC_TOP_K 5 #define CBM_SEM_DOC_MIN_SCORE 0.20F -float cbm_sem_doc_calibrated_p(float score); + +/* A section's document format, by its file's extension: candidates are + * judged, and stored, per format. */ +typedef enum { + CBM_SEM_DOC_FMT_OTHER = 0, + CBM_SEM_DOC_FMT_MARKDOWN, + CBM_SEM_DOC_FMT_RST, /* .rst, and .txt: a .txt file only has sections as Sphinx source */ + CBM_SEM_DOC_FMT_ADOC, + CBM_SEM_DOC_FMT_PDF, + CBM_SEM_DOC_FMT_COUNT, +} cbm_sem_doc_format_t; +cbm_sem_doc_format_t cbm_sem_doc_format(const char *path); + +/* Where a candidate sits relative to the doc's home folder (the folder whose + * code the doc documents, see pass_semantic_edges.c): the home is the + * repository root (the doc documents the whole project), or the candidate is + * inside or outside the home. */ +typedef enum { + CBM_SEM_DOC_POS_GLOBAL = 0, + CBM_SEM_DOC_POS_LOCAL, + CBM_SEM_DOC_POS_OUTSIDE, + CBM_SEM_DOC_POS_COUNT, +} cbm_sem_doc_pos_t; + +/* What a candidate points at: one of a section's best CBM_SEM_DOC_TOP_K + * functions; a function inside the home below that top K (at most + * CBM_SEM_DOC_LOCAL_K per section); a whole file; the doc's home folder. */ +typedef enum { + CBM_SEM_DOC_KIND_FUNCTION = 0, + CBM_SEM_DOC_KIND_LOCAL, + CBM_SEM_DOC_KIND_FILE, + CBM_SEM_DOC_KIND_FOLDER, + CBM_SEM_DOC_KIND_COUNT, +} cbm_sem_doc_kind_t; +#define CBM_SEM_DOC_LOCAL_K 2 +#define CBM_SEM_DOC_FILE_K 3 + +/* The probability that a section of format fmt is about a candidate of this + * kind and position with this tfidf, from the judged curves; 0 when that band + * is not stored (judged below 0.20) or has no judged curve. A folder + * candidate has no score: cbm_sem_doc_folder_p. */ +float cbm_sem_doc_calibrated_p(cbm_sem_doc_format_t fmt, cbm_sem_doc_kind_t kind, + cbm_sem_doc_pos_t pos, float score); + +/* The probability that the doc at doc_path documents its home folder, by + * the doc's kind (a README or index file, or any other doc); 0 = not + * stored. */ +float cbm_sem_doc_folder_p(cbm_sem_doc_format_t fmt, const char *doc_path); + +/* p as stored and shown: two decimals, half up (0.225 -> 0.23, 0.325 -> + * 0.33; the epsilon absorbs the float representation of a judged x.xx5). */ +double cbm_sem_p_2dp(float p); /* Module proximity multiplier based on file paths. */ float cbm_sem_proximity(const char *path_a, const char *path_b); diff --git a/src/store/store.c b/src/store/store.c index 7a64fd17b..b974e5e39 100644 --- a/src/store/store.c +++ b/src/store/store.c @@ -10808,14 +10808,9 @@ int cbm_store_doc_candidates_replace(cbm_store_t *s, const char *project, return exec_sql(s, "COMMIT;"); } -int cbm_store_doc_candidates_get(cbm_store_t *s, const char *project, const char *section_qn, - const char *target_qn, int limit, cbm_doc_candidate_t **out, - int *count) { - if (!s || !s->db || !project || !out || !count || (!section_qn == !target_qn) || limit <= 0) { - return CBM_STORE_ERR; - } - *out = NULL; - *count = 0; +/* Does the database hold the candidates table? CBM_STORE_NOT_FOUND: no (a + * database written by an older build). */ +static int doc_candidates_table(cbm_store_t *s) { sqlite3_stmt *probe = NULL; if (sqlite3_prepare_v2(s->db, "SELECT 1 FROM sqlite_master WHERE type='table' AND " @@ -10833,19 +10828,15 @@ int cbm_store_doc_candidates_get(cbm_store_t *s, const char *project, const char store_set_error_sqlite(s, "doc_candidates probe step"); return CBM_STORE_ERR; } - const char *sql = section_qn ? "SELECT section_qn, target_qn, rank, score, p, evidence FROM " - "doc_link_candidates WHERE project=?1 AND section_qn=?2 " - "ORDER BY p DESC, score DESC, target_qn LIMIT ?3;" - : "SELECT section_qn, target_qn, rank, score, p, evidence FROM " - "doc_link_candidates WHERE project=?1 AND target_qn=?2 " - "ORDER BY p DESC, score DESC, section_qn LIMIT ?3;"; - sqlite3_stmt *stmt = NULL; - if (sqlite3_prepare_v2(s->db, sql, CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) { - store_set_error_sqlite(s, "doc_candidates get"); - return CBM_STORE_ERR; - } + return CBM_STORE_OK; +} + +/* Run a prepared candidates query (project ?1, key ?2, limit ?3; columns + * section_qn, target_qn, rank, score, p, evidence) into *out; finalizes it. */ +static int doc_candidates_read(sqlite3_stmt *stmt, const char *project, const char *key, int limit, + cbm_doc_candidate_t **out, int *count) { bind_text(stmt, SKIP_ONE, project); - bind_text(stmt, ST_COL_2, section_qn ? section_qn : target_qn); + bind_text(stmt, ST_COL_2, key); sqlite3_bind_int(stmt, ST_COL_3, limit); cbm_doc_candidate_t *rows = cbm_calloc(CBM_MEM_CLASS_STORE, (size_t)limit * sizeof(*rows)); if (!rows) { @@ -10871,6 +10862,53 @@ int cbm_store_doc_candidates_get(cbm_store_t *s, const char *project, const char return CBM_STORE_OK; } +static int doc_candidates_query(cbm_store_t *s, const char *sql, const char *project, + const char *key, int limit, cbm_doc_candidate_t **out, int *count) { + *out = NULL; + *count = 0; + int rc = doc_candidates_table(s); + if (rc != CBM_STORE_OK) { + return rc; + } + sqlite3_stmt *stmt = NULL; + if (sqlite3_prepare_v2(s->db, sql, CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) { + store_set_error_sqlite(s, "doc_candidates get"); + return CBM_STORE_ERR; + } + return doc_candidates_read(stmt, project, key, limit, out, count); +} + +int cbm_store_doc_candidates_get(cbm_store_t *s, const char *project, const char *section_qn, + const char *target_qn, int limit, cbm_doc_candidate_t **out, + int *count) { + if (!s || !s->db || !project || !out || !count || (!section_qn == !target_qn) || limit <= 0) { + return CBM_STORE_ERR; + } + const char *sql = section_qn ? "SELECT section_qn, target_qn, rank, score, p, evidence FROM " + "doc_link_candidates WHERE project=?1 AND section_qn=?2 " + "ORDER BY p DESC, score DESC, target_qn LIMIT ?3;" + : "SELECT section_qn, target_qn, rank, score, p, evidence FROM " + "doc_link_candidates WHERE project=?1 AND target_qn=?2 " + "ORDER BY p DESC, score DESC, section_qn LIMIT ?3;"; + return doc_candidates_query(s, sql, project, section_qn ? section_qn : target_qn, limit, out, + count); +} + +int cbm_store_doc_candidates_for_path(cbm_store_t *s, const char *project, const char *file_path, + int limit, cbm_doc_candidate_t **out, int *count) { + if (!s || !s->db || !project || !file_path || !file_path[0] || !out || !count || limit <= 0) { + return CBM_STORE_ERR; + } + static const char SQL[] = "SELECT c.section_qn, c.target_qn, c.rank, c.score, c.p, c.evidence " + "FROM doc_link_candidates c JOIN nodes n " + "ON n.project = c.project AND n.qualified_name = c.target_qn " + "WHERE c.project=?1 AND ((n.label='File' AND n.file_path=?2) OR " + "(n.label='Folder' AND n.file_path <> '' AND " + "substr(?2, 1, length(n.file_path) + 1) = n.file_path || '/')) " + "ORDER BY c.p DESC, c.score DESC, c.section_qn LIMIT ?3;"; + return doc_candidates_query(s, SQL, project, file_path, limit, out, count); +} + void cbm_store_doc_candidates_free(cbm_doc_candidate_t *rows, int count) { for (int i = 0; rows && i < count; i++) { cbm_free(CBM_MEM_CLASS_STORE, (void *)rows[i].section_qn); diff --git a/src/store/store.h b/src/store/store.h index e7fb3142b..2a8d64048 100644 --- a/src/store/store.h +++ b/src/store/store.h @@ -1110,6 +1110,12 @@ int cbm_store_doc_candidates_replace(cbm_store_t *s, const char *project, int cbm_store_doc_candidates_get(cbm_store_t *s, const char *project, const char *section_qn, const char *target_qn, int limit, cbm_doc_candidate_t **out, int *count); + +/* The candidates that point at a file as a whole: at its File node, or at a + * Folder node that holds it (a doc's home folder), highest p first, then + * score, at most limit. Allocation and CBM_STORE_NOT_FOUND as above. */ +int cbm_store_doc_candidates_for_path(cbm_store_t *s, const char *project, const char *file_path, + int limit, cbm_doc_candidate_t **out, int *count); void cbm_store_doc_candidates_free(cbm_doc_candidate_t *rows, int count); /* ADR section parsing/rendering (pure functions, no store needed) */ diff --git a/tests/test_mcp.c b/tests/test_mcp.c index 6ae9a0814..abd19c124 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -2546,6 +2546,98 @@ TEST(tool_get_file_outline_returns_bounded_filtered_columnar_rows_issue469) { PASS(); } +/* get_file_outline shows the sections possibly about the file as a whole or + * about a folder holding it (highest p first, kind shown), in both formats, + * on the first page only; a candidate for a function in the file is not one + * of them. */ +TEST(tool_get_file_outline_shows_doc_candidates_of_file_and_folders) { + cbm_mcp_server_t *srv = setup_mcp_with_data(); + cbm_store_t *store = cbm_mcp_server_store(srv); + ASSERT_NOT_NULL(store); + ASSERT_EQ(cbm_store_upsert_project(store, "doc-outline", "/tmp/doc-outline"), CBM_STORE_OK); + cbm_mcp_server_set_project(srv, "doc-outline"); + const struct { + const char *label, *qn, *path; + int start; + } nodes[] = {{"File", "doc-outline.pkg.mail.parse.__file__", "pkg/mail/parse.py", 0}, + {"Function", "doc-outline.pkg.mail.parse.parse_mail", "pkg/mail/parse.py", 3}, + {"Folder", "doc-outline.pkg.mail", "pkg/mail", 0}, + {"Section", "doc-outline.pkg.mail.README.Mail", "pkg/mail/README.md", 1}, + {"Section", "doc-outline.docs.parsing.Parsing", "docs/parsing.md", 5}}; + for (size_t i = 0; i < sizeof(nodes) / sizeof(nodes[0]); i++) { + cbm_node_t node = {.project = "doc-outline", + .label = nodes[i].label, + .name = nodes[i].qn, + .qualified_name = nodes[i].qn, + .file_path = nodes[i].path, + .start_line = nodes[i].start, + .end_line = nodes[i].start + 2}; + ASSERT_GT(cbm_store_upsert_node(store, &node), 0); + } + const cbm_doc_candidate_t rows[] = { + {.section_qn = "doc-outline.docs.parsing.Parsing", + .target_qn = "doc-outline.pkg.mail.parse.__file__", + .rank = 1, + .score = 0.41, + .p = 0.5, + .evidence = "{\"shared_terms\":4,\"name_tokens\":0,\"kind\":\"file\"," + "\"position\":\"global\"}"}, + {.section_qn = "doc-outline.pkg.mail.README.Mail", + .target_qn = "doc-outline.pkg.mail", + .rank = 1, + .p = 0.8, + .evidence = "{\"shared_terms\":0,\"name_tokens\":0,\"kind\":\"folder\"," + "\"position\":\"local\"}"}, + {.section_qn = "doc-outline.docs.parsing.Parsing", + .target_qn = "doc-outline.pkg.mail.parse.parse_mail", + .rank = 1, + .score = 0.5, + .p = 0.9, + .evidence = "{\"shared_terms\":5,\"name_tokens\":1,\"kind\":\"function\"," + "\"position\":\"global\"}"}, + }; + ASSERT_EQ(cbm_store_doc_candidates_replace(store, "doc-outline", rows, 3), CBM_STORE_OK); + + char *raw = + cbm_mcp_handle_tool(srv, "get_file_outline", + "{\"project\":\"doc-outline\",\"file_path\":\"pkg/mail/parse.py\"," + "\"format\":\"json\"}"); + char *text = extract_text_content(raw); + free(raw); + ASSERT_NOT_NULL(text); + yyjson_doc *doc = yyjson_read(text, strlen(text), 0); + ASSERT_NOT_NULL(doc); + yyjson_val *in = yyjson_obj_get(yyjson_doc_get_root(doc), "possibly_described_in"); + ASSERT_EQ((int)yyjson_arr_size(in), 2); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(yyjson_arr_get(in, 0), "section")), + "doc-outline.pkg.mail.README.Mail"); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(yyjson_arr_get(in, 0), "kind")), "folder"); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(yyjson_arr_get(in, 1), "kind")), "file"); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(yyjson_arr_get(in, 1), "file_path")), + "docs/parsing.md"); + ASSERT_NOT_NULL(yyjson_obj_get(yyjson_doc_get_root(doc), "candidates_note")); + yyjson_doc_free(doc); + free(text); + + raw = cbm_mcp_handle_tool(srv, "get_file_outline", + "{\"project\":\"doc-outline\",\"file_path\":\"pkg/mail/parse.py\"}"); + ASSERT_NOT_NULL(raw); + ASSERT_NOT_NULL(strstr(raw, "possibly_described_in")); + ASSERT_NOT_NULL(strstr(raw, "doc-outline.pkg.mail.README.Mail")); + ASSERT_NOT_NULL(strstr(raw, "folder")); + free(raw); + + raw = cbm_mcp_handle_tool(srv, "get_file_outline", + "{\"project\":\"doc-outline\",\"file_path\":\"pkg/mail/parse.py\"," + "\"offset\":1}"); + ASSERT_NOT_NULL(raw); + ASSERT_NULL(strstr(raw, "possibly_described_in")); + free(raw); + + cbm_mcp_server_free(srv); + PASS(); +} + TEST(tool_get_file_outline_validates_json_path_limit_and_cancel_issue469) { cbm_mcp_server_t *srv = setup_mcp_with_data(); cbm_store_t *store = cbm_mcp_server_store(srv); @@ -7618,9 +7710,10 @@ TEST(tool_trace_call_path_qn_fallback_frees_name_miss) { cbm_store_free_nodes(by_name, by_name_count); char *resp = cbm_mcp_server_handle( - srv, "{\"jsonrpc\":\"2.0\",\"id\":63,\"method\":\"tools/call\"," - "\"params\":{\"name\":\"trace_call_path\",\"arguments\":{\"function_name\":" - "\"qnfb-proj.src.qnfb_entry\",\"project\":\"qnfb-proj\",\"direction\":\"outbound\"}}}"); + srv, + "{\"jsonrpc\":\"2.0\",\"id\":63,\"method\":\"tools/call\"," + "\"params\":{\"name\":\"trace_call_path\",\"arguments\":{\"function_name\":" + "\"qnfb-proj.src.qnfb_entry\",\"project\":\"qnfb-proj\",\"direction\":\"outbound\"}}}"); ASSERT_NOT_NULL(resp); char *inner = extract_text_content(resp); ASSERT_NOT_NULL(inner); @@ -18537,7 +18630,8 @@ TEST(snippet_doc_candidates_both_directions) { .rank = 1, .score = 0.45, .p = 0.548, - .evidence = "{\"shared_terms\":5,\"name_tokens\":2}"}, + .evidence = "{\"shared_terms\":5,\"name_tokens\":2,\"kind\":\"function\"," + "\"position\":\"local\"}"}, {.section_qn = "test-project.README.Orders", .target_qn = "test-project.gone.Missing", .rank = 3, @@ -18560,6 +18654,9 @@ TEST(snippet_doc_candidates_both_directions) { ASSERT_FLOAT_EQ(yyjson_get_num(yyjson_obj_get(first, "p")), 0.548, 1e-9); ASSERT_EQ((int)yyjson_get_int(yyjson_obj_get(first, "shared_terms")), 5); ASSERT_EQ((int)yyjson_get_int(yyjson_obj_get(first, "start_line")), 7); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(first, "label")), "Function"); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(first, "kind")), "function"); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(first, "position")), "local"); ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(yyjson_arr_get(about, 1), "qualified_name")), "test-project.cmd.server.main.HandleRequest"); ASSERT_NOT_NULL(yyjson_obj_get(yyjson_doc_get_root(doc), "candidates_note")); @@ -23784,6 +23881,7 @@ SUITE(mcp) { RUN_TEST(tool_compare_graphs_validation_and_scan_cap_are_atomic_issue525); RUN_TEST(tool_compare_graphs_cancel_and_readonly_handles_release_issue525); RUN_TEST(tool_get_file_outline_returns_bounded_filtered_columnar_rows_issue469); + RUN_TEST(tool_get_file_outline_shows_doc_candidates_of_file_and_folders); RUN_TEST(tool_get_file_outline_validates_json_path_limit_and_cancel_issue469); RUN_TEST(tool_search_graph_basic); RUN_TEST(tool_search_graph_empty_hint_describes_combined_filters_issue1366); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index c11c2842c..ddf7bc5a5 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -18948,14 +18948,44 @@ TEST(pipeline_semantic_edges_carry_p) { PASS(); } +/* The p a stored candidate row must carry: the curve of the row's kind and + * position (read from its evidence) at its score, two decimals; -1 for an + * unknown kind or position. *is_function: a function candidate (top 5). */ +static double doc_row_want(const char *evidence, cbm_sem_doc_format_t fmt, float score, + bool *is_function) { + static const struct { + const char *name; + cbm_sem_doc_kind_t kind; + } KINDS[] = {{"\"kind\":\"function\"", CBM_SEM_DOC_KIND_FUNCTION}, + {"\"kind\":\"local\"", CBM_SEM_DOC_KIND_LOCAL}, + {"\"kind\":\"file\"", CBM_SEM_DOC_KIND_FILE}}; + static const struct { + const char *name; + cbm_sem_doc_pos_t pos; + } POSITIONS[] = {{"\"position\":\"global\"", CBM_SEM_DOC_POS_GLOBAL}, + {"\"position\":\"local\"", CBM_SEM_DOC_POS_LOCAL}, + {"\"position\":\"outside\"", CBM_SEM_DOC_POS_OUTSIDE}}; + int k = -1; + int p = -1; + for (int i = 0; evidence && i < 3; i++) { + k = k < 0 && strstr(evidence, KINDS[i].name) ? i : k; + p = p < 0 && strstr(evidence, POSITIONS[i].name) ? i : p; + } + *is_function = k == 0; + return k < 0 || p < 0 ? -1.0 + : cbm_sem_p_2dp(cbm_sem_doc_calibrated_p(fmt, KINDS[k].kind, + POSITIONS[p].pos, score)); +} + /* Count a section's stored doc -> code candidates (section QN containing - * heading) and check each row's shape: rank within the top - * CBM_SEM_DOC_TOP_K, score at or above the floor, p the doc curve's band at - * two decimals, never a test function. Returns -1 on a bad row. */ + * heading) and check each row's shape: a function within the top + * CBM_SEM_DOC_TOP_K (top: the best of them), score at or above the floor, p + * the curve of its kind and position at two decimals, never a test + * function. Returns -1 on a bad row. */ static int doc_candidate_rows_for(cbm_store_t *s, const char *heading, char *top, size_t top_sz) { sqlite3_stmt *st = NULL; if (sqlite3_prepare_v2(cbm_store_get_db(s), - "SELECT target_qn, rank, score, p FROM doc_link_candidates " + "SELECT target_qn, rank, score, p, evidence FROM doc_link_candidates " "WHERE instr(section_qn, ?1) > 0 ORDER BY rank", -1, &st, NULL) != SQLITE_OK) { return -1; @@ -18963,16 +18993,20 @@ static int doc_candidate_rows_for(cbm_store_t *s, const char *heading, char *top sqlite3_bind_text(st, 1, heading, -1, SQLITE_TRANSIENT); int n = 0; bool ok = true; + bool have_top = false; while (ok && sqlite3_step(st) == SQLITE_ROW) { const char *target = (const char *)sqlite3_column_text(st, 0); int rank = sqlite3_column_int(st, 1); double score = sqlite3_column_double(st, 2); double p = sqlite3_column_double(st, 3); - double want = (double)(int)(cbm_sem_doc_calibrated_p((float)score) * 100.0F + 0.5F) / 100.0; - ok = target && rank >= 1 && rank <= CBM_SEM_DOC_TOP_K && + bool is_function = false; + double want = doc_row_want((const char *)sqlite3_column_text(st, 4), + CBM_SEM_DOC_FMT_MARKDOWN, (float)score, &is_function); + ok = target && rank >= 1 && (!is_function || rank <= CBM_SEM_DOC_TOP_K) && score >= (double)CBM_SEM_DOC_MIN_SCORE && p == want && !strstr(target, ".Test"); - if (ok && n == 0 && top) { + if (ok && is_function && !have_top && top) { snprintf(top, top_sz, "%s", target); + have_top = true; } n++; } @@ -19061,6 +19095,306 @@ TEST(pipeline_doc_candidates_published_and_kept_by_delta) { PASS(); } +/* One Python family per subject: parse / load / validate / write, each with + * a docstring, so the functions' corpus has the subject words to match. */ +static void write_py_family(const char *base, const char *subject) { + char path[512]; + snprintf(path, sizeof(path), "%s/%s.py", base, subject); + char body[4096]; + snprintf(body, sizeof(body), + "def parse_%s_config(path):\n" + " \"\"\"Parse the %s config file and return the parsed %s settings.\"\"\"\n" + " return load_%s_config(path)\n\n" + "def load_%s_config(path):\n" + " \"\"\"Load the %s config from disk and validate every %s field.\"\"\"\n" + " cfg = {}\n validate_%s_config(cfg)\n return cfg\n\n" + "def validate_%s_config(cfg):\n" + " \"\"\"Check the %s config for missing fields.\"\"\"\n" + " return cfg is not None\n\n" + "def write_%s_config(path, cfg):\n" + " \"\"\"Write the %s config back to disk after validating it.\"\"\"\n" + " validate_%s_config(cfg)\n", + subject, subject, subject, subject, subject, subject, subject, subject, subject, + subject, subject, subject, subject); + th_write_file(path, body); +} + +/* The candidate count, and the bad rows, of one section (its QN contains + * heading and its file ends in ext): p must be the format's curve value at + * two decimals and the tfidf at or above the format's first stored band. */ +static int doc_rows_of(cbm_store_t *s, const char *heading, const char *ext, + cbm_sem_doc_format_t fmt, const char *forbidden_target, int *bad) { + sqlite3_stmt *st = NULL; + *bad = 0; + if (sqlite3_prepare_v2( + cbm_store_get_db(s), + "SELECT c.target_qn, c.score, c.p, c.evidence FROM doc_link_candidates c " + "JOIN nodes n ON n.qualified_name = c.section_qn " + "WHERE instr(c.section_qn, ?1) > 0 AND n.file_path LIKE ?2", + -1, &st, NULL) != SQLITE_OK) { + return -1; + } + char like[32]; + snprintf(like, sizeof(like), "%%%s", ext); + sqlite3_bind_text(st, 1, heading, -1, SQLITE_TRANSIENT); + sqlite3_bind_text(st, 2, like, -1, SQLITE_TRANSIENT); + int n = 0; + while (sqlite3_step(st) == SQLITE_ROW) { + const char *target = (const char *)sqlite3_column_text(st, 0); + float score = (float)sqlite3_column_double(st, 1); + double p = sqlite3_column_double(st, 2); + bool is_function = false; + double want = + doc_row_want((const char *)sqlite3_column_text(st, 3), fmt, score, &is_function); + *bad += want <= 0.0 || p != want || + (forbidden_target && target && strstr(target, forbidden_target)); + n++; + } + sqlite3_finalize(st); + return n; +} + +/* Candidates never repeat an exact link: a section whose code span names a + * function (a MENTIONS edge) does not also list that function as a + * candidate. And each format carries its own judged curve: reST stores only + * from tfidf 0.30, with reST's p. */ +TEST(pipeline_doc_candidates_skip_exact_links_and_use_format_curves) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_docfmt_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + static const char *const subjects[] = {"mail", "cache", "queue", "user", "server", "client"}; + for (size_t i = 0; i < sizeof(subjects) / sizeof(subjects[0]); i++) { + write_py_family(tmp, subjects[i]); + } + char docs[512]; + snprintf(docs, sizeof(docs), "%s/docs", tmp); + ASSERT_TRUE(cbm_mkdir_p(docs, 0755)); + write_temp_file(tmp, "docs/mail.md", + "# Mail settings\n\n" + "The mail config file is parsed by `mail.parse_mail_config`, which loads " + "the mail config from disk and validates every mail field before the " + "parsed mail settings are returned to the caller for use.\n"); + write_temp_file(tmp, "docs/mail.rst", + "Mail loading\n============\n\n" + "Load the mail config from disk and validate every mail field: missing " + "mail fields fail the validation, and the loaded mail config is written " + "back to disk after validating it.\n"); + char db[512]; + snprintf(db, sizeof(db), "%s/docfmt.db", tmp); + cbm_pipeline_incremental_test_reset_faults(); + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + cbm_pipeline_free(p); + + cbm_store_t *s = cbm_store_open_path(db); + ASSERT_NOT_NULL(s); + /* the exact link exists ... */ + sqlite3_stmt *st = NULL; + ASSERT_EQ(sqlite3_prepare_v2(cbm_store_get_db(s), + "SELECT count(*) FROM edges e JOIN nodes a ON a.id = e.source_id " + "JOIN nodes b ON b.id = e.target_id WHERE e.type = 'MENTIONS' " + "AND a.label = 'Section' AND a.file_path = 'docs/mail.md' " + "AND b.name = 'parse_mail_config'", + -1, &st, NULL), + SQLITE_OK); + ASSERT_EQ(sqlite3_step(st), SQLITE_ROW); + int linked = sqlite3_column_int(st, 0); + sqlite3_finalize(st); + /* ... and is not repeated as a candidate, while the rest of the section's + * candidates are stored on the Markdown curve */ + int md_bad = 0; + int rst_bad = 0; + int md = doc_rows_of(s, "Mail-settings", ".md", CBM_SEM_DOC_FMT_MARKDOWN, ".parse_mail_config", + &md_bad); + int rst = doc_rows_of(s, "Mail-loading", ".rst", CBM_SEM_DOC_FMT_RST, NULL, &rst_bad); + cbm_store_close(s); + th_rmtree(tmp); + cbm_pipeline_incremental_test_reset_faults(); + ASSERT_EQ(linked, 1); + ASSERT_GT(md, 0); + ASSERT_EQ(md_bad, 0); + ASSERT_GT(rst, 0); + ASSERT_EQ(rst_bad, 0); + PASS(); +} + +/* Lines of a CBM_SEM_DOC_SIGNALS dump whose section file, kind, position, + * home and target file equal the given values (NULL: any). */ +static int doc_signal_count(const char *dump, const char *sec_file, const char *kind, + const char *pos, const char *home, const char *target_file) { + FILE *f = cbm_fopen(dump, "r"); + if (!f) { + return -1; + } + char line[4096]; + int n = 0; + while (fgets(line, sizeof(line), f)) { + line[strcspn(line, "\r\n")] = '\0'; + char *col[12]; + int nc = 0; + for (char *p = line; nc < 12; nc++) { + col[nc] = p; + char *tab = strchr(p, '\t'); + if (!tab) { + nc++; + break; + } + *tab = '\0'; + p = tab + 1; + } + if (nc != 12) { + continue; + } + char *colon = strrchr(col[5], ':'); + if (colon) { + *colon = '\0'; + } + colon = strrchr(col[6], ':'); + if (colon) { + *colon = '\0'; + } + n += (!sec_file || strcmp(col[5], sec_file) == 0) && (!kind || strcmp(col[9], kind) == 0) && + (!pos || strcmp(col[10], pos) == 0) && (!home || strcmp(col[11], home) == 0) && + (!target_file || strcmp(col[6], target_file) == 0); + } + fclose(f); + return n; +} + +/* Document position: a doc's home is its folder (a docs folder documents its + * parent; a folder without code climbs to one with code; the root = the + * whole project); a candidate inside the home is local, elsewhere outside; + * a doc with a home below the root gets its home folder as a candidate; + * whole files are candidates; local functions below the top 5 are added. */ +TEST(pipeline_doc_candidates_home_position_and_kinds) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_docpos_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char dir[512]; + static const char *const pkgs[] = {"pkg/mail", "pkg/mail/sub", "pkg/cache", "pkg/queue", + "pkg/mail/docs", "docs", "guides"}; + for (size_t i = 0; i < sizeof(pkgs) / sizeof(pkgs[0]); i++) { + snprintf(dir, sizeof(dir), "%s/%s", tmp, pkgs[i]); + ASSERT_TRUE(cbm_mkdir_p(dir, 0755)); + } + snprintf(dir, sizeof(dir), "%s/pkg/mail", tmp); + write_py_family(dir, "mail"); + snprintf(dir, sizeof(dir), "%s/pkg/mail/sub", tmp); + write_py_family(dir, "mail"); + snprintf(dir, sizeof(dir), "%s/pkg/cache", tmp); + write_py_family(dir, "cache"); + snprintf(dir, sizeof(dir), "%s/pkg/queue", tmp); + write_py_family(dir, "queue"); + static const char body[] = + "The mail config file is parsed, loaded from disk and validated: every mail " + "field is checked, missing mail fields fail the validation, and the parsed mail " + "settings are written back to disk after validating the mail config.\n"; + static const char *const doc_files[] = {"pkg/mail/README.md", "pkg/mail/docs/guide.md", + "docs/overview.md", "guides/notes.md"}; + for (size_t i = 0; i < sizeof(doc_files) / sizeof(doc_files[0]); i++) { + char text[1024]; + snprintf(text, sizeof(text), "# Mail config %d\n\n%s", (int)i, body); + write_temp_file(tmp, doc_files[i], text); + } + char db[512]; + char dump[512]; + snprintf(db, sizeof(db), "%s/docpos.db", tmp); + snprintf(dump, sizeof(dump), "%s/signals.tsv", tmp); + cbm_setenv("CBM_SEM_DOC_SIGNALS", dump, 1); + cbm_pipeline_incremental_test_reset_faults(); + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + int rc = cbm_pipeline_run(p); + cbm_pipeline_free(p); + cbm_unsetenv("CBM_SEM_DOC_SIGNALS"); + ASSERT_EQ(rc, 0); + + const char *readme = "pkg/mail/README.md"; + const char *guide = "pkg/mail/docs/guide.md"; + /* homes: the package; its docs folder documents the package; top-level + * docs and a folder without code document the whole project */ + int readme_lines = doc_signal_count(dump, readme, NULL, NULL, NULL, NULL); + int readme_home = doc_signal_count(dump, readme, NULL, NULL, "pkg/mail", NULL); + int guide_lines = doc_signal_count(dump, guide, NULL, NULL, NULL, NULL); + int guide_home = doc_signal_count(dump, guide, NULL, NULL, "pkg/mail", NULL); + int overview_lines = doc_signal_count(dump, "docs/overview.md", NULL, NULL, NULL, NULL); + int overview_root = doc_signal_count(dump, "docs/overview.md", NULL, "global", ".", NULL); + int notes_lines = doc_signal_count(dump, "guides/notes.md", NULL, NULL, NULL, NULL); + int notes_root = doc_signal_count(dump, "guides/notes.md", NULL, "global", ".", NULL); + /* positions */ + int local_fn = + doc_signal_count(dump, readme, "function", "local", "pkg/mail", "pkg/mail/mail.py"); + int local_sub = + doc_signal_count(dump, readme, NULL, "local", "pkg/mail", "pkg/mail/sub/mail.py"); + int outside_fn = doc_signal_count(dump, readme, "function", "outside", NULL, NULL); + int wrong_local = doc_signal_count(dump, readme, NULL, "local", NULL, "pkg/cache/cache.py"); + /* kinds */ + int folders = doc_signal_count(dump, NULL, "folder", "local", NULL, "pkg/mail"); + int folder_readme = doc_signal_count(dump, readme, "folder", "local", "pkg/mail", "pkg/mail"); + int root_folders = doc_signal_count(dump, "docs/overview.md", "folder", NULL, NULL, NULL) + + doc_signal_count(dump, "guides/notes.md", "folder", NULL, NULL, NULL); + int files = doc_signal_count(dump, readme, "file", "local", NULL, NULL); + int extras = doc_signal_count(dump, readme, "local", "local", "pkg/mail", NULL); + /* stored: the README's home folder (a README: p 0.87), not the guide's + * (another doc), no function outside the home, every row on its curve */ + cbm_store_t *s = cbm_store_open_path(db); + ASSERT_NOT_NULL(s); + sqlite3_stmt *st = NULL; + ASSERT_EQ(sqlite3_prepare_v2(cbm_store_get_db(s), + "SELECT c.score, c.p, c.evidence, n.file_path FROM " + "doc_link_candidates c JOIN nodes n ON n.qualified_name = " + "c.section_qn", + -1, &st, NULL), + SQLITE_OK); + int stored_folders = 0; + int stored_outside = 0; + int off_curve = 0; + double folder_p = 0.0; + while (sqlite3_step(st) == SQLITE_ROW) { + const char *ev = (const char *)sqlite3_column_text(st, 2); + const char *path = (const char *)sqlite3_column_text(st, 3); + double p = sqlite3_column_double(st, 1); + if (ev && strstr(ev, "\"kind\":\"folder\"")) { + stored_folders++; + folder_p = p; + off_curve += strcmp(path, readme) != 0; + continue; + } + stored_outside += + ev && strstr(ev, "\"position\":\"outside\"") && strstr(ev, "\"kind\":\"function\""); + bool is_function = false; + off_curve += p != doc_row_want(ev, CBM_SEM_DOC_FMT_MARKDOWN, + (float)sqlite3_column_double(st, 0), &is_function); + } + sqlite3_finalize(st); + cbm_store_close(s); + th_rmtree(tmp); + cbm_pipeline_incremental_test_reset_faults(); + ASSERT_GT(readme_lines, 0); + ASSERT_EQ(readme_home, readme_lines); + ASSERT_GT(guide_lines, 0); + ASSERT_EQ(guide_home, guide_lines); + ASSERT_GT(overview_lines, 0); + ASSERT_EQ(overview_root, overview_lines); + ASSERT_GT(notes_lines, 0); + ASSERT_EQ(notes_root, notes_lines); + ASSERT_GT(local_fn, 0); + ASSERT_GT(local_sub, 0); + ASSERT_GT(outside_fn, 0); + ASSERT_EQ(wrong_local, 0); + ASSERT_EQ(folders, 2); /* README and the docs guide; one per doc, never per section */ + ASSERT_EQ(folder_readme, 1); + ASSERT_EQ(root_folders, 0); + ASSERT_GT(files, 0); + ASSERT_GT(extras, 0); + ASSERT_EQ(stored_folders, 1); + ASSERT_FLOAT_EQ(folder_p, 0.87, 1e-9); + ASSERT_EQ(stored_outside, 0); + ASSERT_EQ(off_curve, 0); + PASS(); +} + TEST(pipeline_semantic_edges_no_functions) { char tmp[256]; snprintf(tmp, sizeof(tmp), "/tmp/cbm_nofunc_XXXXXX"); @@ -19976,6 +20310,8 @@ SUITE(pipeline) { RUN_TEST(pipeline_semantic_pair_signals_never_change_the_graph); RUN_TEST(pipeline_semantic_edges_carry_p); RUN_TEST(pipeline_doc_candidates_published_and_kept_by_delta); + RUN_TEST(pipeline_doc_candidates_skip_exact_links_and_use_format_curves); + RUN_TEST(pipeline_doc_candidates_home_position_and_kinds); RUN_TEST(pipeline_cpp_static_factory_pointer_receiver_issue1153); RUN_TEST(pipeline_cpp_static_factory_reference_receiver_issue1153); RUN_TEST(pipeline_cpp_static_factory_unique_ptr_receiver_issue1153); diff --git a/tests/test_semantic.c b/tests/test_semantic.c index 006cad45c..f1ebd09f7 100644 --- a/tests/test_semantic.c +++ b/tests/test_semantic.c @@ -513,25 +513,135 @@ TEST(sem_calibrated_p_bands) { PASS(); } -/* Doc -> code candidates: the judged bands (sample 4), monotone, with nothing - * stored below CBM_SEM_DOC_MIN_SCORE (that band judged 0.06). */ +static float docp(cbm_sem_doc_format_t fmt, cbm_sem_doc_kind_t kind, cbm_sem_doc_pos_t pos, + float score) { + return cbm_sem_doc_calibrated_p(fmt, kind, pos, score); +} + +/* Doc -> code candidates, Markdown (sample 7): p by kind and position -- + * functions outside the doc's home never stored, inside it from 0.30, for a + * doc of the whole project (samples 4 and 7 pooled) from 0.30 too; local + * extras 0.32 inside the home + * only; whole files from 0.40; nothing below CBM_SEM_DOC_MIN_SCORE; every + * curve monotone. A format without a judged curve stores nothing. */ TEST(sem_doc_calibrated_p_bands) { - ASSERT_FLOAT_EQ(cbm_sem_doc_calibrated_p(0.10F), 0.06F, 0.0); - ASSERT_FLOAT_EQ(cbm_sem_doc_calibrated_p(0.199F), 0.06F, 0.0); - ASSERT_FLOAT_EQ(cbm_sem_doc_calibrated_p(CBM_SEM_DOC_MIN_SCORE), 0.36F, 0.0); - ASSERT_FLOAT_EQ(cbm_sem_doc_calibrated_p(0.299F), 0.36F, 0.0); - ASSERT_FLOAT_EQ(cbm_sem_doc_calibrated_p(0.30F), 0.42F, 0.0); - ASSERT_FLOAT_EQ(cbm_sem_doc_calibrated_p(0.40F), 0.548F, 0.0); - ASSERT_FLOAT_EQ(cbm_sem_doc_calibrated_p(1.00F), 0.548F, 0.0); - float last = 0.0F; + const cbm_sem_doc_format_t md = CBM_SEM_DOC_FMT_MARKDOWN; + const cbm_sem_doc_kind_t fn = CBM_SEM_DOC_KIND_FUNCTION; + const cbm_sem_doc_pos_t global = CBM_SEM_DOC_POS_GLOBAL; + const cbm_sem_doc_pos_t local = CBM_SEM_DOC_POS_LOCAL; + const cbm_sem_doc_pos_t outside = CBM_SEM_DOC_POS_OUTSIDE; + ASSERT_FLOAT_EQ(docp(md, fn, local, 0.199F), 0.0F, 0.0); + ASSERT_FLOAT_EQ(docp(md, fn, local, CBM_SEM_DOC_MIN_SCORE), 0.0F, 0.0); + ASSERT_FLOAT_EQ(docp(md, fn, local, 0.30F), 0.325F, 0.0); + ASSERT_FLOAT_EQ(docp(md, fn, local, 0.40F), 0.525F, 0.0); + ASSERT_FLOAT_EQ(docp(md, fn, global, 0.299F), 0.0F, 0.0); + ASSERT_FLOAT_EQ(docp(md, fn, global, 0.30F), 0.217F, 0.0); + ASSERT_FLOAT_EQ(docp(md, fn, global, 0.40F), 0.509F, 0.0); + ASSERT_FLOAT_EQ(docp(md, fn, outside, 0.90F), 0.0F, 0.0); + ASSERT_FLOAT_EQ(docp(md, CBM_SEM_DOC_KIND_LOCAL, local, 0.199F), 0.0F, 0.0); + ASSERT_FLOAT_EQ(docp(md, CBM_SEM_DOC_KIND_LOCAL, local, 0.20F), 0.317F, 0.0); + ASSERT_FLOAT_EQ(docp(md, CBM_SEM_DOC_KIND_LOCAL, local, 0.90F), 0.317F, 0.0); + ASSERT_FLOAT_EQ(docp(md, CBM_SEM_DOC_KIND_FILE, outside, 0.399F), 0.0F, 0.0); + ASSERT_FLOAT_EQ(docp(md, CBM_SEM_DOC_KIND_FILE, outside, 0.40F), 0.30F, 0.0); + ASSERT_FLOAT_EQ(docp(md, CBM_SEM_DOC_KIND_FOLDER, local, 0.90F), 0.0F, 0.0); + static const cbm_sem_doc_kind_t KINDS[] = {CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_KIND_LOCAL, + CBM_SEM_DOC_KIND_FILE}; + for (size_t k = 0; k < sizeof(KINDS) / sizeof(KINDS[0]); k++) { + for (int pos = 0; pos < CBM_SEM_DOC_POS_COUNT; pos++) { + float last = 0.0F; + for (int milli = 0; milli <= 1000; milli++) { + float s = (float)milli / 1000.0F; + float p = docp(md, KINDS[k], (cbm_sem_doc_pos_t)pos, s); + ASSERT_TRUE(p >= last); + last = p; + ASSERT_TRUE(pos != CBM_SEM_DOC_POS_OUTSIDE || KINDS[k] == CBM_SEM_DOC_KIND_FILE || + p == 0.0F); + ASSERT_FLOAT_EQ(docp(CBM_SEM_DOC_FMT_OTHER, KINDS[k], (cbm_sem_doc_pos_t)pos, s), + 0.0F, 0.0); + ASSERT_FLOAT_EQ(docp(CBM_SEM_DOC_FMT_PDF, KINDS[k], (cbm_sem_doc_pos_t)pos, s), + 0.0F, 0.0); + } + } + } + ASSERT_FLOAT_EQ(docp(CBM_SEM_DOC_FMT_COUNT, fn, global, 0.9F), 0.0F, 0.0); + /* a home folder: README and index docs only, Markdown only */ + ASSERT_FLOAT_EQ(cbm_sem_doc_folder_p(md, "pkg/mail/README.md"), 0.867F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_doc_folder_p(md, "pkg/mail/docs/index.mdx"), 0.867F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_doc_folder_p(md, "pkg/mail/readme"), 0.867F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_doc_folder_p(md, "pkg/mail/guide.md"), 0.0F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_doc_folder_p(md, "pkg/mail/README-old.md"), 0.0F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_doc_folder_p(CBM_SEM_DOC_FMT_RST, "pkg/mail/index.rst"), 0.0F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_doc_folder_p(md, NULL), 0.0F, 0.0); + /* reST (sample 6): 0.20-0.30 judged 0.125, not stored */ + const cbm_sem_doc_format_t rst = CBM_SEM_DOC_FMT_RST; + ASSERT_FLOAT_EQ( + cbm_sem_doc_calibrated_p(rst, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, 0.25F), + 0.0F, 0.0); + ASSERT_FLOAT_EQ( + cbm_sem_doc_calibrated_p(rst, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, 0.30F), + 0.35F, 0.0); + ASSERT_FLOAT_EQ( + cbm_sem_doc_calibrated_p(rst, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, 0.40F), + 0.625F, 0.0); + /* AsciiDoc (sample 6): every band at or above the floor is stored */ + const cbm_sem_doc_format_t adoc = CBM_SEM_DOC_FMT_ADOC; + ASSERT_FLOAT_EQ( + cbm_sem_doc_calibrated_p(adoc, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, 0.199F), + 0.0F, 0.0); + ASSERT_FLOAT_EQ( + cbm_sem_doc_calibrated_p(adoc, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, 0.20F), + 0.225F, 0.0); + ASSERT_FLOAT_EQ( + cbm_sem_doc_calibrated_p(adoc, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, 0.35F), + 0.25F, 0.0); + ASSERT_FLOAT_EQ( + cbm_sem_doc_calibrated_p(adoc, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, 0.90F), + 0.50F, 0.0); for (int milli = 0; milli <= 1000; milli++) { - float p = cbm_sem_doc_calibrated_p((float)milli / 1000.0F); - ASSERT_TRUE(p >= last); - last = p; + float s = (float)milli / 1000.0F; + ASSERT_TRUE( + cbm_sem_doc_calibrated_p(rst, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, s) >= + cbm_sem_doc_calibrated_p(rst, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, + s - 0.001F)); + ASSERT_TRUE( + cbm_sem_doc_calibrated_p(adoc, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, s) >= + cbm_sem_doc_calibrated_p(adoc, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, + s - 0.001F)); } PASS(); } +/* p is stored and shown at two decimals, half up, whatever the float + * representation of the judged value (0.225F is 0.22499999...). */ +TEST(sem_p_two_decimals_half_up) { + ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.225F), 0.23, 1e-12); + ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.325F), 0.33, 1e-12); /* float 0.32499998... */ + ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.525F), 0.53, 1e-12); /* float 0.52499997... */ + ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.324F), 0.32, 1e-12); + ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.317F), 0.32, 1e-12); + ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.867F), 0.87, 1e-12); + ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.625F), 0.63, 1e-12); + ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.548F), 0.55, 1e-12); + ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.36F), 0.36, 1e-12); + ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.0F), 0.0, 1e-12); + PASS(); +} + +/* A section's format comes from its file's extension, case-insensitively; + * a .txt section is reStructuredText (plain text has no sections). */ +TEST(sem_doc_format_by_extension) { + ASSERT_EQ(cbm_sem_doc_format("docs/guide.md"), CBM_SEM_DOC_FMT_MARKDOWN); + ASSERT_EQ(cbm_sem_doc_format("README.MARKDOWN"), CBM_SEM_DOC_FMT_MARKDOWN); + ASSERT_EQ(cbm_sem_doc_format("docs/index.rst"), CBM_SEM_DOC_FMT_RST); + ASSERT_EQ(cbm_sem_doc_format("docs/topics/db.txt"), CBM_SEM_DOC_FMT_RST); + ASSERT_EQ(cbm_sem_doc_format("manual/intro.adoc"), CBM_SEM_DOC_FMT_ADOC); + ASSERT_EQ(cbm_sem_doc_format("paper.pdf"), CBM_SEM_DOC_FMT_PDF); + ASSERT_EQ(cbm_sem_doc_format("src/main.go"), CBM_SEM_DOC_FMT_OTHER); + ASSERT_EQ(cbm_sem_doc_format("v1.2/NOTES"), CBM_SEM_DOC_FMT_OTHER); + ASSERT_EQ(cbm_sem_doc_format(NULL), CBM_SEM_DOC_FMT_OTHER); + PASS(); +} + /* A properties string holding a quote is read whole, and escapes are not * glued to the next word (until 2026-10 the read stopped at the first quote * and kept "\n" raw, so "\nreturn" became the token "nreturn"). */ @@ -731,6 +841,8 @@ SUITE(semantic) { RUN_TEST(sem_admit_best_first_keeps_the_best); RUN_TEST(sem_calibrated_p_bands); RUN_TEST(sem_doc_calibrated_p_bands); + RUN_TEST(sem_doc_format_by_extension); + RUN_TEST(sem_p_two_decimals_half_up); RUN_TEST(sem_json_str_honours_escapes); RUN_TEST(sem_corpus_add_null_doc); RUN_TEST(sem_corpus_free_null); diff --git a/tests/test_store_arch.c b/tests/test_store_arch.c index 6c6037770..23ea5c935 100644 --- a/tests/test_store_arch.c +++ b/tests/test_store_arch.c @@ -275,7 +275,8 @@ TEST(arch_path_scoping) { ASSERT_TRUE(whole_pkg_nodes > scoped_pkg_nodes); ASSERT_EQ(scoped_pkg_nodes, 1); - ASSERT_TRUE(cbm_store_count_nodes(s, "pscope") > cbm_store_count_nodes_scoped(s, "pscope", "apps/foo")); + ASSERT_TRUE(cbm_store_count_nodes(s, "pscope") > + cbm_store_count_nodes_scoped(s, "pscope", "apps/foo")); cbm_architecture_info_t scoped_slash; memset(&scoped_slash, 0, sizeof(scoped_slash)); @@ -703,6 +704,63 @@ TEST(doc_candidates_replace_and_get) { PASS(); } +/* Whole-file candidates of a path: the rows whose target is the path's File + * node or a Folder holding it -- never a sibling folder sharing a name prefix + * (pkg/mailer, pk), a function in the file, or another project's rows. */ +TEST(doc_candidates_for_path_file_and_folders) { + cbm_store_t *s = cbm_store_open_memory(); + ASSERT_NOT_NULL(s); + cbm_doc_candidate_t *got = NULL; + int n = -1; + ASSERT_EQ(cbm_store_doc_candidates_for_path(s, "p", "pkg/mail/parse.py", 5, &got, &n), + CBM_STORE_NOT_FOUND); + cbm_store_upsert_project(s, "p", "/tmp/p"); + const struct { + const char *label, *qn, *path; + } nodes[] = {{"File", "p.pkg.mail.parse.__file__", "pkg/mail/parse.py"}, + {"Function", "p.pkg.mail.parse.parse_mail", "pkg/mail/parse.py"}, + {"Folder", "p.pkg.mail", "pkg/mail"}, + {"Folder", "p.pkg", "pkg"}, + {"Folder", "p.pkg.mailer", "pkg/mailer"}, + {"Folder", "p.pk", "pk"}}; + for (size_t i = 0; i < sizeof(nodes) / sizeof(nodes[0]); i++) { + cbm_node_t node = {.project = "p", + .label = nodes[i].label, + .name = nodes[i].qn, + .qualified_name = nodes[i].qn, + .file_path = nodes[i].path}; + ASSERT_TRUE(cbm_store_upsert_node(s, &node) > 0); + } + const cbm_doc_candidate_t rows[] = { + {.section_qn = "p.docs.a", .target_qn = "p.pkg.mail.parse.__file__", .p = 0.5}, + {.section_qn = "p.docs.b", .target_qn = "p.pkg.mail", .p = 0.7}, + {.section_qn = "p.docs.c", .target_qn = "p.pkg", .p = 0.3}, + {.section_qn = "p.docs.d", .target_qn = "p.pkg.mailer", .p = 0.9}, + {.section_qn = "p.docs.e", .target_qn = "p.pk", .p = 0.9}, + {.section_qn = "p.docs.f", .target_qn = "p.pkg.mail.parse.parse_mail", .p = 0.95}, + }; + ASSERT_EQ(cbm_store_doc_candidates_replace(s, "p", rows, 6), CBM_STORE_OK); + const cbm_doc_candidate_t other = { + .section_qn = "q.docs.a", .target_qn = "p.pkg.mail", .p = 0.99}; + ASSERT_EQ(cbm_store_doc_candidates_replace(s, "q", &other, 1), CBM_STORE_OK); + + ASSERT_EQ(cbm_store_doc_candidates_for_path(s, "p", "pkg/mail/parse.py", 5, &got, &n), + CBM_STORE_OK); + ASSERT_EQ(n, 3); + ASSERT_STR_EQ(got[0].section_qn, "p.docs.b"); /* highest p first */ + ASSERT_STR_EQ(got[1].section_qn, "p.docs.a"); + ASSERT_STR_EQ(got[2].section_qn, "p.docs.c"); + cbm_store_doc_candidates_free(got, n); + + ASSERT_EQ(cbm_store_doc_candidates_for_path(s, "p", "pkg/mail/parse.py", 1, &got, &n), + CBM_STORE_OK); + ASSERT_EQ(n, 1); + cbm_store_doc_candidates_free(got, n); + ASSERT_EQ(cbm_store_doc_candidates_for_path(s, "p", "", 5, &got, &n), CBM_STORE_ERR); + cbm_store_close(s); + PASS(); +} + TEST(adr_parse_sections_basic) { cbm_adr_sections_t sec = cbm_adr_parse_sections("## PURPOSE\nFoo\n\n## STACK\nBar"); ASSERT_EQ(sec.count, 2); @@ -1051,11 +1109,11 @@ static int adr_check_splice(const char *in, const char *name, const char *body, return 0; } -#define CHECK_SPLICE(in, name, body, expect) \ - do { \ - if (adr_check_splice((in), (name), (body), (expect)) != 0) { \ - return 1; \ - } \ +#define CHECK_SPLICE(in, name, body, expect) \ + do { \ + if (adr_check_splice((in), (name), (body), (expect)) != 0) { \ + return 1; \ + } \ } while (0) /* THE acceptance property. Every document below is a case where rebuilding @@ -1170,9 +1228,9 @@ TEST(adr_splice_mixed_line_endings) { TEST(adr_splice_matches_headings_across_line_endings) { adr_name_collect_t c; memset(&c, 0, sizeof(c)); - ASSERT_EQ(cbm_adr_scan_headings("## PURPOSE\r\nFoo\r\n\r\n## STACK\r\nBar", - adr_collect_names, &c), - CBM_STORE_OK); + ASSERT_EQ( + cbm_adr_scan_headings("## PURPOSE\r\nFoo\r\n\r\n## STACK\r\nBar", adr_collect_names, &c), + CBM_STORE_OK); ASSERT_STR_EQ(c.buf, "[PURPOSE][STACK]"); /* A fenced block with CRLF still hides its heading, and still closes. */ @@ -1778,6 +1836,7 @@ SUITE(store_arch) { RUN_TEST(adr_delete); RUN_TEST(adr_delete_not_found); RUN_TEST(doc_candidates_replace_and_get); + RUN_TEST(doc_candidates_for_path_file_and_folders); RUN_TEST(adr_parse_sections_basic); RUN_TEST(adr_parse_sections_all_six); RUN_TEST(adr_parse_sections_non_canonical);