From 0c7ca344190468120fde1001874598cfd72d868b Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 10 Oct 2026 10:19:42 +0200 Subject: [PATCH 01/17] doc-links(go): Go doc links become MENTIONS edges, held until their audit Go doc comments name code as doc links ([Name], [Recv.Name], [pkg.Name], [pkg.Recv.Name], an import path for pkg), the way go/doc/comment reads them. The Go leg (doclink_go.c extracts them and a go.mod's module path as the scope; doc_links_go.c resolves them against the package's declarations and its imports) is ported onto the restacked core: the family row carries its edge type, and resolve() takes the per-file state the core now passes (Go keeps none). The family is held (ships=false) until its held-out audit passes: a resolved doc link is a below_bar_tier row until then. The suite ships it through the test seam, like the other held families; doc_mentions_go_tokens asserts the production default. Kubernetes at 8c078bc32e90 with the family shipped (a measurement build): 1,928 edges (1,115 exact, 813 unique) and 101 rows (external 61, graph_gap 22, missing 18), the same numbers the leg measured on the old core. Signed-off-by: Martin Vogel --- Makefile.cbm | 4 +- internal/cbm/doclink.c | 10 + internal/cbm/doclink.h | 42 + internal/cbm/doclink_go.c | 2045 ++++++++++++++++++++++++++++ src/pipeline/doc_links.c | 2 +- src/pipeline/doc_links.h | 1 + src/pipeline/doc_links_go.c | 1905 ++++++++++++++++++++++++++ tests/test_doc_mentions_go.c | 2112 +++++++++++++++++++++++++++++ tests/test_doc_mentions_helpers.h | 2 +- tests/test_main.c | 2 + 10 files changed, 6122 insertions(+), 3 deletions(-) create mode 100644 internal/cbm/doclink_go.c create mode 100644 src/pipeline/doc_links_go.c create mode 100644 tests/test_doc_mentions_go.c diff --git a/Makefile.cbm b/Makefile.cbm index cd5eb4124..bee0f531d 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -386,6 +386,7 @@ EXTRACTION_SRCS = \ $(CBM_DIR)/pdf/pdf_text.c \ $(CBM_DIR)/pdf/pdf_tables.c \ $(CBM_DIR)/pdf/pdf_unicode.c \ + $(CBM_DIR)/doclink_go.c \ $(CBM_DIR)/sql_values.c \ $(CBM_DIR)/helpers.c \ $(CBM_DIR)/cpp_branch_views.c \ @@ -512,6 +513,7 @@ PIPELINE_SRCS = \ src/pipeline/doc_links_pdf.c \ src/pipeline/doc_links_rst.c \ src/pipeline/doc_links_adoc.c \ + src/pipeline/doc_links_go.c \ src/pipeline/doc_links_msbuild.c \ src/pipeline/pass_route_nodes.c \ src/pipeline/pass_enrichment.c \ @@ -774,7 +776,7 @@ TEST_DISCOVER_SRCS = \ TEST_GRAPH_BUFFER_SRCS = tests/test_graph_buffer.c -TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_pipeline_conventions.c tests/test_pipeline_frozen.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_doclinks.c tests/test_doc_mentions.c tests/test_doc_links_md.c tests/test_doc_links_pdf.c tests/test_doc_links_rst.c tests/test_doc_links_adoc.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c +TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_pipeline_conventions.c tests/test_pipeline_frozen.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_doclinks.c tests/test_doc_mentions.c tests/test_doc_links_md.c tests/test_doc_links_pdf.c tests/test_doc_links_rst.c tests/test_doc_links_adoc.c tests/test_doc_mentions_go.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c TEST_WATCHER_SRCS = tests/test_watcher.c diff --git a/internal/cbm/doclink.c b/internal/cbm/doclink.c index 09ee36c25..0d8c1e753 100644 --- a/internal/cbm/doclink.c +++ b/internal/cbm/doclink.c @@ -180,6 +180,16 @@ static const doclink_lang_t DOCLINK_LANGS[] = { {.lang = CBM_LANG_YAML, .scan_scope = cbm_doclink_antora_scan_scope, .scope_tag = CBM_DOCLINK_ADOC_SCOPE_TAG}, + {.lang = CBM_LANG_GO, + .parse_doc = cbm_doclink_go_parse_doc, + .scan_scope = cbm_doclink_go_scan_scope, + .scope_tag = CBM_DOCLINK_GO_SCOPE_TAG, + .portable_scope = cbm_doclink_go_portable_scope}, + {.lang = CBM_LANG_GOMOD, + .parse_doc = cbm_doclink_gomod_parse_doc, + .scan_scope = cbm_doclink_gomod_scan_scope, + .scope_tag = CBM_DOCLINK_GO_SCOPE_TAG, + .portable_scope = cbm_doclink_go_portable_scope}, }; static const doclink_lang_t *doclink_lang(CBMLanguage lang) { diff --git a/internal/cbm/doclink.h b/internal/cbm/doclink.h index b33812dec..ab97ac0f4 100644 --- a/internal/cbm/doclink.h +++ b/internal/cbm/doclink.h @@ -83,6 +83,8 @@ X(CBM_DOCLINK_CS_SEEALSO, CBM_LANG_CSHARP, "seealso", false, true, "MENTIONS") \ X(CBM_DOCLINK_CS_EXCEPTION, CBM_LANG_CSHARP, "exception", false, true, "MENTIONS") \ X(CBM_DOCLINK_CS_INHERITDOC, CBM_LANG_CSHARP, "inheritdoc", false, true, "MENTIONS") \ + /* go (doclink_go.c): [Name] doc links; held until its held-out audit passes */ \ + X(CBM_DOCLINK_GO_DOC_LINK, CBM_LANG_GO, "doc_link", false, false, "MENTIONS") \ /* markdown (doclink_md.c): what a document's text names explicitly */ \ X(CBM_DOCLINK_MD_LINK, CBM_LANG_MARKDOWN, "link", false, true, "MENTIONS") \ X(CBM_DOCLINK_MD_PATH, CBM_LANG_MARKDOWN, "path", false, true, "MENTIONS") \ @@ -385,5 +387,45 @@ void cbm_pdf_test_scan_page(CBMExtractCtx *ctx, const char *text, size_t len, ui /* candidates the scanner's joins have compared so far (a cost counter) */ size_t cbm_pdf_test_join_steps(void); #endif +/* ── Go (doclink_go.c) ─────────────────────────────────────────────── */ + +void cbm_doclink_go_parse_doc(CBMExtractCtx *ctx, const CBMDefinition *def, const char *doc, + uint32_t doc_line); +const char *cbm_doclink_go_scan_scope(CBMExtractCtx *ctx); +char *cbm_doclink_go_portable_scope(const char *scope); + +/* A go.mod file has no doc links; its scope is the module path, in the Go + * scope format, so that import paths map to the repository's directories. */ +void cbm_doclink_gomod_parse_doc(CBMExtractCtx *ctx, const CBMDefinition *def, const char *doc, + uint32_t doc_line); +const char *cbm_doclink_gomod_scan_scope(CBMExtractCtx *ctx); + +/* The parts of a Go doc link as go/doc/comment splits it (docLink, + * splitDocName): `pkg`, `recv` and `name` are slices of the reference text, + * each possibly empty. [Name] {name}; [Recv.Name] {recv, name}; [pkg] {pkg}; + * [pkg.Name] {pkg, name}; [pkg.Recv.Name] {pkg, recv, name}. */ +typedef struct { + const char *pkg; /* a package name, or an import path when it has a '/' */ + size_t pkg_len; + const char *recv; + size_t recv_len; + const char *name; + size_t name_len; +} CBMDocLinkGoRef; + +/* The longest reference text, and the longest name, the Go leg handles. A + * longer bracket text made of doc-link characters is harvested clipped to at + * most one byte more than this, and the resolver reports it as unparseable: + * a reference is never resolved by its beginning. */ +enum { CBM_DOCLINK_GO_REF_MAX = 512 }; + +/* Split the text between the brackets (a leading `*` is skipped). False for + * text that go/doc/comment could not take for a doc link whatever the package + * declares or imports (nothing is harvested from it), and for text longer than + * CBM_DOCLINK_GO_REF_MAX. */ +bool cbm_doclink_go_parse_ref(const char *raw, size_t len, CBMDocLinkGoRef *out); + +/* The Go scope blob (Go files and go.mod files) starts with this tag line. */ +#define CBM_DOCLINK_GO_SCOPE_TAG "go1" #endif /* CBM_DOCLINK_H */ diff --git a/internal/cbm/doclink_go.c b/internal/cbm/doclink_go.c new file mode 100644 index 000000000..97dab618d --- /dev/null +++ b/internal/cbm/doclink_go.c @@ -0,0 +1,2045 @@ +/* + * doclink_go.c — Go doc links and the Go doc-link scope. + * + * Tokens. A Go doc comment has no link markup of its own kind: a doc link is + * bracketed text that go/doc/comment decides to treat as one. This file ports + * the half of that decision that needs only the comment (go/ast + * CommentGroup.Text, then go/doc/comment unindent, parseSpans, link + * definitions, parseLinkedText's bracket scan with its border rule, and the + * name shape of docLink/splitDocName). What it emits are CANDIDATES: + * [Name] [Recv.Name] [pkg] [pkg.Name] [pkg.Recv.Name] + * each optionally with a leading `*`, where pkg is an identifier or an import + * path. The other half of the decision (does the package declare the name, + * does the file import the package) needs the whole package and is the + * resolver's (src/pipeline/doc_links_go.c): a candidate that fails it is plain + * text, exactly as go/doc renders it, and leaves neither an edge nor a row. + * A `[Text]` with a link definition `[Text]: scheme://...` in the same comment + * is a URL: one `href` token per use. + * + * Never a link: code blocks (indented lines), `# ` headings (go/doc keeps + * heading text plain), the link-definition paragraphs themselves, and a + * bracket pair that touches a letter or digit (`map[K]V`, `a[i]`). + * + * Scope blob (first line "go1", then one record per line, tab-separated, + * sorted; only exported names are recorded, a doc link can name nothing else): + * P name flags package clause; flags: i build-ignored file + * (`//go:build ignore`), c another build-constraint + * line, e parse errors in the file, - none + * I local path import; local `-` when not renamed (dot and blank + * imports are no qualifiers and are left out); path + * `?` when it is none this scan can record + * F name w function + * M recv name w method (recv `?` when the receiver type is not a + * plain name) + * T name kind alias w type; kind s struct, i interface, a alias, o other; + * alias: the target of `type X = Y` / `= pkg.Y` + * V name w variable or constant + * f type name node struct field (node 0: embedded, or not the first + * name of its declaration) + * m type name interface method + * g first last names type parameters (','-joined) of the declaration + * that spans the lines first..last. Only in the + * blob of a fresh extraction; the persisted blob + * drops these lines and so carries no line number + * at all. + * G path go.mod: the module path + * w says who keeps the node. A Go method node has a FLAT qualified name + * (.), so a function, the methods of every receiver, a type and + * a value of one name all claim the node ., and within a file the + * declaration on the LAST line keeps it (cbm_gbuf_upsert_node). w is 1 for + * that declaration, 0 for the others of that name, x when two of them start + * on that last line, and - for a value the extractor gives no node at all + * (it gives one to the first name of a constant spec and of a parenthesis- + * free `var`). + * The records are sorted and w is the only order-dependent field: the blob + * does not change when declarations merely move, and it does change when a + * move hands a node to another declaration. + */ +#include "doclink.h" + +#include "cbm.h" +#include "foundation/arena.h" +#include "foundation/constants.h" +#include "foundation/mem_core.h" +#include "tree_sitter/api.h" + +#include +#include +#include +#include +#include +#include + +enum { + GO_REF_MAX = CBM_DOCLINK_GO_REF_MAX, /* longest bracket text and name handled */ + GO_URL_MAX = 2048, /* a link-definition URL is stored up to here */ + GO_PATH_MAX = 1024, /* longest import or module path recorded */ + GO_ARRAY_INIT = 16, + GO_MARKER_LEN = 2, /* "//", the opening and the closing of a block comment */ + GO_LINKDEF_TAIL = 3, /* "]: " */ + GO_ASCII_END = 0x80, /* first non-ASCII byte */ + GO_ASCII_DEL = 0x7F, + GO_RECV_UNWRAP = 4, /* pointer_type / generic_type levels of a receiver, as the extractor */ + GO_EMBED_UNWRAP = 8, /* the same for an embedded field: *pkg.T[A] and some slack */ + GO_SB_INIT = 64, + GO_NODE_FLAG = 1, /* "node" field of a V or f record */ +}; + +/* ── Small helpers ───────────────────────────────────────────────── */ + +static CBMArena *go_scratch(CBMExtractCtx *ctx) { + return ctx->scratch ? ctx->scratch : ctx->arena; +} + +/* Room for one more element of an arena array (the old block is abandoned to + * the arena); NULL when memory ran out. */ +static void *go_grow(CBMArena *a, void *items, int count, int *cap, size_t elem) { + if (count < *cap) { + return items; + } + int ncap = *cap ? *cap * PAIR_LEN : GO_ARRAY_INIT; + void *grown = cbm_arena_alloc(a, (size_t)ncap * elem); + if (!grown) { + return NULL; + } + if (count > 0) { + memcpy(grown, items, (size_t)count * elem); + } + *cap = ncap; + return grown; +} + +static bool go_space(char c) { + return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\v' || c == '\f'; +} + +static bool go_blank_or_tab(char c) { + return c == ' ' || c == '\t'; +} + +static bool go_has_prefix(const char *s, size_t n, const char *prefix) { + size_t pl = strlen(prefix); + return n >= pl && memcmp(s, prefix, pl) == 0; +} + +static bool go_ascii_alpha(unsigned char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); +} + +static bool go_ascii_digit(unsigned char c) { + return c >= '0' && c <= '9'; +} + +static void go_trim(const char **s, size_t *n) { + while (*n > 0 && go_space((*s)[0])) { + (*s)++; + (*n)--; + } + while (*n > 0 && go_space((*s)[*n - SKIP_ONE])) { + (*n)--; + } +} + +/* ── UTF-8 and the two Unicode classes go/doc/comment asks for ───── */ + +typedef struct { + uint32_t lo; + uint32_t hi; + uint32_t stride; +} go_range_t; + +/* General category P* above U+007F (Unicode 16.0.0), ascending. */ +static const go_range_t GO_PUNCT[] = { + {0x00A1, 0x00A1, 1}, {0x00A7, 0x00A7, 1}, {0x00AB, 0x00AB, 1}, {0x00B6, 0x00B7, 1}, + {0x00BB, 0x00BB, 1}, {0x00BF, 0x00BF, 1}, {0x037E, 0x037E, 1}, {0x0387, 0x0387, 1}, + {0x055A, 0x055F, 1}, {0x0589, 0x058A, 1}, {0x05BE, 0x05BE, 1}, {0x05C0, 0x05C0, 1}, + {0x05C3, 0x05C3, 1}, {0x05C6, 0x05C6, 1}, {0x05F3, 0x05F4, 1}, {0x0609, 0x060A, 1}, + {0x060C, 0x060D, 1}, {0x061B, 0x061B, 1}, {0x061D, 0x061F, 1}, {0x066A, 0x066D, 1}, + {0x06D4, 0x06D4, 1}, {0x0700, 0x070D, 1}, {0x07F7, 0x07F9, 1}, {0x0830, 0x083E, 1}, + {0x085E, 0x085E, 1}, {0x0964, 0x0965, 1}, {0x0970, 0x0970, 1}, {0x09FD, 0x09FD, 1}, + {0x0A76, 0x0A76, 1}, {0x0AF0, 0x0AF0, 1}, {0x0C77, 0x0C77, 1}, {0x0C84, 0x0C84, 1}, + {0x0DF4, 0x0DF4, 1}, {0x0E4F, 0x0E4F, 1}, {0x0E5A, 0x0E5B, 1}, {0x0F04, 0x0F12, 1}, + {0x0F14, 0x0F14, 1}, {0x0F3A, 0x0F3D, 1}, {0x0F85, 0x0F85, 1}, {0x0FD0, 0x0FD4, 1}, + {0x0FD9, 0x0FDA, 1}, {0x104A, 0x104F, 1}, {0x10FB, 0x10FB, 1}, {0x1360, 0x1368, 1}, + {0x1400, 0x1400, 1}, {0x166E, 0x166E, 1}, {0x169B, 0x169C, 1}, {0x16EB, 0x16ED, 1}, + {0x1735, 0x1736, 1}, {0x17D4, 0x17D6, 1}, {0x17D8, 0x17DA, 1}, {0x1800, 0x180A, 1}, + {0x1944, 0x1945, 1}, {0x1A1E, 0x1A1F, 1}, {0x1AA0, 0x1AA6, 1}, {0x1AA8, 0x1AAD, 1}, + {0x1B4E, 0x1B4F, 1}, {0x1B5A, 0x1B60, 1}, {0x1B7D, 0x1B7F, 1}, {0x1BFC, 0x1BFF, 1}, + {0x1C3B, 0x1C3F, 1}, {0x1C7E, 0x1C7F, 1}, {0x1CC0, 0x1CC7, 1}, {0x1CD3, 0x1CD3, 1}, + {0x2010, 0x2027, 1}, {0x2030, 0x2043, 1}, {0x2045, 0x2051, 1}, {0x2053, 0x205E, 1}, + {0x207D, 0x207E, 1}, {0x208D, 0x208E, 1}, {0x2308, 0x230B, 1}, {0x2329, 0x232A, 1}, + {0x2768, 0x2775, 1}, {0x27C5, 0x27C6, 1}, {0x27E6, 0x27EF, 1}, {0x2983, 0x2998, 1}, + {0x29D8, 0x29DB, 1}, {0x29FC, 0x29FD, 1}, {0x2CF9, 0x2CFC, 1}, {0x2CFE, 0x2CFF, 1}, + {0x2D70, 0x2D70, 1}, {0x2E00, 0x2E2E, 1}, {0x2E30, 0x2E4F, 1}, {0x2E52, 0x2E5D, 1}, + {0x3001, 0x3003, 1}, {0x3008, 0x3011, 1}, {0x3014, 0x301F, 1}, {0x3030, 0x3030, 1}, + {0x303D, 0x303D, 1}, {0x30A0, 0x30A0, 1}, {0x30FB, 0x30FB, 1}, {0xA4FE, 0xA4FF, 1}, + {0xA60D, 0xA60F, 1}, {0xA673, 0xA673, 1}, {0xA67E, 0xA67E, 1}, {0xA6F2, 0xA6F7, 1}, + {0xA874, 0xA877, 1}, {0xA8CE, 0xA8CF, 1}, {0xA8F8, 0xA8FA, 1}, {0xA8FC, 0xA8FC, 1}, + {0xA92E, 0xA92F, 1}, {0xA95F, 0xA95F, 1}, {0xA9C1, 0xA9CD, 1}, {0xA9DE, 0xA9DF, 1}, + {0xAA5C, 0xAA5F, 1}, {0xAADE, 0xAADF, 1}, {0xAAF0, 0xAAF1, 1}, {0xABEB, 0xABEB, 1}, + {0xFD3E, 0xFD3F, 1}, {0xFE10, 0xFE19, 1}, {0xFE30, 0xFE52, 1}, {0xFE54, 0xFE61, 1}, + {0xFE63, 0xFE63, 1}, {0xFE68, 0xFE68, 1}, {0xFE6A, 0xFE6B, 1}, {0xFF01, 0xFF03, 1}, + {0xFF05, 0xFF0A, 1}, {0xFF0C, 0xFF0F, 1}, {0xFF1A, 0xFF1B, 1}, {0xFF1F, 0xFF20, 1}, + {0xFF3B, 0xFF3D, 1}, {0xFF3F, 0xFF3F, 1}, {0xFF5B, 0xFF5B, 1}, {0xFF5D, 0xFF5D, 1}, + {0xFF5F, 0xFF65, 1}, {0x10100, 0x10102, 1}, {0x1039F, 0x1039F, 1}, {0x103D0, 0x103D0, 1}, + {0x1056F, 0x1056F, 1}, {0x10857, 0x10857, 1}, {0x1091F, 0x1091F, 1}, {0x1093F, 0x1093F, 1}, + {0x10A50, 0x10A58, 1}, {0x10A7F, 0x10A7F, 1}, {0x10AF0, 0x10AF6, 1}, {0x10B39, 0x10B3F, 1}, + {0x10B99, 0x10B9C, 1}, {0x10D6E, 0x10D6E, 1}, {0x10EAD, 0x10EAD, 1}, {0x10F55, 0x10F59, 1}, + {0x10F86, 0x10F89, 1}, {0x11047, 0x1104D, 1}, {0x110BB, 0x110BC, 1}, {0x110BE, 0x110C1, 1}, + {0x11140, 0x11143, 1}, {0x11174, 0x11175, 1}, {0x111C5, 0x111C8, 1}, {0x111CD, 0x111CD, 1}, + {0x111DB, 0x111DB, 1}, {0x111DD, 0x111DF, 1}, {0x11238, 0x1123D, 1}, {0x112A9, 0x112A9, 1}, + {0x113D4, 0x113D5, 1}, {0x113D7, 0x113D8, 1}, {0x1144B, 0x1144F, 1}, {0x1145A, 0x1145B, 1}, + {0x1145D, 0x1145D, 1}, {0x114C6, 0x114C6, 1}, {0x115C1, 0x115D7, 1}, {0x11641, 0x11643, 1}, + {0x11660, 0x1166C, 1}, {0x116B9, 0x116B9, 1}, {0x1173C, 0x1173E, 1}, {0x1183B, 0x1183B, 1}, + {0x11944, 0x11946, 1}, {0x119E2, 0x119E2, 1}, {0x11A3F, 0x11A46, 1}, {0x11A9A, 0x11A9C, 1}, + {0x11A9E, 0x11AA2, 1}, {0x11B00, 0x11B09, 1}, {0x11BE1, 0x11BE1, 1}, {0x11C41, 0x11C45, 1}, + {0x11C70, 0x11C71, 1}, {0x11EF7, 0x11EF8, 1}, {0x11F43, 0x11F4F, 1}, {0x11FFF, 0x11FFF, 1}, + {0x12470, 0x12474, 1}, {0x12FF1, 0x12FF2, 1}, {0x16A6E, 0x16A6F, 1}, {0x16AF5, 0x16AF5, 1}, + {0x16B37, 0x16B3B, 1}, {0x16B44, 0x16B44, 1}, {0x16D6D, 0x16D6F, 1}, {0x16E97, 0x16E9A, 1}, + {0x16FE2, 0x16FE2, 1}, {0x1BC9F, 0x1BC9F, 1}, {0x1DA87, 0x1DA8B, 1}, {0x1E5FF, 0x1E5FF, 1}, + {0x1E95E, 0x1E95F, 1}, +}; + +/* General category Lu above U+007F (Unicode 16.0.0), ascending; a stride of 2 + * covers the runs where upper and lower case alternate. */ +static const go_range_t GO_UPPER[] = { + {0x00C0, 0x00D6, 1}, {0x00D8, 0x00DE, 1}, {0x0100, 0x0136, 2}, {0x0139, 0x0147, 2}, + {0x014A, 0x0176, 2}, {0x0178, 0x0179, 1}, {0x017B, 0x017D, 2}, {0x0181, 0x0182, 1}, + {0x0184, 0x0184, 1}, {0x0186, 0x0187, 1}, {0x0189, 0x018B, 1}, {0x018E, 0x0191, 1}, + {0x0193, 0x0194, 1}, {0x0196, 0x0198, 1}, {0x019C, 0x019D, 1}, {0x019F, 0x01A0, 1}, + {0x01A2, 0x01A4, 2}, {0x01A6, 0x01A7, 1}, {0x01A9, 0x01A9, 1}, {0x01AC, 0x01AC, 1}, + {0x01AE, 0x01AF, 1}, {0x01B1, 0x01B3, 1}, {0x01B5, 0x01B5, 1}, {0x01B7, 0x01B8, 1}, + {0x01BC, 0x01BC, 1}, {0x01C4, 0x01C4, 1}, {0x01C7, 0x01C7, 1}, {0x01CA, 0x01CA, 1}, + {0x01CD, 0x01DB, 2}, {0x01DE, 0x01EE, 2}, {0x01F1, 0x01F1, 1}, {0x01F4, 0x01F4, 1}, + {0x01F6, 0x01F8, 1}, {0x01FA, 0x0232, 2}, {0x023A, 0x023B, 1}, {0x023D, 0x023E, 1}, + {0x0241, 0x0241, 1}, {0x0243, 0x0246, 1}, {0x0248, 0x024E, 2}, {0x0370, 0x0372, 2}, + {0x0376, 0x0376, 1}, {0x037F, 0x037F, 1}, {0x0386, 0x0386, 1}, {0x0388, 0x038A, 1}, + {0x038C, 0x038C, 1}, {0x038E, 0x038F, 1}, {0x0391, 0x03A1, 1}, {0x03A3, 0x03AB, 1}, + {0x03CF, 0x03CF, 1}, {0x03D2, 0x03D4, 1}, {0x03D8, 0x03EE, 2}, {0x03F4, 0x03F4, 1}, + {0x03F7, 0x03F7, 1}, {0x03F9, 0x03FA, 1}, {0x03FD, 0x042F, 1}, {0x0460, 0x0480, 2}, + {0x048A, 0x04BE, 2}, {0x04C0, 0x04C1, 1}, {0x04C3, 0x04CD, 2}, {0x04D0, 0x052E, 2}, + {0x0531, 0x0556, 1}, {0x10A0, 0x10C5, 1}, {0x10C7, 0x10C7, 1}, {0x10CD, 0x10CD, 1}, + {0x13A0, 0x13F5, 1}, {0x1C89, 0x1C89, 1}, {0x1C90, 0x1CBA, 1}, {0x1CBD, 0x1CBF, 1}, + {0x1E00, 0x1E94, 2}, {0x1E9E, 0x1EFE, 2}, {0x1F08, 0x1F0F, 1}, {0x1F18, 0x1F1D, 1}, + {0x1F28, 0x1F2F, 1}, {0x1F38, 0x1F3F, 1}, {0x1F48, 0x1F4D, 1}, {0x1F59, 0x1F5F, 2}, + {0x1F68, 0x1F6F, 1}, {0x1FB8, 0x1FBB, 1}, {0x1FC8, 0x1FCB, 1}, {0x1FD8, 0x1FDB, 1}, + {0x1FE8, 0x1FEC, 1}, {0x1FF8, 0x1FFB, 1}, {0x2102, 0x2102, 1}, {0x2107, 0x2107, 1}, + {0x210B, 0x210D, 1}, {0x2110, 0x2112, 1}, {0x2115, 0x2115, 1}, {0x2119, 0x211D, 1}, + {0x2124, 0x2128, 2}, {0x212A, 0x212D, 1}, {0x2130, 0x2133, 1}, {0x213E, 0x213F, 1}, + {0x2145, 0x2145, 1}, {0x2183, 0x2183, 1}, {0x2C00, 0x2C2F, 1}, {0x2C60, 0x2C60, 1}, + {0x2C62, 0x2C64, 1}, {0x2C67, 0x2C6B, 2}, {0x2C6D, 0x2C70, 1}, {0x2C72, 0x2C72, 1}, + {0x2C75, 0x2C75, 1}, {0x2C7E, 0x2C80, 1}, {0x2C82, 0x2CE2, 2}, {0x2CEB, 0x2CED, 2}, + {0x2CF2, 0x2CF2, 1}, {0xA640, 0xA66C, 2}, {0xA680, 0xA69A, 2}, {0xA722, 0xA72E, 2}, + {0xA732, 0xA76E, 2}, {0xA779, 0xA77B, 2}, {0xA77D, 0xA77E, 1}, {0xA780, 0xA786, 2}, + {0xA78B, 0xA78D, 2}, {0xA790, 0xA792, 2}, {0xA796, 0xA7A8, 2}, {0xA7AA, 0xA7AE, 1}, + {0xA7B0, 0xA7B4, 1}, {0xA7B6, 0xA7C2, 2}, {0xA7C4, 0xA7C7, 1}, {0xA7C9, 0xA7C9, 1}, + {0xA7CB, 0xA7CC, 1}, {0xA7D0, 0xA7D0, 1}, {0xA7D6, 0xA7DC, 2}, {0xA7F5, 0xA7F5, 1}, + {0xFF21, 0xFF3A, 1}, {0x10400, 0x10427, 1}, {0x104B0, 0x104D3, 1}, {0x10570, 0x1057A, 1}, + {0x1057C, 0x1058A, 1}, {0x1058C, 0x10592, 1}, {0x10594, 0x10595, 1}, {0x10C80, 0x10CB2, 1}, + {0x10D50, 0x10D65, 1}, {0x118A0, 0x118BF, 1}, {0x16E40, 0x16E5F, 1}, {0x1D400, 0x1D419, 1}, + {0x1D434, 0x1D44D, 1}, {0x1D468, 0x1D481, 1}, {0x1D49C, 0x1D49C, 1}, {0x1D49E, 0x1D49F, 1}, + {0x1D4A2, 0x1D4A2, 1}, {0x1D4A5, 0x1D4A6, 1}, {0x1D4A9, 0x1D4AC, 1}, {0x1D4AE, 0x1D4B5, 1}, + {0x1D4D0, 0x1D4E9, 1}, {0x1D504, 0x1D505, 1}, {0x1D507, 0x1D50A, 1}, {0x1D50D, 0x1D514, 1}, + {0x1D516, 0x1D51C, 1}, {0x1D538, 0x1D539, 1}, {0x1D53B, 0x1D53E, 1}, {0x1D540, 0x1D544, 1}, + {0x1D546, 0x1D546, 1}, {0x1D54A, 0x1D550, 1}, {0x1D56C, 0x1D585, 1}, {0x1D5A0, 0x1D5B9, 1}, + {0x1D5D4, 0x1D5ED, 1}, {0x1D608, 0x1D621, 1}, {0x1D63C, 0x1D655, 1}, {0x1D670, 0x1D689, 1}, + {0x1D6A8, 0x1D6C0, 1}, {0x1D6E2, 0x1D6FA, 1}, {0x1D71C, 0x1D734, 1}, {0x1D756, 0x1D76E, 1}, + {0x1D790, 0x1D7A8, 1}, {0x1D7CA, 0x1D7CA, 1}, {0x1E900, 0x1E921, 1}, +}; + +enum { + GO_PUNCT_COUNT = sizeof(GO_PUNCT) / sizeof(GO_PUNCT[0]), + GO_UPPER_COUNT = sizeof(GO_UPPER) / sizeof(GO_UPPER[0]), + GO_U8_CONT_MASK = 0xC0, + GO_U8_CONT = 0x80, + GO_U8_PAYLOAD = 0x3F, + GO_U8_SHIFT = 6, + GO_U8_LEAD2 = 0xC0, + GO_U8_LEAD3 = 0xE0, + GO_U8_LEAD4 = 0xF0, + GO_U8_LEAD_END = 0xF8, + GO_U8_MASK2 = 0x1F, + GO_U8_MASK3 = 0x0F, + GO_U8_MASK4 = 0x07, + GO_U8_MIN2 = 0x80, + GO_U8_MIN3 = 0x800, + GO_U8_MIN4 = 0x10000, + GO_U8_LEN3 = 3, + GO_U8_LEN4 = 4, + GO_SURROGATE_LO = 0xD800, + GO_SURROGATE_HI = 0xDFFF, + GO_RUNE_MAX = 0x10FFFF, +}; + +/* Not a rune: invalid UTF-8. */ +static const uint32_t GO_RUNE_BAD = 0xFFFFFFFFU; + +/* Length of the sequence a lead byte announces; 0 for a byte that leads none. */ +static int go_u8_len(unsigned char c) { + if (c < GO_ASCII_END) { + return SKIP_ONE; + } + if (c < GO_U8_LEAD2 || c >= GO_U8_LEAD_END) { + return 0; + } + if (c < GO_U8_LEAD3) { + return PAIR_LEN; + } + return c < GO_U8_LEAD4 ? GO_U8_LEN3 : GO_U8_LEN4; +} + +/* The rune encoded by exactly s[0..n), or GO_RUNE_BAD. Overlong forms, + * surrogates and values past U+10FFFF are invalid, as for utf8.DecodeRune. */ +static uint32_t go_rune_decode(const unsigned char *s, int n) { + if (n == SKIP_ONE) { + return s[0]; + } + static const uint32_t min_value[] = {0, 0, GO_U8_MIN2, GO_U8_MIN3, GO_U8_MIN4}; + static const unsigned char lead_mask[] = {0, 0, GO_U8_MASK2, GO_U8_MASK3, GO_U8_MASK4}; + uint32_t r = (uint32_t)(s[0] & lead_mask[n]); + for (int i = SKIP_ONE; i < n; i++) { + if ((s[i] & GO_U8_CONT_MASK) != GO_U8_CONT) { + return GO_RUNE_BAD; + } + r = (r << GO_U8_SHIFT) | (uint32_t)(s[i] & GO_U8_PAYLOAD); + } + if (r < min_value[n] || r > GO_RUNE_MAX || (r >= GO_SURROGATE_LO && r <= GO_SURROGATE_HI)) { + return GO_RUNE_BAD; + } + return r; +} + +/* The rune that starts at s[at] (s has n bytes, at < n). */ +static uint32_t go_rune_at(const char *s, size_t n, size_t at) { + const unsigned char *u = (const unsigned char *)s + at; + int len = go_u8_len(u[0]); + if (len == 0 || (size_t)len > n - at) { + return GO_RUNE_BAD; + } + return go_rune_decode(u, len); +} + +/* The rune that ends just before s[end] (end > 0). */ +static uint32_t go_rune_before(const char *s, size_t end) { + const unsigned char *u = (const unsigned char *)s; + size_t start = end - SKIP_ONE; + int back = SKIP_ONE; + while (start > 0 && back < GO_U8_LEN4 && (u[start] & GO_U8_CONT_MASK) == GO_U8_CONT) { + start--; + back++; + } + if (go_u8_len(u[start]) != back) { + return GO_RUNE_BAD; + } + return go_rune_decode(u + start, back); +} + +static bool go_in_ranges(const go_range_t *ranges, int count, uint32_t r) { + int lo = 0; + int hi = count - SKIP_ONE; + while (lo <= hi) { + int mid = lo + ((hi - lo) / PAIR_LEN); + if (r < ranges[mid].lo) { + hi = mid - SKIP_ONE; + } else if (r > ranges[mid].hi) { + lo = mid + SKIP_ONE; + } else { + return (r - ranges[mid].lo) % ranges[mid].stride == 0; + } + } + return false; +} + +/* unicode.IsPunct. */ +static bool go_is_punct(uint32_t r) { + if (r < GO_ASCII_END) { + return r != 0 && strchr("!\"#%&'()*,-./:;?@[\\]_{}", (int)r) != NULL; + } + return go_in_ranges(GO_PUNCT, GO_PUNCT_COUNT, r); +} + +/* unicode.IsUpper. */ +static bool go_is_upper(uint32_t r) { + if (r < GO_ASCII_END) { + return r >= 'A' && r <= 'Z'; + } + return r != GO_RUNE_BAD && go_in_ranges(GO_UPPER, GO_UPPER_COUNT, r); +} + +/* docLink's border rule for one neighbour of the brackets. */ +static bool go_border_ok(uint32_t r) { + return r == ' ' || r == '\t' || r == '\n' || (r != GO_RUNE_BAD && go_is_punct(r)); +} + +/* ── Identifiers and reference shape (docLink, splitDocName) ─────── */ + +/* A byte of a Go identifier. Every non-ASCII byte counts as a letter: text + * that is no identifier then names nothing a package declares, and the + * resolver treats it as plain text. */ +static bool go_ident_byte(unsigned char c) { + return go_ascii_alpha(c) || go_ascii_digit(c) || c == '_' || c >= GO_ASCII_END; +} + +/* go/doc/comment ident(): s is one Go identifier. */ +static bool go_is_ident(const char *s, size_t n) { + if (n == 0 || n > GO_REF_MAX || go_ascii_digit((unsigned char)s[0])) { + return false; + } + for (size_t i = 0; i < n; i++) { + if (!go_ident_byte((unsigned char)s[i])) { + return false; + } + } + return true; +} + +/* go/doc/comment isName(): an identifier whose first letter is upper case, + * which is also what makes a Go identifier exported. */ +static bool go_is_name(const char *s, size_t n) { + return go_is_ident(s, n) && go_is_upper(go_rune_at(s, n, 0)); +} + +static bool go_import_path_byte(unsigned char c) { + return go_ascii_alpha(c) || go_ascii_digit(c) || c == '-' || c == '.' || c == '_' || c == '~' || + c == '+'; +} + +static bool go_import_elem_ok(const char *s, size_t n) { + if (n == 0 || s[0] == '.' || s[n - SKIP_ONE] == '.') { + return false; + } + for (size_t i = 0; i < n; i++) { + if (!go_import_path_byte((unsigned char)s[i])) { + return false; + } + } + return true; +} + +/* go/doc/comment validImportPath. */ +static bool go_valid_import_path(const char *s, size_t n) { + if (n == 0 || n > GO_PATH_MAX || s[0] == '-' || s[n - SKIP_ONE] == '/') { + return false; + } + size_t start = 0; + for (size_t i = 0; i <= n; i++) { + if (i == n || s[i] == '/') { + if (!go_import_elem_ok(s + start, i - start)) { + return false; + } + start = i + SKIP_ONE; + } + } + return true; +} + +/* An import name as a file can give one: identifier bytes only. */ +static bool go_pkg_word(const char *s, size_t n) { + if (n == 0 || n > GO_REF_MAX) { + return false; + } + for (size_t i = 0; i < n; i++) { + if (!go_ident_byte((unsigned char)s[i])) { + return false; + } + } + return true; +} + +/* splitDocName: `before.Name` with a capitalized Name. False when the text + * after the last dot is no Name (then nothing is split off). */ +static bool go_split_name(const char *s, size_t n, size_t *before_len, const char **name, + size_t *name_len) { + size_t dot = n; + for (size_t i = n; i > 0; i--) { + if (s[i - SKIP_ONE] == '.') { + dot = i - SKIP_ONE; + break; + } + } + size_t ns = dot < n ? dot + SKIP_ONE : 0; + if (!go_is_name(s + ns, n - ns)) { + return false; + } + *name = s + ns; + *name_len = n - ns; + *before_len = dot < n ? dot : 0; + return true; +} + +bool cbm_doclink_go_parse_ref(const char *raw, size_t len, CBMDocLinkGoRef *out) { + memset(out, 0, sizeof(*out)); + if (!raw || len == 0 || len > GO_REF_MAX) { + return false; + } + const char *s = raw; + size_t n = len; + if (s[0] == '*') { + s++; + n--; + } + size_t before = 0; + out->pkg = s; + out->pkg_len = n; + if (go_split_name(s, n, &before, &out->name, &out->name_len)) { + size_t pkg_len = 0; + out->pkg_len = before; + if (go_split_name(s, before, &pkg_len, &out->recv, &out->recv_len)) { + out->pkg_len = pkg_len; + } + } + if (out->pkg_len == 0) { + return out->name_len > 0; /* [Name], [Recv.Name]: the current package */ + } + if (memchr(out->pkg, '/', out->pkg_len)) { + return go_valid_import_path(out->pkg, out->pkg_len); + } + return go_pkg_word(out->pkg, out->pkg_len); +} + +/* Bracket text longer than this scan handles: true when its bytes are those a + * doc link is made of, so that it is reported instead of dropped. */ +static bool go_long_ref_shaped(const char *s, size_t n) { + size_t i = (n > 0 && s[0] == '*') ? SKIP_ONE : 0; + if (i >= n) { + return false; + } + for (; i < n; i++) { + unsigned char c = (unsigned char)s[i]; + if (!go_ident_byte(c) && !go_import_path_byte(c) && c != '/') { + return false; + } + } + return true; +} + +/* The longest prefix of s[0..n) of at most `max` bytes that ends on a rune + * boundary. */ +static size_t go_clip(const char *s, size_t n, size_t max) { + if (n <= max) { + return n; + } + size_t cut = max; + while (cut > 0 && ((unsigned char)s[cut] & GO_U8_CONT_MASK) == GO_U8_CONT) { + cut--; + } + return cut; +} + +/* ── Doc text model ──────────────────────────────────────────────── */ + +/* One line of the comment text: a slice of the doc string. */ +typedef struct { + const char *s; + uint32_t len; + uint32_t line; /* 1-based source line */ +} go_line_t; + +enum { GO_SPAN_CODE = 1, GO_SPAN_HEADING, GO_SPAN_LIST, GO_SPAN_PARA }; + +typedef struct { + int start; + int end; + int kind; +} go_span_t; + +/* `[text]: url`, one line of a link-definition paragraph. */ +typedef struct { + const char *text; + size_t text_len; + const char *url; + size_t url_len; + int order; +} go_linkdef_t; + +typedef struct { + CBMExtractCtx *ctx; + CBMArena *tmp; + const char *source_qn; + const char **params; /* the documented definition's parameter names, or NULL */ + uint32_t def_line; + go_line_t *lines; + int nlines; + int lines_cap; + go_span_t *spans; + int nspans; + int spans_cap; + go_linkdef_t *defs; + int ndefs; + int defs_cap; + go_line_t *para; /* the lines of one list-item paragraph */ + bool failed; +} go_doc_t; + +static void go_push_line(go_doc_t *d, const char *s, size_t len, uint32_t line) { + while (len > 0 && go_space(s[len - SKIP_ONE])) { + len--; /* ast.CommentGroup.Text strips trailing white space */ + } + go_line_t *grown = + (go_line_t *)go_grow(d->tmp, d->lines, d->nlines, &d->lines_cap, sizeof(go_line_t)); + if (!grown) { + d->failed = true; + return; + } + d->lines = grown; + d->lines[d->nlines++] = (go_line_t){.s = s, .len = (uint32_t)len, .line = line}; +} + +static bool go_lower_or_digit(char b) { + return (b >= 'a' && b <= 'z') || (b >= '0' && b <= '9'); +} + +/* go/ast isDirective: //line, //extern, //export and //[a-z0-9]+:[a-z0-9]. */ +static bool go_is_directive(const char *c, size_t n) { + if (go_has_prefix(c, n, "line ") || go_has_prefix(c, n, "extern ") || + go_has_prefix(c, n, "export ")) { + return true; + } + const char *colon = (const char *)memchr(c, ':', n); + if (!colon || colon == c || (size_t)(colon - c) + SKIP_ONE >= n) { + return false; + } + size_t ci = (size_t)(colon - c); + for (size_t i = 0; i <= ci + SKIP_ONE; i++) { + if (i != ci && !go_lower_or_digit(c[i])) { + return false; + } + } + return true; +} + +/* The text of one `//` comment (c = what follows the slashes). */ +static void go_line_comment(go_doc_t *d, const char *c, size_t n, uint32_t line) { + if (n > 0 && c[0] == ' ') { + go_push_line(d, c + SKIP_ONE, n - SKIP_ONE, line); + } else if (n == 0 || !go_is_directive(c, n)) { + go_push_line(d, c, n, line); + } +} + +/* The lines of one block comment body [c, end); returns the line breaks in it. */ +static uint32_t go_block_comment(go_doc_t *d, const char *c, const char *end, uint32_t line) { + uint32_t added = 0; + for (;;) { + const char *nl = (const char *)memchr(c, '\n', (size_t)(end - c)); + const char *le = nl ? nl : end; + go_push_line(d, c, (size_t)(le - c), line + added); + if (!nl) { + return added; + } + added++; + c = nl + SKIP_ONE; + } +} + +/* ast.CommentGroup.Text over the stored comment group: one comment per text + * line, a block comment keeping its own lines. Text line k of the group is + * source line doc_line + k. Every byte is looked at once: a comment is + * consumed whole, then the scan continues behind its line. */ +static void go_group_lines(go_doc_t *d, const char *doc, uint32_t doc_line) { + const char *p = doc; + uint32_t line = doc_line; + while (*p && !d->failed) { + const char *rest = p; + if (p[0] == '/' && p[SKIP_ONE] == '/') { + const char *eol = strchr(p, '\n'); + const char *end = eol ? eol : p + strlen(p); + go_line_comment(d, p + GO_MARKER_LEN, (size_t)(end - p) - GO_MARKER_LEN, line); + rest = end; + } else if (p[0] == '/' && p[SKIP_ONE] == '*') { + const char *body = p + GO_MARKER_LEN; + const char *close = strstr(body, "*/"); + const char *end = close ? close : body + strlen(body); + line += go_block_comment(d, body, end, line); + rest = close ? close + GO_MARKER_LEN : end; + } + const char *eol = strchr(rest, '\n'); + if (!eol) { + break; + } + p = eol + SKIP_ONE; + line++; + } +} + +static size_t go_leading_space(const go_line_t *l) { + size_t i = 0; + while (i < l->len && go_blank_or_tab(l->s[i])) { + i++; + } + return i; +} + +/* go/doc/comment unindent: drop the blank lines at both ends and the common + * space/tab prefix of the others. */ +static void go_unindent(go_doc_t *d) { + int first = 0; + int last = d->nlines; + while (first < last && d->lines[first].len == 0) { + first++; + } + while (last > first && d->lines[last - SKIP_ONE].len == 0) { + last--; + } + d->lines += first; + d->nlines = last - first; + d->lines_cap -= first; + if (d->nlines == 0) { + return; + } + const char *prefix = d->lines[0].s; + size_t plen = go_leading_space(&d->lines[0]); + for (int i = SKIP_ONE; i < d->nlines && plen > 0; i++) { + const go_line_t *l = &d->lines[i]; + if (l->len == 0) { + continue; + } + size_t ls = go_leading_space(l); + size_t k = 0; + while (k < plen && k < ls && prefix[k] == l->s[k]) { + k++; + } + plen = k; + } + for (int i = 0; i < d->nlines && plen > 0; i++) { + go_line_t *l = &d->lines[i]; + if (l->len >= plen) { + l->s += plen; + l->len -= (uint32_t)plen; + } + } +} + +static bool go_indented(const go_line_t *l) { + return l->len > 0 && go_blank_or_tab(l->s[0]); +} + +/* go/doc/comment listMarker: the text after a bullet (`*`, `+`, `-`, U+2022) + * or a number followed by `.` or `)`; NULL when the line starts no list item. + * The marker must be followed by a space or tab and then text. */ +static const char *go_list_marker(const go_line_t *l, size_t *rest_len, bool *numbered) { + static const char bullet[] = "\xE2\x80\xA2"; + const size_t bullet_len = sizeof(bullet) - SKIP_ONE; + const char *s = l->s; + size_t n = l->len; + go_trim(&s, &n); + size_t skip = 0; + *numbered = false; + if (n > 0 && (s[0] == '*' || s[0] == '+' || s[0] == '-')) { + skip = SKIP_ONE; + } else if (go_has_prefix(s, n, bullet)) { + skip = bullet_len; + } else { + while (skip < n && go_ascii_digit((unsigned char)s[skip])) { + skip++; + } + if (skip == 0 || skip >= n || (s[skip] != '.' && s[skip] != ')')) { + return NULL; + } + skip++; + *numbered = true; + } + const char *rest = s + skip; + size_t rn = n - skip; + if (rn == 0 || !go_blank_or_tab(rest[0])) { + return NULL; + } + *rest_len = rn; + return rest; /* n was trimmed, so text follows the white space */ +} + +static bool go_is_list(const go_line_t *l) { + size_t rest_len = 0; + bool numbered = false; + return go_list_marker(l, &rest_len, &numbered) != NULL; +} + +/* go/doc/comment isHeading (trailing white space is already gone). */ +static bool go_is_heading(const go_line_t *l) { + return l->len >= PAIR_LEN && l->s[0] == '#' && go_blank_or_tab(l->s[SKIP_ONE]); +} + +static bool go_push_span(go_doc_t *d, int start, int end, int kind) { + go_span_t *grown = + (go_span_t *)go_grow(d->tmp, d->spans, d->nspans, &d->spans_cap, sizeof(go_span_t)); + if (!grown) { + d->failed = true; + return false; + } + d->spans = grown; + d->spans[d->nspans++] = (go_span_t){.start = start, .end = end, .kind = kind}; + return true; +} + +/* An indented (or force-indented) run starting at `start`: a code block or a + * list. Blank lines belong to it; an unindented list that is being treated as + * indented takes further unindented list items up to the first blank line. A + * `}` line right after the run is taken along (pasted, unindented code). */ +static int go_indented_span(const go_doc_t *d, int start, int force_indent, int *kind) { + const go_line_t *L = d->lines; + bool starts_list = go_is_list(&L[start]); + bool unindented_list_ok = starts_list && start < force_indent; + int i = start + SKIP_ONE; + while (i < d->nlines && (L[i].len == 0 || i < force_indent || go_indented(&L[i]) || + (unindented_list_ok && go_is_list(&L[i])))) { + if (L[i].len == 0) { + unindented_list_ok = false; + } + i++; + } + int end = i; + while (end > start && L[end - SKIP_ONE].len == 0) { + end--; + } + if (end < d->nlines && L[end].len > 0 && L[end].s[0] == '}') { + end++; + } + *kind = starts_list ? GO_SPAN_LIST : GO_SPAN_CODE; + return end; +} + +static bool go_ends_with(const go_line_t *l, char c) { + return l->len > 0 && l->s[l->len - SKIP_ONE] == c; +} + +/* An unindented run starting at `start`: a paragraph or a heading. When an + * indented code block follows without a blank line, trailing lines that look + * like its unindented beginning (list items; a line ending in `{` or `\`) + * are left for it: *force_indent is then the block's first indented line. */ +static int go_unindented_span(const go_doc_t *d, int start, int *force_indent) { + const go_line_t *L = d->lines; + int i = start + SKIP_ONE; + while (i < d->nlines && L[i].len > 0 && !go_indented(&L[i])) { + i++; + } + int end = i; + if (i < d->nlines && L[i].len > 0 && !go_is_list(&L[i])) { + if (go_is_list(&L[i - SKIP_ONE])) { + *force_indent = end; + end--; + while (end > start && go_is_list(&L[end - SKIP_ONE])) { + end--; + } + } else if (go_ends_with(&L[i - SKIP_ONE], '{') || go_ends_with(&L[i - SKIP_ONE], '\\')) { + *force_indent = end; + end--; + } + } + return end; +} + +/* go/doc/comment parseSpans. A line is looked at as unindented text at most + * once and as part of an indented run at most once. */ +static void go_parse_spans(go_doc_t *d) { + int i = 0; + int force_indent = 0; + while (!d->failed) { + while (i < d->nlines && d->lines[i].len == 0) { + i++; + } + if (i >= d->nlines) { + return; + } + int kind = GO_SPAN_PARA; + bool indented = i < force_indent || go_indented(&d->lines[i]); + int end = indented ? go_indented_span(d, i, force_indent, &kind) + : go_unindented_span(d, i, &force_indent); + if (end == i) { + continue; /* every line was left for the block: now force-indented */ + } + if (!indented && end == i + SKIP_ONE && go_is_heading(&d->lines[i])) { + kind = GO_SPAN_HEADING; /* one unindented line: `# Title` */ + } + if (!go_push_span(d, i, end, kind)) { + return; + } + i = end; + } +} + +/* ── Link definitions ────────────────────────────────────────────── */ + +static bool go_is_scheme(const char *s, size_t n) { + static const char *const schemes[] = {"file", "ftp", "gopher", "http", + "https", "mailto", "nntp"}; + for (size_t i = 0; i < sizeof(schemes) / sizeof(schemes[0]); i++) { + if (strlen(schemes[i]) == n && memcmp(schemes[i], s, n) == 0) { + return true; + } + } + return false; +} + +/* Offset of the first occurrence of the two bytes `a b` in s, or n. */ +static size_t go_find2(const char *s, size_t n, char a, char b) { + for (size_t i = 0; i + SKIP_ONE < n; i++) { + if (s[i] == a && s[i + SKIP_ONE] == b) { + return i; + } + } + return n; +} + +/* go/doc/comment parseLink: `[text]: url` with a known scheme. */ +static bool go_parse_linkdef(const go_line_t *l, go_linkdef_t *out) { + const char *s = l->s; + size_t n = l->len; + if (n == 0 || s[0] != '[') { + return false; + } + size_t close = go_find2(s, n, ']', ':'); + if (close + GO_LINKDEF_TAIL >= n || !go_blank_or_tab(s[close + PAIR_LEN])) { + return false; + } + const char *url = s + close + GO_LINKDEF_TAIL; + size_t url_len = n - close - GO_LINKDEF_TAIL; + go_trim(&url, &url_len); + size_t colon = go_find2(url, url_len, ':', '/'); + if (colon + PAIR_LEN >= url_len || url[colon + PAIR_LEN] != '/' || !go_is_scheme(url, colon)) { + return false; + } + out->text = s + SKIP_ONE; + out->text_len = close - SKIP_ONE; + out->url = url; + out->url_len = url_len; + return true; +} + +/* A paragraph whose every line is a link definition defines links and is no + * text. */ +static bool go_para_is_linkdefs(const go_line_t *lines, int n) { + go_linkdef_t def; + for (int i = 0; i < n; i++) { + if (!go_parse_linkdef(&lines[i], &def)) { + return false; + } + } + return n > 0; +} + +typedef void (*go_para_fn)(go_doc_t *d, const go_line_t *lines, int n); + +static void go_collect_linkdefs(go_doc_t *d, const go_line_t *lines, int n) { + if (!go_para_is_linkdefs(lines, n)) { + return; + } + for (int i = 0; i < n && !d->failed; i++) { + go_linkdef_t def = {0}; + (void)go_parse_linkdef(&lines[i], &def); + go_linkdef_t *grown = + (go_linkdef_t *)go_grow(d->tmp, d->defs, d->ndefs, &d->defs_cap, sizeof(go_linkdef_t)); + if (!grown) { + d->failed = true; + return; + } + d->defs = grown; + def.order = d->ndefs; + d->defs[d->ndefs++] = def; + } +} + +/* Byte order of two link texts; the key side reads newline and tab as a + * space, as parseLinkedText does for the text between the brackets. */ +static int go_linktext_cmp(const char *key, size_t key_len, const char *text, size_t text_len, + bool fold_key) { + size_t n = key_len < text_len ? key_len : text_len; + for (size_t i = 0; i < n; i++) { + unsigned char k = (unsigned char)key[i]; + if (fold_key && (k == '\n' || k == '\t')) { + k = ' '; + } + unsigned char t = (unsigned char)text[i]; + if (k != t) { + return k < t ? -SKIP_ONE : SKIP_ONE; + } + } + if (key_len == text_len) { + return 0; + } + return key_len < text_len ? -SKIP_ONE : SKIP_ONE; +} + +static int go_linkdef_cmp(const void *a, const void *b) { + const go_linkdef_t *x = (const go_linkdef_t *)a; + const go_linkdef_t *y = (const go_linkdef_t *)b; + int c = go_linktext_cmp(x->text, x->text_len, y->text, y->text_len, false); + return c ? c : x->order - y->order; +} + +/* The first definition of the link text `key`, or NULL. */ +static const go_linkdef_t *go_find_linkdef(const go_doc_t *d, const char *key, size_t key_len) { + int lo = 0; + int hi = d->ndefs - SKIP_ONE; + const go_linkdef_t *hit = NULL; + while (lo <= hi) { + int mid = lo + ((hi - lo) / PAIR_LEN); + int c = go_linktext_cmp(key, key_len, d->defs[mid].text, d->defs[mid].text_len, true); + if (c == 0) { + hit = &d->defs[mid]; + hi = mid - SKIP_ONE; /* the first of equal texts: defined first */ + } else if (c < 0) { + hi = mid - SKIP_ONE; + } else { + lo = mid + SKIP_ONE; + } + } + return hit; +} + +/* ── Bracket scan (parseLinkedText, docLink) ─────────────────────── */ + +static void go_emit(go_doc_t *d, const char *raw, size_t len, uint32_t line, int syntax) { + char *copy = cbm_arena_strndup(d->ctx->arena, raw, len); + if (!copy) { + d->failed = true; + return; + } + CBMDocLink link = { + .source_qn = d->source_qn, + .raw = copy, + .line = line, + .def_line = d->def_line, + .syntax = (uint16_t)syntax, + }; + cbm_doclinks_push(&d->ctx->result->doc_links, d->ctx->arena, link); +} + +/* True when the reference is one word that names a parameter of the + * documented definition: local, never a reference. */ +static bool go_names_own_param(const go_doc_t *d, const CBMDocLinkGoRef *ref) { + if (!d->params || ref->recv_len > 0 || (ref->pkg_len > 0 && ref->name_len > 0)) { + return false; + } + const char *word = ref->pkg_len > 0 ? ref->pkg : ref->name; + size_t len = ref->pkg_len > 0 ? ref->pkg_len : ref->name_len; + for (const char **p = d->params; *p; p++) { + if (strlen(*p) == len && memcmp(*p, word, len) == 0) { + return true; + } + } + return false; +} + +/* The link definition the pair text[open] .. text[close] names, or NULL. + * parseLinkedText collects the looked-up text from the FIRST opening bracket + * since the last closing one and leaves the brackets themselves out: with one + * opening bracket that is the pair's own text, with `[a [b]` it is `a b`. */ +static const go_linkdef_t *go_pair_linkdef(go_doc_t *d, const char *text, size_t first, size_t open, + size_t close) { + if (d->ndefs == 0) { + return NULL; + } + if (first == open) { + return go_find_linkdef(d, text + open + SKIP_ONE, close - open - SKIP_ONE); + } + char *key = (char *)cbm_arena_alloc(d->tmp, close - first); + if (!key) { + d->failed = true; + return NULL; + } + size_t w = 0; + for (size_t i = first + SKIP_ONE; i < close; i++) { + if (text[i] != '[') { + key[w++] = text[i]; + } + } + return go_find_linkdef(d, key, w); +} + +/* One bracket pair text[open] .. text[close] of a paragraph; `first` is the + * first opening bracket since the last closing one. */ +static void go_bracket(go_doc_t *d, const char *text, size_t len, size_t first, size_t open, + size_t close, uint32_t line) { + const char *inner = text + open + SKIP_ONE; + size_t inner_len = close - open - SKIP_ONE; + const go_linkdef_t *def = go_pair_linkdef(d, text, first, open, close); + if (def) { + /* a URL is external whatever it says: a long one is stored clipped */ + go_emit(d, def->url, go_clip(def->url, def->url_len, GO_URL_MAX), line, CBM_DOCLINK_HREF); + return; + } + if (open > 0 && !go_border_ok(go_rune_before(text, open))) { + return; + } + if (close + SKIP_ONE < len && !go_border_ok(go_rune_at(text, len, close + SKIP_ONE))) { + return; + } + if (inner_len > GO_REF_MAX) { + /* Not decided here, and never by its beginning: the resolver reports + * the clipped text as unparseable. */ + if (go_long_ref_shaped(inner, inner_len)) { + go_emit(d, inner, go_clip(inner, inner_len, (size_t)GO_REF_MAX + SKIP_ONE), line, + CBM_DOCLINK_GO_DOC_LINK); + } + return; + } + CBMDocLinkGoRef ref; + if (!cbm_doclink_go_parse_ref(inner, inner_len, &ref) || go_names_own_param(d, &ref)) { + return; + } + go_emit(d, inner, inner_len, line, CBM_DOCLINK_GO_DOC_LINK); +} + +/* The paragraph's lines joined by newlines, scanned for bracket pairs: `[` + * (re)starts a pair, the next `]` closes it. */ +static void go_scan_paragraph(go_doc_t *d, const go_line_t *lines, int n) { + if (go_para_is_linkdefs(lines, n)) { + return; + } + size_t total = 0; + bool any = false; + for (int i = 0; i < n; i++) { + total += (size_t)lines[i].len + SKIP_ONE; + any = any || memchr(lines[i].s, '[', lines[i].len) != NULL; + } + if (!any) { + return; + } + char *text = (char *)cbm_arena_alloc(d->tmp, total + SKIP_ONE); + if (!text) { + d->failed = true; + return; + } + size_t len = 0; + for (int i = 0; i < n; i++) { + if (i > 0) { + text[len++] = '\n'; + } + memcpy(text + len, lines[i].s, lines[i].len); + len += lines[i].len; + } + text[len] = '\0'; + int piece = 0; /* the line the scan is in */ + int open_piece = 0; + size_t open = len; /* len: no open bracket */ + size_t first = len; /* the first opening bracket since the last closing one */ + for (size_t i = 0; i < len && !d->failed; i++) { + char c = text[i]; + if (c == '\n') { + piece++; + } else if (c == '[') { + first = open < len ? first : i; + open = i; + open_piece = piece; + } else if (c == ']') { + if (open < len) { + go_bracket(d, text, len, first, open, i, lines[open_piece].line); + } + open = len; + } + } +} + +/* ── Paragraphs of the comment ───────────────────────────────────── */ + +/* go/doc/comment list(): each item is a run of paragraphs; an item starts at + * a marker of the list's own kind, a blank line ends a paragraph, and every + * line is trimmed. */ +static void go_list_paragraphs(go_doc_t *d, const go_span_t *sp, go_para_fn fn) { + size_t rest_len = 0; + bool numbered = false; + bool list_numbered = false; + (void)go_list_marker(&d->lines[sp->start], &rest_len, &list_numbered); + int count = 0; + for (int i = sp->start; i < sp->end; i++) { + go_line_t l = d->lines[i]; + const char *rest = go_list_marker(&l, &rest_len, &numbered); + bool new_item = rest && numbered == list_numbered; + if (new_item) { + l.s = rest; + l.len = (uint32_t)rest_len; + } + const char *s = l.s; + size_t n = l.len; + go_trim(&s, &n); + if ((new_item || n == 0) && count > 0) { + fn(d, d->para, count); + count = 0; + } + if (n > 0) { + d->para[count++] = (go_line_t){.s = s, .len = (uint32_t)n, .line = l.line}; + } + } + if (count > 0) { + fn(d, d->para, count); + } +} + +static void go_each_paragraph(go_doc_t *d, go_para_fn fn) { + for (int i = 0; i < d->nspans && !d->failed; i++) { + const go_span_t *sp = &d->spans[i]; + if (sp->kind == GO_SPAN_PARA) { + fn(d, &d->lines[sp->start], sp->end - sp->start); + } else if (sp->kind == GO_SPAN_LIST) { + go_list_paragraphs(d, sp, fn); + } + } +} + +void cbm_doclink_go_parse_doc(CBMExtractCtx *ctx, const CBMDefinition *def, const char *doc, + uint32_t doc_line) { + if (!doc || !strchr(doc, '[') || !strchr(doc, ']')) { + return; /* nearly every comment: no bracket, nothing to model */ + } + go_doc_t d = {.ctx = ctx, + .tmp = go_scratch(ctx), + .source_qn = def->qualified_name, + .params = def->param_names, + .def_line = def->start_line}; + go_group_lines(&d, doc, doc_line); + go_unindent(&d); + if (d.failed || d.nlines == 0) { + return; + } + d.para = (go_line_t *)cbm_arena_alloc(d.tmp, (size_t)d.nlines * sizeof(go_line_t)); + if (!d.para) { + return; + } + go_parse_spans(&d); + /* Definitions first: a link may be used before the paragraph defining it. */ + go_each_paragraph(&d, go_collect_linkdefs); + if (d.ndefs > SKIP_ONE) { + qsort(d.defs, (size_t)d.ndefs, sizeof(d.defs[0]), go_linkdef_cmp); + } + go_each_paragraph(&d, go_scan_paragraph); +} + +void cbm_doclink_gomod_parse_doc(CBMExtractCtx *ctx, const CBMDefinition *def, const char *doc, + uint32_t doc_line) { + (void)ctx; + (void)def; + (void)doc; + (void)doc_line; /* a go.mod comment is no Go doc comment */ +} + +/* ── Scope scan ──────────────────────────────────────────────────── */ + +/* A declaration that claims the flat node name .: its record gets + * the w flag once the file's claims of that name are all known. */ +typedef struct { + const char *name; + size_t name_len; + uint32_t start; /* byte offset: document order */ + uint32_t row; + int rec; /* index into recs */ +} go_claim_t; + +typedef struct { + CBMExtractCtx *ctx; + CBMArena *tmp; + char **recs; + int nrecs; + int recs_cap; + go_claim_t *claims; + int nclaims; + int claims_cap; + bool failed; +} go_scan_t; + +/* A string that grows in the scratch arena. */ +typedef struct { + CBMArena *a; + char *buf; + size_t len; + size_t cap; + bool failed; +} go_sb_t; + +static void go_sb_put(go_sb_t *sb, const char *s, size_t n) { + if (sb->failed) { + return; + } + if (sb->len + n + SKIP_ONE > sb->cap) { + size_t ncap = sb->cap ? sb->cap : GO_SB_INIT; + while (ncap < sb->len + n + SKIP_ONE) { + ncap *= PAIR_LEN; + } + char *grown = (char *)cbm_arena_alloc(sb->a, ncap); + if (!grown) { + sb->failed = true; + return; + } + if (sb->len > 0) { + memcpy(grown, sb->buf, sb->len); + } + sb->buf = grown; + sb->cap = ncap; + } + memcpy(sb->buf + sb->len, s, n); + sb->len += n; + sb->buf[sb->len] = '\0'; +} + +/* Append one name to a ','-joined list. */ +static void go_sb_name(go_sb_t *sb, const char *s, size_t n) { + if (sb->len > 0) { + go_sb_put(sb, ",", SKIP_ONE); + } + go_sb_put(sb, s, n); +} + +/* The children of a node, one after the other, in linear time (ts_node_child + * by index is linear per call). After the last child, or after go_kids_end, + * the cursor is released. */ +typedef struct { + TSTreeCursor cur; + bool live; +} go_kids_t; + +static void go_kids_end(go_kids_t *k) { + if (k->live) { + ts_tree_cursor_delete(&k->cur); + k->live = false; + } +} + +static bool go_kids_first(go_kids_t *k, TSNode parent, TSNode *out) { + *k = (go_kids_t){0}; /* both are defined on every path, for the caller's loop */ + *out = (TSNode){0}; + if (ts_node_is_null(parent)) { + return false; + } + k->cur = ts_tree_cursor_new(parent); + k->live = true; + if (!ts_tree_cursor_goto_first_child(&k->cur)) { + go_kids_end(k); + return false; + } + *out = ts_tree_cursor_current_node(&k->cur); + return true; +} + +static bool go_kids_next(go_kids_t *k, TSNode *out) { + if (!k->live || !ts_tree_cursor_goto_next_sibling(&k->cur)) { + go_kids_end(k); + return false; + } + *out = ts_tree_cursor_current_node(&k->cur); + return true; +} + +/* True when the child the iteration stands on is the parent's `field`. */ +static bool go_kid_is_field(go_kids_t *k, const char *field) { + const char *name = k->live ? ts_tree_cursor_current_field_name(&k->cur) : NULL; + return name && strcmp(name, field) == 0; +} + +static const char *go_node_span(const go_scan_t *s, TSNode n, size_t *len) { + *len = 0; + if (ts_node_is_null(n) || !s->ctx->source) { + return ""; + } + uint32_t sb = ts_node_start_byte(n); + uint32_t eb = ts_node_end_byte(n); + if (eb < sb || eb > (uint32_t)s->ctx->source_len) { + return ""; + } + *len = eb - sb; + return s->ctx->source + sb; +} + +static bool go_kind_is(TSNode n, const char *kind) { + return !ts_node_is_null(n) && strcmp(ts_node_type(n), kind) == 0; +} + +/* The first child of `parent` with this node kind (a null node when none). */ +static TSNode go_first_child_of_kind(TSNode parent, const char *kind) { + TSNode found = {0}; + go_kids_t k; + TSNode ch; + for (bool ok = go_kids_first(&k, parent, &ch); ok; ok = go_kids_next(&k, &ch)) { + if (strcmp(ts_node_type(ch), kind) == 0) { + found = ch; + go_kids_end(&k); + } + } + return found; +} + +/* The text of an exported identifier node; NULL for anything else. */ +static const char *go_exported(const go_scan_t *s, TSNode n, size_t *len) { + const char *t = go_node_span(s, n, len); + return go_is_name(t, *len) ? t : NULL; +} + +/* Append a record; returns its index, or CBM_NOT_FOUND. */ +static int go_push_rec(go_scan_t *s, char *rec) { + if (!rec) { + s->failed = true; + return CBM_NOT_FOUND; + } + char **grown = (char **)go_grow(s->tmp, s->recs, s->nrecs, &s->recs_cap, sizeof(char *)); + if (!grown) { + s->failed = true; + return CBM_NOT_FOUND; + } + s->recs = grown; + s->recs[s->nrecs] = rec; + return s->nrecs++; +} + +/* The names a type parameter list declares. */ +static void go_type_param_names(const go_scan_t *s, TSNode list, go_sb_t *out) { + go_kids_t decls; + TSNode decl; + for (bool ok = go_kids_first(&decls, list, &decl); ok; ok = go_kids_next(&decls, &decl)) { + if (!go_kind_is(decl, "type_parameter_declaration")) { + continue; + } + go_kids_t ids; + TSNode id; + for (bool more = go_kids_first(&ids, decl, &id); more; more = go_kids_next(&ids, &id)) { + size_t len = 0; + const char *t = go_node_span(s, id, &len); + if (go_kind_is(id, "identifier") && go_kid_is_field(&ids, "name") && + go_is_ident(t, len)) { + go_sb_name(out, t, len); + } + } + } +} + +/* The type arguments of a generic receiver `(l *List[T, U])` are the method's + * type parameters. */ +static void go_type_arg_names(const go_scan_t *s, TSNode args, go_sb_t *out) { + go_kids_t k; + TSNode arg; + for (bool ok = go_kids_first(&k, args, &arg); ok; ok = go_kids_next(&k, &arg)) { + TSNode id = go_kind_is(arg, "type_elem") && ts_node_named_child_count(arg) > 0 + ? ts_node_named_child(arg, 0) + : arg; + size_t len = 0; + const char *t = go_node_span(s, id, &len); + if (go_kind_is(id, "type_identifier") && go_is_ident(t, len)) { + go_sb_name(out, t, len); + } + } +} + +/* `g first last names`: the type parameters in scope of the declaration at + * `node`, from its first line to its last (a generic type's fields and method + * specifications are declared inside it). */ +static void go_push_generics(go_scan_t *s, TSNode node, const go_sb_t *names) { + if (names->failed) { + s->failed = true; + } else if (names->len > 0) { + (void)go_push_rec(s, cbm_arena_sprintf(s->tmp, "g\t%u\t%u\t%s", + ts_node_start_point(node).row + TS_LINE_OFFSET, + ts_node_end_point(node).row + TS_LINE_OFFSET, + names->buf)); + } +} + +/* The type parameters a declaration itself lists (`type_parameters`). */ +static void go_scan_type_params(go_scan_t *s, TSNode decl) { + TSNode tparams = ts_node_child_by_field_name(decl, TS_FIELD("type_parameters")); + if (ts_node_is_null(tparams)) { + return; + } + go_sb_t names = {.a = s->tmp}; + go_type_param_names(s, tparams, &names); + go_push_generics(s, decl, &names); +} + +/* Append the record of a declaration that claims the node .; its w + * field is added by go_finish_claims. `at` is the declaration's node. */ +static void go_push_claim(go_scan_t *s, char *rec, const char *name, size_t len, TSNode at) { + int idx = go_push_rec(s, rec); + go_claim_t *grown = + (go_claim_t *)go_grow(s->tmp, s->claims, s->nclaims, &s->claims_cap, sizeof(go_claim_t)); + if (idx < 0 || !grown) { + s->failed = true; + return; + } + s->claims = grown; + s->claims[s->nclaims++] = (go_claim_t){.name = name, + .name_len = len, + .start = ts_node_start_byte(at), + .row = ts_node_start_point(at).row, + .rec = idx}; +} + +static void go_scan_func(go_scan_t *s, TSNode fn) { + go_scan_type_params(s, fn); + size_t len = 0; + const char *name = go_exported(s, ts_node_child_by_field_name(fn, TS_FIELD("name")), &len); + if (name) { + go_push_claim(s, cbm_arena_sprintf(s->tmp, "F\t%.*s", (int)len, name), name, len, fn); + } +} + +/* The receiver's base type name (as extract_defs.c go_receiver_type_name: + * through pointer_type and generic_type); its type arguments go to + * `generics`. NULL when the receiver type is no plain name. */ +static const char *go_receiver_type(const go_scan_t *s, TSNode recv, size_t *len, + go_sb_t *generics) { + TSNode decl = go_first_child_of_kind(recv, "parameter_declaration"); + if (ts_node_is_null(decl)) { + return NULL; + } + TSNode tn = ts_node_child_by_field_name(decl, TS_FIELD("type")); + for (int guard = 0; guard < GO_RECV_UNWRAP && !ts_node_is_null(tn); guard++) { + if (go_kind_is(tn, "type_identifier")) { + const char *t = go_node_span(s, tn, len); + return go_is_ident(t, *len) ? t : NULL; + } + if (!go_kind_is(tn, "pointer_type") && !go_kind_is(tn, "generic_type")) { + return NULL; + } + TSNode targs = ts_node_child_by_field_name(tn, TS_FIELD("type_arguments")); + if (!ts_node_is_null(targs)) { + go_type_arg_names(s, targs, generics); + } + TSNode inner = ts_node_child_by_field_name(tn, TS_FIELD("type")); + tn = ts_node_is_null(inner) ? ts_node_named_child(tn, 0) : inner; + } + return NULL; +} + +static void go_scan_method(go_scan_t *s, TSNode m) { + go_sb_t generics = {.a = s->tmp}; + size_t recv_len = 0; + const char *recv = go_receiver_type(s, ts_node_child_by_field_name(m, TS_FIELD("receiver")), + &recv_len, &generics); + go_push_generics(s, m, &generics); + size_t len = 0; + const char *name = go_exported(s, ts_node_child_by_field_name(m, TS_FIELD("name")), &len); + if (!name) { + return; + } + if (!recv) { + recv = "?"; + recv_len = SKIP_ONE; + } + go_push_claim(s, + cbm_arena_sprintf(s->tmp, "M\t%.*s\t%.*s", (int)recv_len, recv, (int)len, name), + name, len, m); +} + +/* The implied name of an embedded field: the type's own name, without + * pointer, package qualifier and type arguments. */ +static const char *go_embedded_name(const go_scan_t *s, TSNode field, size_t *len) { + TSNode tn = ts_node_child_by_field_name(field, TS_FIELD("type")); + for (int guard = 0; guard < GO_EMBED_UNWRAP && !ts_node_is_null(tn); guard++) { + if (go_kind_is(tn, "type_identifier")) { + return go_exported(s, tn, len); + } + if (go_kind_is(tn, "qualified_type")) { + tn = ts_node_child_by_field_name(tn, TS_FIELD("name")); + } else if (go_kind_is(tn, "generic_type") || go_kind_is(tn, "pointer_type")) { + TSNode inner = ts_node_child_by_field_name(tn, TS_FIELD("type")); + tn = ts_node_is_null(inner) ? ts_node_named_child(tn, 0) : inner; + } else { + return NULL; + } + } + return NULL; +} + +static void go_push_field(go_scan_t *s, const char *type, size_t type_len, const char *name, + size_t len, bool node) { + (void)go_push_rec(s, cbm_arena_sprintf(s->tmp, "f\t%.*s\t%.*s\t%d", (int)type_len, type, + (int)len, name, node ? GO_NODE_FLAG : 0)); +} + +/* `f type name node` for every exported name of one field declaration. The + * extractor gives a node to the declaration's first name only. */ +static void go_scan_field(go_scan_t *s, const char *type, size_t type_len, TSNode field) { + bool first = true; + go_kids_t k; + TSNode ch; + for (bool ok = go_kids_first(&k, field, &ch); ok; ok = go_kids_next(&k, &ch)) { + if (!go_kid_is_field(&k, "name")) { + continue; + } + size_t len = 0; + const char *name = go_exported(s, ch, &len); + if (name) { + go_push_field(s, type, type_len, name, len, first); + } + first = false; + } + if (first) { /* no name: an embedded field */ + size_t len = 0; + const char *name = go_embedded_name(s, field, &len); + if (name) { + go_push_field(s, type, type_len, name, len, false); + } + } +} + +/* The fields of a struct type / the methods of an interface type. */ +static void go_scan_members(go_scan_t *s, const char *type, size_t type_len, TSNode body) { + TSNode list = go_kind_is(body, "struct_type") + ? go_first_child_of_kind(body, "field_declaration_list") + : body; + go_kids_t k; + TSNode ch; + for (bool ok = go_kids_first(&k, list, &ch); ok; ok = go_kids_next(&k, &ch)) { + if (go_kind_is(ch, "field_declaration")) { + go_scan_field(s, type, type_len, ch); + } else if (go_kind_is(ch, "method_elem")) { + size_t len = 0; + const char *name = + go_exported(s, ts_node_child_by_field_name(ch, TS_FIELD("name")), &len); + if (name) { + (void)go_push_rec(s, cbm_arena_sprintf(s->tmp, "m\t%.*s\t%.*s", (int)type_len, type, + (int)len, name)); + } + } + } +} + +/* The target of `type X = Y`: "Y" or "pkg.Y" (type arguments dropped); "-" + * for anything that is no named type. */ +static const char *go_alias_target(go_scan_t *s, TSNode type) { + TSNode tn = go_kind_is(type, "generic_type") + ? ts_node_child_by_field_name(type, TS_FIELD("type")) + : type; + size_t len = 0; + const char *target = "-"; + if (go_kind_is(tn, "type_identifier")) { + const char *t = go_exported(s, tn, &len); + target = t ? cbm_arena_strndup(s->tmp, t, len) : "-"; + } else if (go_kind_is(tn, "qualified_type")) { + size_t pkg_len = 0; + const char *pkg = + go_node_span(s, ts_node_child_by_field_name(tn, TS_FIELD("package")), &pkg_len); + const char *name = go_exported(s, ts_node_child_by_field_name(tn, TS_FIELD("name")), &len); + if (name && go_is_ident(pkg, pkg_len)) { + target = cbm_arena_sprintf(s->tmp, "%.*s.%.*s", (int)pkg_len, pkg, (int)len, name); + } + } + s->failed = s->failed || !target; + return target ? target : "-"; +} + +/* One type_spec or type_alias. */ +static void go_scan_type(go_scan_t *s, TSNode spec) { + go_scan_type_params(s, spec); + size_t len = 0; + const char *name = go_exported(s, ts_node_child_by_field_name(spec, TS_FIELD("name")), &len); + if (!name) { + return; + } + TSNode type = ts_node_child_by_field_name(spec, TS_FIELD("type")); + char kind = 'o'; + const char *alias = "-"; + if (go_kind_is(spec, "type_alias")) { + kind = 'a'; + alias = go_alias_target(s, type); + } else if (go_kind_is(type, "struct_type")) { + kind = 's'; + } else if (go_kind_is(type, "interface_type")) { + kind = 'i'; + } + go_push_claim(s, cbm_arena_sprintf(s->tmp, "T\t%.*s\t%c\t%s", (int)len, name, kind, alias), + name, len, spec); + if (kind == 's' || kind == 'i') { + go_scan_members(s, name, len, type); + } +} + +static void go_scan_type_decl(go_scan_t *s, TSNode decl) { + go_kids_t k; + TSNode ch; + for (bool ok = go_kids_first(&k, decl, &ch); ok; ok = go_kids_next(&k, &ch)) { + if (go_kind_is(ch, "type_spec") || go_kind_is(ch, "type_alias")) { + go_scan_type(s, ch); + } + } +} + +/* `V name w` for every exported name of a const_spec / var_spec. The extractor + * gives a node to the FIRST name of a spec that is a direct child of its + * declaration (every constant spec; a `var` without parentheses): that name + * claims ., the others have w `-`. */ +static void go_scan_value_spec(go_scan_t *s, TSNode spec, bool spec_has_node) { + bool first = true; + go_kids_t k; + TSNode ch; + for (bool ok = go_kids_first(&k, spec, &ch); ok; ok = go_kids_next(&k, &ch)) { + if (!go_kid_is_field(&k, "name")) { + continue; + } + size_t len = 0; + const char *name = go_exported(s, ch, &len); + if (name && first && spec_has_node) { + go_push_claim(s, cbm_arena_sprintf(s->tmp, "V\t%.*s", (int)len, name), name, len, spec); + } else if (name) { + (void)go_push_rec(s, cbm_arena_sprintf(s->tmp, "V\t%.*s\t-", (int)len, name)); + } + first = false; + } +} + +/* The specs of a parenthesized `var ( ... )`: none of them has a node. */ +static void go_scan_var_list(go_scan_t *s, TSNode list) { + go_kids_t k; + TSNode spec; + for (bool ok = go_kids_first(&k, list, &spec); ok; ok = go_kids_next(&k, &spec)) { + if (go_kind_is(spec, "var_spec")) { + go_scan_value_spec(s, spec, false); + } + } +} + +static void go_scan_value_decl(go_scan_t *s, TSNode decl) { + go_kids_t k; + TSNode ch; + for (bool ok = go_kids_first(&k, decl, &ch); ok; ok = go_kids_next(&k, &ch)) { + if (go_kind_is(ch, "const_spec") || go_kind_is(ch, "var_spec")) { + go_scan_value_spec(s, ch, true); + } else if (go_kind_is(ch, "var_spec_list")) { + go_scan_var_list(s, ch); + } + } +} + +/* The content of an import path literal; NULL when it is none this scan can + * record (not a plain string, empty, too long, or with bytes no path has). */ +static const char *go_import_path(const go_scan_t *s, TSNode lit, size_t *len) { + const char *t = go_node_span(s, lit, len); + if (*len < PAIR_LEN || (t[0] != '"' && t[0] != '`') || t[*len - SKIP_ONE] != t[0]) { + return NULL; + } + t++; + *len -= PAIR_LEN; + if (*len == 0 || *len > GO_PATH_MAX) { + return NULL; + } + for (size_t i = 0; i < *len; i++) { + unsigned char c = (unsigned char)t[i]; + if (c <= ' ' || c == '\\' || c == '"' || c == '`' || c == GO_ASCII_DEL) { + return NULL; + } + } + return t; +} + +static void go_scan_import_spec(go_scan_t *s, TSNode spec) { + size_t path_len = 0; + const char *path = + go_import_path(s, ts_node_child_by_field_name(spec, TS_FIELD("path")), &path_len); + TSNode name = ts_node_child_by_field_name(spec, TS_FIELD("name")); + size_t len = 0; + const char *local = go_node_span(s, name, &len); + if (ts_node_is_null(name)) { + local = "-"; + len = SKIP_ONE; + } else if (!go_kind_is(name, "package_identifier") || !go_is_ident(local, len) || + (len == SKIP_ONE && local[0] == '_')) { + return; /* `.` and `_` imports give no qualifier */ + } + if (!path && ts_node_is_null(name)) { + return; /* neither a name nor a path to derive one from */ + } + if (!path) { + path = "?"; /* the qualifier exists; where it leads is not known */ + path_len = SKIP_ONE; + } + (void)go_push_rec( + s, cbm_arena_sprintf(s->tmp, "I\t%.*s\t%.*s", (int)len, local, (int)path_len, path)); +} + +static void go_scan_import_list(go_scan_t *s, TSNode list) { + go_kids_t k; + TSNode spec; + for (bool ok = go_kids_first(&k, list, &spec); ok; ok = go_kids_next(&k, &spec)) { + if (go_kind_is(spec, "import_spec")) { + go_scan_import_spec(s, spec); + } + } +} + +static void go_scan_imports(go_scan_t *s, TSNode decl) { + go_kids_t k; + TSNode ch; + for (bool ok = go_kids_first(&k, decl, &ch); ok; ok = go_kids_next(&k, &ch)) { + if (go_kind_is(ch, "import_spec")) { + go_scan_import_spec(s, ch); + } else if (go_kind_is(ch, "import_spec_list")) { + go_scan_import_list(s, ch); + } + } +} + +/* A `//go:build` or `// +build` line: the file is built only under a + * constraint. *ignore is set when the constraint is the word `ignore`: such a + * file is in no package. */ +static bool go_comment_is_build_line(const go_scan_t *s, TSNode comment, bool *ignore) { + size_t n = 0; + const char *t = go_node_span(s, comment, &n); + static const char *const forms[] = {"//go:build", "// +build", "//+build"}; + static const char word[] = "ignore"; + for (size_t i = 0; i < sizeof(forms) / sizeof(forms[0]); i++) { + size_t fl = strlen(forms[i]); + if (!go_has_prefix(t, n, forms[i]) || (n > fl && !go_space(t[fl]))) { + continue; + } + const char *rest = t + fl; + size_t rn = n - fl; + go_trim(&rest, &rn); + *ignore = *ignore || (rn == sizeof(word) - SKIP_ONE && memcmp(rest, word, rn) == 0); + return true; + } + return false; +} + +/* A declaration that names nodes wherever the extractor's definition walk + * finds it. */ +static bool go_scan_decl(go_scan_t *s, TSNode n) { + if (go_kind_is(n, "function_declaration")) { + go_scan_func(s, n); + } else if (go_kind_is(n, "method_declaration")) { + go_scan_method(s, n); + } else if (go_kind_is(n, "type_declaration")) { + go_scan_type_decl(s, n); + } else { + return false; + } + return true; +} + +/* Push the children of `n` on the walk's stack. */ +static TSNode *go_push_children(go_scan_t *s, TSNode n, TSNode *stack, int *top, int *cap) { + go_kids_t k; + TSNode ch; + for (bool ok = go_kids_first(&k, n, &ch); ok; ok = go_kids_next(&k, &ch)) { + TSNode *grown = (TSNode *)go_grow(s->tmp, stack, *top, cap, sizeof(TSNode)); + if (!grown) { + s->failed = true; + go_kids_end(&k); + continue; + } + stack = grown; + stack[(*top)++] = ch; + } + return stack; +} + +/* A tree with parse errors: functions, methods and types are found wherever + * the extractor's walk finds them (it descends into everything but function + * nodes), so that this scan and the graph agree on what is declared. An + * explicit stack: the depth is the input's. */ +static void go_scan_error_tree(go_scan_t *s, TSNode root) { + int cap = 0; + int top = 0; + TSNode *stack = (TSNode *)go_grow(s->tmp, NULL, 0, &cap, sizeof(TSNode)); + if (!stack) { + s->failed = true; + return; + } + stack[top++] = root; + while (top > 0 && !s->failed) { + TSNode n = stack[--top]; + if (go_scan_decl(s, n) || go_kind_is(n, "method_elem") || go_kind_is(n, "func_literal")) { + continue; + } + stack = go_push_children(s, n, stack, &top, &cap); + } +} + +static int go_claim_cmp(const void *a, const void *b) { + const go_claim_t *x = (const go_claim_t *)a; + const go_claim_t *y = (const go_claim_t *)b; + size_t n = x->name_len < y->name_len ? x->name_len : y->name_len; + int c = memcmp(x->name, y->name, n); + if (c == 0 && x->name_len != y->name_len) { + c = x->name_len < y->name_len ? -SKIP_ONE : SKIP_ONE; + } + if (c != 0) { + return c; + } + return (x->start > y->start) - (x->start < y->start); +} + +static bool go_claim_same_name(const go_claim_t *a, const go_claim_t *b) { + return a->name_len == b->name_len && memcmp(a->name, b->name, a->name_len) == 0; +} + +/* The w flag of the claims [first, end) of one name, in text order: they share + * one node name, and the graph keeps the declaration that starts on the LAST + * line (cbm_gbuf_upsert_node: largest start_line). Two on that line: + * contested. */ +static void go_flag_claims(go_scan_t *s, int first, int end) { + uint32_t last_row = s->claims[end - SKIP_ONE].row; + bool contested = end - first > SKIP_ONE && s->claims[end - PAIR_LEN].row == last_row; + for (int k = first; k < end; k++) { + char w = '0'; + if (contested && s->claims[k].row == last_row) { + w = 'x'; + } else if (!contested && k == end - SKIP_ONE) { + w = '1'; + } + int rec = s->claims[k].rec; + s->recs[rec] = cbm_arena_sprintf(s->tmp, "%s\t%c", s->recs[rec], w); + s->failed = s->failed || !s->recs[rec]; + } +} + +static void go_finish_claims(go_scan_t *s) { + if (s->nclaims > SKIP_ONE) { + qsort(s->claims, (size_t)s->nclaims, sizeof(s->claims[0]), go_claim_cmp); + } + int i = 0; + while (i < s->nclaims && !s->failed) { + int end = i + SKIP_ONE; + while (end < s->nclaims && go_claim_same_name(&s->claims[i], &s->claims[end])) { + end++; + } + go_flag_claims(s, i, end); + i = end; + } +} + +static int go_rec_cmp(const void *a, const void *b) { + return strcmp(*(char *const *)a, *(char *const *)b); +} + +/* "go1" and the sorted, de-duplicated records, in the result arena. */ +static const char *go_blob(go_scan_t *s) { + if (s->failed) { + return NULL; + } + if (s->nrecs > SKIP_ONE) { + qsort(s->recs, (size_t)s->nrecs, sizeof(s->recs[0]), go_rec_cmp); + } + size_t total = sizeof(CBM_DOCLINK_GO_SCOPE_TAG); + for (int i = 0; i < s->nrecs; i++) { + total += strlen(s->recs[i]) + SKIP_ONE; + } + char *blob = (char *)cbm_arena_alloc(s->ctx->arena, total + SKIP_ONE); + if (!blob) { + return NULL; + } + size_t w = sizeof(CBM_DOCLINK_GO_SCOPE_TAG) - SKIP_ONE; + memcpy(blob, CBM_DOCLINK_GO_SCOPE_TAG, w); + for (int i = 0; i < s->nrecs; i++) { + if (i > 0 && strcmp(s->recs[i], s->recs[i - SKIP_ONE]) == 0) { + continue; + } + size_t len = strlen(s->recs[i]); + blob[w++] = '\n'; + memcpy(blob + w, s->recs[i], len); + w += len; + } + blob[w] = '\0'; + return blob; +} + +/* What the file's head says about its package. */ +typedef struct { + const char *pkg; + size_t pkg_len; + bool seen_package; + bool ignored; + bool constrained; + bool has_error; +} go_head_t; + +/* One child of the source file. Values are declared at this level only (the + * extractor reads variables and constants from the root's children); the + * other declarations are taken here when the tree has no parse error. */ +static void go_scan_top(go_scan_t *s, TSNode ch, go_head_t *h) { + if (go_kind_is(ch, "comment")) { + if (!h->seen_package && go_comment_is_build_line(s, ch, &h->ignored)) { + h->constrained = true; + } + } else if (go_kind_is(ch, "package_clause")) { + h->seen_package = true; + if (!h->pkg && ts_node_named_child_count(ch) > 0) { + const char *t = go_node_span(s, ts_node_named_child(ch, 0), &h->pkg_len); + h->pkg = go_is_ident(t, h->pkg_len) ? t : NULL; + } + } else if (go_kind_is(ch, "import_declaration")) { + go_scan_imports(s, ch); + } else if (go_kind_is(ch, "const_declaration") || go_kind_is(ch, "var_declaration")) { + go_scan_value_decl(s, ch); + } else if (!h->has_error) { + (void)go_scan_decl(s, ch); + } +} + +enum { GO_HEAD_FLAGS = 4 }; /* i, c, e and the terminator */ + +/* The flags field of the P record: `-` when none is set. */ +static void go_head_flags(const go_head_t *h, char out[GO_HEAD_FLAGS]) { + size_t n = 0; + if (h->ignored) { + out[n++] = 'i'; + } + if (h->constrained && !h->ignored) { + out[n++] = 'c'; + } + if (h->has_error) { + out[n++] = 'e'; + } + if (n == 0) { + out[n++] = '-'; + } + out[n] = '\0'; +} + +const char *cbm_doclink_go_scan_scope(CBMExtractCtx *ctx) { + go_scan_t s = {.ctx = ctx, .tmp = go_scratch(ctx)}; + go_head_t h = {.has_error = ts_node_has_error(ctx->root)}; + go_kids_t k; + TSNode ch; + for (bool ok = go_kids_first(&k, ctx->root, &ch); ok; ok = go_kids_next(&k, &ch)) { + go_scan_top(&s, ch, &h); + } + if (h.has_error) { + go_scan_error_tree(&s, ctx->root); + } + if (h.pkg) { + char flags[GO_HEAD_FLAGS]; + go_head_flags(&h, flags); + (void)go_push_rec(&s, + cbm_arena_sprintf(s.tmp, "P\t%.*s\t%s", (int)h.pkg_len, h.pkg, flags)); + } + go_finish_claims(&s); + return go_blob(&s); +} + +/* The module path of a go.mod file: `G path`. */ +const char *cbm_doclink_gomod_scan_scope(CBMExtractCtx *ctx) { + go_scan_t s = {.ctx = ctx, .tmp = go_scratch(ctx)}; + TSNode directive = go_first_child_of_kind(ctx->root, "module_directive"); + TSNode path = go_first_child_of_kind(directive, "module_path"); + size_t len = 0; + const char *t = go_node_span(&s, path, &len); + if (len >= PAIR_LEN && (t[0] == '"' || t[0] == '`') && t[len - SKIP_ONE] == t[0]) { + t++; + len -= PAIR_LEN; + } + if (go_valid_import_path(t, len)) { + (void)go_push_rec(&s, cbm_arena_sprintf(s.tmp, "G\t%.*s", (int)len, t)); + } + return go_blob(&s); +} + +/* The persisted scope: the blob without its `g` lines (the only records that + * carry a line number, and the only ones no other file reads). */ +char *cbm_doclink_go_portable_scope(const char *scope) { + size_t n = strlen(scope); + char *out = (char *)cbm_alloc(CBM_MEM_CLASS_OTHER, n + SKIP_ONE); + if (!out) { + return NULL; + } + size_t w = 0; + for (const char *p = scope; *p;) { + const char *nl = strchr(p, '\n'); + size_t len = nl ? (size_t)(nl - p) : strlen(p); + if (!(len >= PAIR_LEN && p[0] == 'g' && p[SKIP_ONE] == '\t')) { + if (w > 0) { + out[w++] = '\n'; + } + memcpy(out + w, p, len); + w += len; + } + p = nl ? nl + SKIP_ONE : p + len; + } + out[w] = '\0'; + return out; +} diff --git a/src/pipeline/doc_links.c b/src/pipeline/doc_links.c index c5023658d..17e84eecb 100644 --- a/src/pipeline/doc_links.c +++ b/src/pipeline/doc_links.c @@ -46,7 +46,7 @@ const char *cbm_doclink_reason_name(int reason) { * to this file. */ static const cbm_doclink_resolver_t *const DOCLINK_RESOLVERS[] = { &cbm_doclink_cs_resolver, &cbm_doclink_md_resolver, &cbm_doclink_pdf_resolver, - &cbm_doclink_rst_resolver, &cbm_doclink_adoc_resolver, + &cbm_doclink_rst_resolver, &cbm_doclink_adoc_resolver, &cbm_doclink_go_resolver, }; enum { DOCLINK_RESOLVER_COUNT = sizeof(DOCLINK_RESOLVERS) / sizeof(DOCLINK_RESOLVERS[0]) }; diff --git a/src/pipeline/doc_links.h b/src/pipeline/doc_links.h index 77a5dfb67..13eaea13c 100644 --- a/src/pipeline/doc_links.h +++ b/src/pipeline/doc_links.h @@ -295,6 +295,7 @@ void cbm_doclink_cs_test_unit_memo(bool on); * names but can move what those lines hold, so an incremental run * re-resolves such documents with the edited file. */ bool cbm_doclinks_binds_lines(CBMLanguage lang); +extern const cbm_doclink_resolver_t cbm_doclink_go_resolver; /* True when `rel_path` is a scope input of some language (scope_input). */ bool cbm_doclinks_is_scope_input(const char *rel_path); diff --git a/src/pipeline/doc_links_go.c b/src/pipeline/doc_links_go.c new file mode 100644 index 000000000..047a7654e --- /dev/null +++ b/src/pipeline/doc_links_go.c @@ -0,0 +1,1905 @@ +/* + * doc_links_go.c — Go doc-link resolution for doc_links.h. + * + * Extraction (internal/cbm/doclink_go.c) emits every bracketed text of a doc + * comment that go/doc/comment COULD take for a doc link. This file makes the + * rest of go/doc's decision, which needs the whole package, and then binds: + * + * [Name], [Recv.Name] the documented file's package. A name the + * package does not declare is plain text for + * go/doc: CBM_DOCLINK_LOCAL, neither edge nor row. + * So is the documented declaration's own type + * parameter. Recv.Name is a method declared with + * that receiver, a method of that interface, or + * a field of that struct (the field is a + * deliberate difference from go/doc: see + * go_bind_member). + * [pkg], [pkg.Name], pkg with a '/' is an import path. Otherwise, in + * [pkg.Recv.Name] order: the documented FILE's import of that + * name; the package's own name; a standard-library + * package with that one-element path. None of + * them: plain text (see go_lookup_qualifier for + * the one place this is stricter than go/doc). + * import path a module of the repository (every go.mod, the + * longest module path first) -> its directory's + * package; no package there: missing. No module: + * external. An import without a name is called by + * the real package name of an in-repo directory, + * else by go/doc's assumed name. + * member through an alias `type X = pkg.Y`: [p.X.Member] is looked up on Y. + * + * The members of a type are the methods declared with that receiver, the + * fields of its own struct declaration and the methods of its own interface + * declaration: nothing promoted through embedding. + * + * Binding. The graph keeps one node per qualified name, Go method nodes have + * a FLAT name (.), and is the directory, not the package + * clause: functions, methods of every receiver, types and values of one name, + * in every file of the directory, share one node, and the node is ONE of those + * declarations (cbm_gbuf_upsert_node keeps the smallest file path, then the + * last line). So a reference binds only when the node IS the entity it names: + * the node's file is a file of the named package, the declaration that keeps + * the name in that file (the w field of its scope record) is the named entity + * (its kind; for a method, its receiver), and the node has that kind's label. + * Every other declared entity is a graph_gap. A receiver-qualified method node + * (..) is tried first, so a graph that names methods that + * way binds them all. Only a node's qualified name, label and file_path are + * read: the proxy nodes of an incremental run carry no lines and no + * properties (the one exception is the SOURCE node, see go_source_owns_node). + * + * One name, several declarations. Files whose build constraints exclude each + * other declare variants of ONE entity, and any of them may hold the node. Two + * files built unconditionally that declare the same name are no package (a + * directory of stand-alone programs, test data): two entities share the name. + * Such a name binds only from a file that declares it itself, to that file's + * own declaration; from anywhere else it is ambiguous. A node kept by a test + * file is no target for product code. + * + * Reasons: external (standard library, module dependencies, vendored or + * not), missing (a repository package that does not declare the name; an + * import path inside a module without a package there), graph_gap (declared, + * but the node is another declaration's or absent: block variables, later + * names of a declaration, embedded fields, methods that lost the flat name; + * also an undeclared name in a package with a parse error), test_only_target + * (product code naming a declaration that only test files have), ambiguous + * (several modules or packages fit; a name declared as two kinds; a name two + * unconstrained files declare), + * unparseable (a reference longer than CBM_DOCLINK_GO_REF_MAX, an import path + * the scan could not record, an alias chain longer than GO_ALIAS_DEPTH). + */ +#include "pipeline/doc_links.h" + +#include "cbm.h" +#include "doclink.h" +#include "helpers.h" /* cbm_fqn_module_source_lang, cbm_is_test_file */ +#include "foundation/arena.h" +#include "foundation/constants.h" +#include "foundation/hash_table.h" +#include "foundation/log.h" +#include "foundation/mem_core.h" +#include "graph_buffer/graph_buffer.h" + +#include +#include +#include +#include +#include +#include + +enum { + GO_KEY_BUF = 2304, /* id + type + name, or id + an import path */ + GO_STACK_BUF = 1024, /* paths and qualified names shorter than this need no heap */ + GO_MAX_FIELDS = 6, /* one more than the fields of the longest scope record */ + GO_ALIAS_DEPTH = 8, /* alias-of-alias hops followed */ + GO_KEY_SEP = 0x1F, /* between the parts of a composite key */ + GO_NAME_MAX = CBM_DOCLINK_GO_REF_MAX, + GO_PATH_MAX = 1024, /* longest import or module path a scope records */ + GO_VERSION_MIN = 2, /* "v" + at least one digit */ + GO_GO_PREFIX = 3, /* strlen("go-") */ + GO_ASCII_END = 0x80, + GO_KIND_FUNC = 1, /* what a name is in a package (go_sym_t) or a file (go_flat_t) */ + GO_KIND_TYPE = 2, + GO_KIND_VALUE = 4, + GO_KIND_METHOD = 8, /* go_flat_t only: a method keeps the file's flat node name */ + GO_MEM_METHOD = 1, /* what Type.Name is in a package (go_mem_t) or a file (go_fmem_t) */ + GO_MEM_FIELD = 2, + GO_MEM_IFACE = 4, +}; + +/* ── Index data ──────────────────────────────────────────────────── */ + +typedef struct { + const char *local; /* the qualifier this import gives the file */ + const char *path; /* "?": the scan could not record it */ + bool conflict; /* the file has two imports called `local` */ +} go_import_t; + +typedef struct { + uint32_t line; /* first line of the generic declaration */ + uint32_t last; /* ... and its last */ + const char *names; /* ','-joined type parameters */ +} go_generic_t; + +struct go_pkg; + +typedef struct go_dir { + int idx; + const char *path; /* "" for the repository root */ + const char *qn; /* .: the prefix of its nodes */ + struct go_pkg **pkgs; + int npkgs; + int pkgs_cap; + bool has_module; /* a go.mod sits here */ + bool is_std; /* ... and it says `module std` */ + const struct go_dir *module_root; /* the nearest directory at or above with a go.mod */ +} go_dir_t; + +typedef struct go_pkg { + int idx; + const go_dir_t *dir; + const char *name; + bool importable; /* has a file that is neither *_test.go nor build-ignored */ + bool has_prod; /* has a file that is not test code */ + bool incomplete; /* a file of it has a parse error: declarations may hide */ +} go_pkg_t; + +typedef struct { + int idx; + const char *rel_path; /* borrowed: valid until destroy */ + const char *scope; /* borrowed: read while the index is built */ + go_dir_t *dir; + go_pkg_t *pkg; /* NULL: no package clause, or a go.mod file */ + bool is_test; /* test code by the indexer's rule: never a product target */ + bool go_test; /* *_test.go: invisible to importers */ + bool ignored; /* //go:build ignore: in no package */ + bool constrained; /* built only under a constraint (a build line, or its name) */ + go_import_t *imports; + int nimports; + int imports_cap; + go_generic_t *generics; /* fresh scopes only */ + int ngenerics; + int generics_cap; +} go_file_t; + +typedef struct { + const char *path; + const go_dir_t *dir; +} go_module_t; + +/* A standard-library package of a `module std` tree in the repository. */ +typedef struct { + const go_dir_t *dir; + int count; /* trees that have it */ +} go_stdpkg_t; + +/* How many files that are built unconditionally declare one name. A package + * that compiles has at most one: two declarations of a name can only live in + * files whose build constraints exclude each other, and those are variants of + * ONE entity. Two unconstrained files that declare the same name are no + * package at all (a directory of stand-alone programs, test data): their + * declarations are different entities that share a name. */ +typedef struct { + const go_file_t *last; /* the unconstrained file counted last */ + int files; /* 0, 1, or GO_TWINS for two and more */ +} go_free_t; + +enum { GO_TWINS = 2 }; + +/* A package-level name. */ +typedef struct { + int kinds; /* GO_KIND_* of its declarations */ + bool prod; /* declared by product code */ + const char *alias; /* every declaration is `type X = alias`, else NULL */ + const go_file_t *alias_file; /* the file whose imports name the alias target's package */ + bool alias_mixed; + go_free_t free; +} go_sym_t; + +/* Type.Name in a package. */ +typedef struct { + int kinds; /* GO_MEM_* */ + bool prod; + go_free_t free; +} go_mem_t; + +/* What ONE FILE declares under the flat node name ., and which of + * its declarations keeps that name (the w field of its scope records): the + * file's last declaration of that name. */ +typedef struct { + int kinds; /* GO_KIND_* the file declares under that name */ + int owner; /* the GO_KIND_* that keeps the name; 0: none */ + const char *method_recv; /* owner GO_KIND_METHOD: that method's receiver */ + bool contested; /* two declarations of that name start on the last line */ +} go_flat_t; + +/* What ONE FILE declares under ... */ +typedef struct { + int kinds; /* GO_MEM_* it declares */ + int node_kinds; /* ... and which of them get a node (not every field does) */ +} go_fmem_t; + +typedef struct { + CBMArena arena; + bool oom; + const char *project; + go_file_t *files; + int nfiles; + go_file_t **run_to_file; + int run_count; + go_dir_t **dirs; + int ndirs; + int dirs_cap; + int npkgs; + go_module_t *modules; /* sorted by (path, directory) */ + int nmodules; + int modules_cap; + int nstd; + CBMHashTable *dir_by_path; /* path -> go_dir_t */ + CBMHashTable *pkg_by_key; /* dir SEP name -> go_pkg_t */ + CBMHashTable *file_by_path; /* rel_path -> go_file_t */ + CBMHashTable *module_first; /* module path -> the first go_module_t of that path */ + CBMHashTable *module_at; /* dir SEP module path -> go_module_t */ + CBMHashTable *std_pkgs; /* path below a `module std` root -> go_stdpkg_t */ + CBMHashTable *syms; /* pkg SEP name -> go_sym_t */ + CBMHashTable *mems; /* pkg SEP type SEP name -> go_mem_t */ + CBMHashTable *flat; /* file SEP name -> go_flat_t */ + CBMHashTable *fmem; /* file SEP type SEP name -> go_fmem_t */ +} go_index_t; + +/* ── Small helpers ───────────────────────────────────────────────── */ + +static void *ix_alloc(go_index_t *ix, size_t n) { + void *p = cbm_arena_alloc(&ix->arena, n ? n : SKIP_ONE); + ix->oom = ix->oom || !p; + return p; +} + +static char *ix_strndup(go_index_t *ix, const char *s, size_t n) { + char *p = cbm_arena_strndup(&ix->arena, s, n); + ix->oom = ix->oom || !p; + return p; +} + +/* Room for one more element of an index array; NULL when memory ran out. */ +static void *ix_grow(go_index_t *ix, void *items, int count, int *cap, size_t elem) { + if (count < *cap) { + return items; + } + int ncap = *cap ? *cap * PAIR_LEN : CBM_SZ_8; + void *grown = ix_alloc(ix, (size_t)ncap * elem); + if (!grown) { + return NULL; + } + if (count > 0) { + memcpy(grown, items, (size_t)count * elem); + } + *cap = ncap; + return grown; +} + +/* A slice of text. */ +typedef struct { + const char *s; + size_t n; +} go_str_t; + +static const go_str_t GO_NO_STR = {NULL, 0}; + +static go_str_t go_str(const char *s) { + return (go_str_t){s, s ? strlen(s) : 0}; +} + +static bool go_str_eq(go_str_t a, const char *b) { + return strlen(b) == a.n && memcmp(a.s, b, a.n) == 0; +} + +/* A text buffer that lives on the stack unless the text is long. */ +typedef struct { + char stack[GO_STACK_BUF]; + char *heap; +} go_buf_t; + +/* At least `need` bytes; NULL when memory ran out. Released with go_buf_free. */ +static char *go_buf_get(go_buf_t *b, size_t need) { + b->heap = NULL; + if (need <= sizeof(b->stack)) { + return b->stack; + } + b->heap = (char *)cbm_alloc(CBM_MEM_CLASS_OTHER, need); + return b->heap; +} + +static void go_buf_free(go_buf_t *b) { + cbm_free(CBM_MEM_CLASS_OTHER, b->heap); + b->heap = NULL; +} + +/* " SEP a [SEP b]" into buf; false when it does not fit (the parts are + * names and paths a scope records, which are bounded). */ +static bool go_key(char *buf, size_t cap, int id, go_str_t a, go_str_t b) { + int n = snprintf(buf, cap, "%d%c", id, GO_KEY_SEP); + if (n < 0 || (size_t)n + a.n + b.n + PAIR_LEN >= cap) { + return false; + } + size_t w = (size_t)n; + memcpy(buf + w, a.s, a.n); + w += a.n; + if (b.s) { + buf[w++] = GO_KEY_SEP; + memcpy(buf + w, b.s, b.n); + w += b.n; + } + buf[w] = '\0'; + return true; +} + +/* The value stored under `key`, or a new zeroed one of `size` bytes. */ +static void *ix_entry(go_index_t *ix, CBMHashTable *ht, const char *key, size_t size) { + void *v = cbm_ht_get(ht, key); + if (v) { + return v; + } + v = ix_alloc(ix, size); + char *k = ix_strndup(ix, key, strlen(key)); + if (!v || !k) { + return NULL; + } + memset(v, 0, size); + cbm_ht_set(ht, k, v); + return v; +} + +/* The entry of a composite key, created when absent; NULL when the key does + * not fit or memory ran out. */ +static void *ix_keyed(go_index_t *ix, CBMHashTable *ht, int id, go_str_t a, go_str_t b, + size_t size) { + char key[GO_KEY_BUF]; + return go_key(key, sizeof(key), id, a, b) ? ix_entry(ix, ht, key, size) : NULL; +} + +static const void *go_lookup(const CBMHashTable *ht, int id, go_str_t a, go_str_t b) { + char key[GO_KEY_BUF]; + return go_key(key, sizeof(key), id, a, b) ? cbm_ht_get(ht, key) : NULL; +} + +/* The value stored under the text s[0..n). */ +static void *go_lookup_text(const CBMHashTable *ht, const char *s, size_t n) { + go_buf_t b; + char *key = go_buf_get(&b, n + SKIP_ONE); + if (!key) { + return NULL; + } + memcpy(key, s, n); + key[n] = '\0'; + void *hit = cbm_ht_get(ht, key); + go_buf_free(&b); + return hit; +} + +static bool go_has_suffix(const char *s, const char *suffix) { + size_t n = strlen(s); + size_t sl = strlen(suffix); + return n >= sl && memcmp(s + n - sl, suffix, sl) == 0; +} + +static bool go_has_non_ascii(go_str_t s) { + for (size_t i = 0; i < s.n; i++) { + if ((unsigned char)s.s[i] >= GO_ASCII_END) { + return true; + } + } + return false; +} + +/* ── Directories and packages ────────────────────────────────────── */ + +static go_dir_t *go_dir_find(const go_index_t *ix, const char *path, size_t len) { + return (go_dir_t *)go_lookup_text(ix->dir_by_path, path, len); +} + +static go_dir_t *go_dir_get(go_index_t *ix, const char *path, size_t len, const char *rel_path) { + go_dir_t *d = go_dir_find(ix, path, len); + if (d) { + return d; + } + go_dir_t **grown = + (go_dir_t **)ix_grow(ix, ix->dirs, ix->ndirs, &ix->dirs_cap, sizeof(go_dir_t *)); + d = (go_dir_t *)ix_alloc(ix, sizeof(*d)); + char *own = ix_strndup(ix, path, len); + if (!grown || !d || !own) { + return NULL; + } + memset(d, 0, sizeof(*d)); + d->idx = ix->ndirs; + d->path = own; + /* The same helper that names the directory's definitions. */ + d->qn = cbm_fqn_module_source_lang(&ix->arena, ix->project, rel_path, CBM_LANG_GO); + ix->oom = ix->oom || !d->qn; + ix->dirs = grown; + ix->dirs[ix->ndirs++] = d; + cbm_ht_set(ix->dir_by_path, own, d); + return d; +} + +static go_pkg_t *go_pkg_get(go_index_t *ix, go_dir_t *dir, go_str_t name) { + char key[GO_KEY_BUF]; + if (!go_key(key, sizeof(key), dir->idx, name, GO_NO_STR)) { + return NULL; + } + go_pkg_t *p = (go_pkg_t *)cbm_ht_get(ix->pkg_by_key, key); + if (p) { + return p; + } + go_pkg_t **dir_pkgs = + (go_pkg_t **)ix_grow(ix, dir->pkgs, dir->npkgs, &dir->pkgs_cap, sizeof(go_pkg_t *)); + p = (go_pkg_t *)ix_alloc(ix, sizeof(*p)); + char *own_key = ix_strndup(ix, key, strlen(key)); + char *own_name = ix_strndup(ix, name.s, name.n); + if (!dir_pkgs || !p || !own_key || !own_name) { + return NULL; + } + memset(p, 0, sizeof(*p)); + p->idx = ix->npkgs++; + p->dir = dir; + p->name = own_name; + dir->pkgs = dir_pkgs; + dir->pkgs[dir->npkgs++] = p; + cbm_ht_set(ix->pkg_by_key, own_key, p); + return p; +} + +/* ── Scope records ───────────────────────────────────────────────── */ + +/* Split a scope line (a copy the caller owns) at its tabs. */ +static int go_fields(char *line, char **out, int max) { + int n = 0; + out[n++] = line; + for (char *p = line; *p && n < max; p++) { + if (*p == '\t') { + *p = '\0'; + out[n++] = p + SKIP_ONE; + } + } + return n; +} + +/* A name field as a scan writes it: not empty, not longer than a name. */ +static bool go_name_ok(const char *s) { + size_t n = strlen(s); + return n > 0 && n <= GO_NAME_MAX; +} + +static bool go_path_ok(const char *s) { + size_t n = strlen(s); + return n > 0 && n <= GO_PATH_MAX; +} + +typedef void (*go_record_fn)(go_index_t *ix, go_file_t *file, char **f, int nf); + +/* Calls `fn` for every record line of a scope blob (the tag line is skipped); + * each line is handed over split into its fields. False when memory ran out. */ +static bool go_each_record(go_index_t *ix, go_file_t *file, const char *blob, go_record_fn fn) { + const char *p = strchr(blob, '\n'); + while (p && p[SKIP_ONE]) { + p++; + const char *nl = strchr(p, '\n'); + size_t len = nl ? (size_t)(nl - p) : strlen(p); + go_buf_t b; + char *line = go_buf_get(&b, len + SKIP_ONE); + if (!line) { + ix->oom = true; + return false; + } + memcpy(line, p, len); + line[len] = '\0'; + char *f[GO_MAX_FIELDS]; + int nf = go_fields(line, f, GO_MAX_FIELDS); + if (f[0][0] && !f[0][SKIP_ONE]) { + fn(ix, file, f, nf); + } + go_buf_free(&b); + p = nl; + } + return !ix->oom; +} + +static void go_add_module(go_index_t *ix, go_file_t *file, const char *path) { + file->dir->has_module = true; + if (strcmp(path, "std") == 0) { + /* the standard library's own module: its packages are imported by + * their bare directory path */ + file->dir->is_std = true; + ix->nstd++; + return; + } + go_module_t *grown = (go_module_t *)ix_grow(ix, ix->modules, ix->nmodules, &ix->modules_cap, + sizeof(go_module_t)); + char *own = ix_strndup(ix, path, strlen(path)); + if (!grown || !own) { + return; + } + ix->modules = grown; + ix->modules[ix->nmodules++] = (go_module_t){.path = own, .dir = file->dir}; +} + +static void go_add_import(go_index_t *ix, go_file_t *f, const char *local, const char *path) { + go_import_t *grown = + (go_import_t *)ix_grow(ix, f->imports, f->nimports, &f->imports_cap, sizeof(go_import_t)); + char *own_path = ix_strndup(ix, path, strlen(path)); + /* "-": not renamed; the name is derived once every package is known */ + char *own_local = strcmp(local, "-") == 0 ? NULL : ix_strndup(ix, local, strlen(local)); + if (!grown || !own_path || ix->oom) { + return; + } + f->imports = grown; + f->imports[f->nimports++] = (go_import_t){.local = own_local, .path = own_path}; +} + +/* A line number field of a scope record; false when it is none. */ +static bool go_line_field(const char *s, uint32_t *out) { + char *end = NULL; + unsigned long long value = strtoull(s, &end, CBM_DECIMAL_BASE); + if (end == s || *end != '\0' || value > UINT32_MAX) { + return false; + } + *out = (uint32_t)value; + return true; +} + +/* `g first last names`. */ +static void go_add_generic(go_index_t *ix, go_file_t *f, const char *first, const char *last, + const char *names) { + go_generic_t g = {.names = ix_strndup(ix, names, strlen(names))}; + go_generic_t *grown = (go_generic_t *)ix_grow(ix, f->generics, f->ngenerics, &f->generics_cap, + sizeof(go_generic_t)); + if (!grown || !g.names || !go_line_field(first, &g.line) || !go_line_field(last, &g.last)) { + return; + } + f->generics = grown; + f->generics[f->ngenerics++] = g; +} + +static bool go_word_in(go_str_t word, const char *const *list, size_t count) { + for (size_t i = 0; i < count; i++) { + if (go_str_eq(word, list[i])) { + return true; + } + } + return false; +} + +/* go/build goodOSArchFile: a file named *_GOOS.go, *_GOARCH.go or + * *_GOOS_GOARCH.go (before an optional _test) is built only for that target. + * The two lists are the toolchain's own closed lists (go/build syslist); a + * name they do not know leaves the file unconstrained, which can only make + * this resolver more careful (see go_free_t). */ +static bool go_name_constrained(const char *rel_path) { + static const char *const known_os[] = { + "aix", "android", "darwin", "dragonfly", "freebsd", "hurd", + "illumos", "ios", "js", "linux", "nacl", "netbsd", + "openbsd", "plan9", "solaris", "wasip1", "windows", "zos", + }; + static const char *const known_arch[] = { + "386", "amd64", "amd64p32", "arm", "armbe", "arm64", + "arm64be", "loong64", "mips", "mipsle", "mips64", "mips64le", + "mips64p32", "mips64p32le", "ppc", "ppc64", "ppc64le", "riscv", + "riscv64", "s390", "s390x", "sparc", "sparc64", "wasm", + }; + static const char ext[] = ".go"; + static const char test[] = "_test"; + const char *slash = strrchr(rel_path, '/'); + go_str_t base = go_str(slash ? slash + SKIP_ONE : rel_path); + if (base.n >= sizeof(ext) - SKIP_ONE && go_has_suffix(base.s, ext)) { + base.n -= sizeof(ext) - SKIP_ONE; + } + size_t tl = sizeof(test) - SKIP_ONE; + if (base.n >= tl && memcmp(base.s + base.n - tl, test, tl) == 0) { + base.n -= tl; + } + size_t last = base.n; + while (last > 0 && base.s[last - SKIP_ONE] != '_') { + last--; + } + if (last == 0) { + return false; /* no underscore: `linux.go` is an ordinary file */ + } + go_str_t tag = {base.s + last, base.n - last}; + return go_word_in(tag, known_os, sizeof(known_os) / sizeof(known_os[0])) || + go_word_in(tag, known_arch, sizeof(known_arch) / sizeof(known_arch[0])); +} + +static void go_set_package(go_index_t *ix, go_file_t *f, const char *name, const char *flags) { + if (f->pkg) { + return; + } + f->ignored = strchr(flags, 'i') != NULL; + f->constrained = strchr(flags, 'c') != NULL || go_name_constrained(f->rel_path); + f->pkg = go_pkg_get(ix, f->dir, go_str(name)); + if (!f->pkg) { + ix->oom = true; + return; + } + go_pkg_t *p = f->pkg; + p->importable = p->importable || (!f->go_test && !f->ignored); + p->has_prod = p->has_prod || (!f->is_test && !f->ignored); + p->incomplete = p->incomplete || strchr(flags, 'e') != NULL; +} + +/* First pass over a file's scope: package, module, imports, generics. */ +static void go_record_head(go_index_t *ix, go_file_t *file, char **f, int nf) { + enum { P_FIELDS = 3, I_FIELDS = 3, G_FIELDS = 2, GEN_FIELDS = 4, GEN_NAMES = 3 }; + char kind = f[0][0]; + if (kind == 'P' && nf == P_FIELDS && go_name_ok(f[SKIP_ONE])) { + go_set_package(ix, file, f[SKIP_ONE], f[PAIR_LEN]); + } else if (kind == 'G' && nf == G_FIELDS && go_path_ok(f[SKIP_ONE])) { + go_add_module(ix, file, f[SKIP_ONE]); + } else if (kind == 'I' && nf == I_FIELDS && go_name_ok(f[SKIP_ONE]) && + go_path_ok(f[PAIR_LEN])) { + go_add_import(ix, file, f[SKIP_ONE], f[PAIR_LEN]); + } else if (kind == 'g' && nf == GEN_FIELDS) { + go_add_generic(ix, file, f[SKIP_ONE], f[PAIR_LEN], f[GEN_NAMES]); + } +} + +/* ── Modules ─────────────────────────────────────────────────────── */ + +static int go_module_cmp(const void *a, const void *b) { + const go_module_t *x = (const go_module_t *)a; + const go_module_t *y = (const go_module_t *)b; + int c = strcmp(x->path, y->path); + return c ? c : strcmp(x->dir->path, y->dir->path); +} + +/* Sort the modules; index the first entry of every module path, and every + * (directory, module path) pair. */ +static void go_finish_modules(go_index_t *ix) { + if (ix->nmodules > SKIP_ONE) { + qsort(ix->modules, (size_t)ix->nmodules, sizeof(ix->modules[0]), go_module_cmp); + } + for (int i = 0; i < ix->nmodules; i++) { + go_module_t *m = &ix->modules[i]; + if (i == 0 || strcmp(m->path, ix->modules[i - SKIP_ONE].path) != 0) { + cbm_ht_set(ix->module_first, m->path, m); + } + char key[GO_KEY_BUF]; + char *own = go_key(key, sizeof(key), m->dir->idx, go_str(m->path), GO_NO_STR) + ? ix_strndup(ix, key, strlen(key)) + : NULL; + if (own) { + cbm_ht_set(ix->module_at, own, m); + } + } +} + +/* The nearest directory at or above `d` that holds a go.mod. */ +static const go_dir_t *go_module_root(const go_index_t *ix, const go_dir_t *d) { + size_t len = strlen(d->path); + for (;;) { + const go_dir_t *up = go_dir_find(ix, d->path, len); + if (up && up->has_module) { + return up; + } + if (len == 0) { + return NULL; + } + while (len > 0 && d->path[len - SKIP_ONE] != '/') { + len--; + } + if (len > 0) { + len--; /* the slash */ + } + } +} + +/* Give every directory its module root, and list the packages of the + * repository's `module std` trees under their bare import paths. */ +static void go_finish_dirs(go_index_t *ix) { + for (int i = 0; i < ix->ndirs; i++) { + go_dir_t *d = ix->dirs[i]; + d->module_root = go_module_root(ix, d); + const go_dir_t *root = d->module_root; + if (!root || !root->is_std || root == d || d->npkgs == 0) { + continue; + } + size_t rl = strlen(root->path); + const char *rel = d->path + rl + (rl > 0 ? SKIP_ONE : 0); + go_stdpkg_t *sp = (go_stdpkg_t *)ix_entry(ix, ix->std_pkgs, rel, sizeof(go_stdpkg_t)); + if (sp) { + sp->dir = sp->count == 0 ? d : sp->dir; + sp->count++; + } + } +} + +/* ── Import paths ────────────────────────────────────────────────── */ + +typedef enum { + GO_PATH_EXTERNAL = 0, /* no module of the repository holds it */ + GO_PATH_PACKAGE, /* a directory of the repository with Go files */ + GO_PATH_NO_PACKAGE, /* inside a module, but no Go package there */ + GO_PATH_AMBIGUOUS, /* several modules of that path, none of them the file's own */ + GO_PATH_UNKNOWN, /* not decidable: an import the scan could not record, no memory */ +} go_path_kind_t; + +/* `/` when it is a directory of the repository with Go files that + * belongs to the module rooted at `root` (no nested go.mod on the way). */ +static go_path_kind_t go_module_pkg_dir(const go_index_t *ix, const go_dir_t *root, go_str_t rest, + const go_dir_t **dir_out) { + size_t rl = strlen(root->path); + go_buf_t b; + char *buf = go_buf_get(&b, rl + rest.n + PAIR_LEN); + if (!buf) { + return GO_PATH_UNKNOWN; + } + size_t w = rl; + memcpy(buf, root->path, rl); + if (rest.n > 0) { + if (w > 0) { + buf[w++] = '/'; + } + memcpy(buf + w, rest.s, rest.n); + w += rest.n; + } + const go_dir_t *dir = go_dir_find(ix, buf, w); + go_buf_free(&b); + if (!dir || dir->npkgs == 0 || dir->module_root != root) { + return GO_PATH_NO_PACKAGE; + } + *dir_out = dir; + return GO_PATH_PACKAGE; +} + +/* The module a module path means for file `src`: the only module of that + * path, else the one the file itself lives in. NULL: nothing picks one. */ +static const go_module_t *go_pick_module(const go_index_t *ix, const go_module_t *first, + const go_file_t *src) { + const go_module_t *next = first + SKIP_ONE; + if (next >= ix->modules + ix->nmodules || strcmp(next->path, first->path) != 0) { + return first; + } + const go_dir_t *own = src ? src->dir->module_root : NULL; + return own ? (const go_module_t *)go_lookup(ix->module_at, own->idx, go_str(first->path), + GO_NO_STR) + : NULL; +} + +/* A path no module prefixes: a package of the repository's own standard + * library, when it has one. */ +static go_path_kind_t go_classify_std(const go_index_t *ix, const go_file_t *src, go_str_t path, + const go_dir_t **dir_out) { + if (ix->nstd == 0) { + return GO_PATH_EXTERNAL; + } + const go_stdpkg_t *sp = (const go_stdpkg_t *)go_lookup_text(ix->std_pkgs, path.s, path.n); + if (!sp) { + return GO_PATH_EXTERNAL; + } + if (sp->count == SKIP_ONE) { + *dir_out = sp->dir; + return GO_PATH_PACKAGE; + } + const go_dir_t *own = src ? src->dir->module_root : NULL; + if (own && own->is_std && go_module_pkg_dir(ix, own, path, dir_out) == GO_PATH_PACKAGE) { + return GO_PATH_PACKAGE; + } + return GO_PATH_AMBIGUOUS; +} + +/* Where an import path leads: the longest module path of the repository that + * prefixes it decides. */ +static go_path_kind_t go_classify_path(const go_index_t *ix, const go_file_t *src, go_str_t path, + const go_dir_t **dir_out) { + if (path.n == 0 || go_str_eq(path, "?")) { + return GO_PATH_UNKNOWN; + } + go_buf_t b; + char *buf = go_buf_get(&b, path.n + SKIP_ONE); + if (!buf) { + return GO_PATH_UNKNOWN; + } + memcpy(buf, path.s, path.n); + buf[path.n] = '\0'; + const go_module_t *first = NULL; + size_t cut = path.n; + for (; cut > 0 && !first; cut--) { + if (cut != path.n && buf[cut] != '/') { + continue; + } + char saved = buf[cut]; + buf[cut] = '\0'; + first = (const go_module_t *)cbm_ht_get(ix->module_first, buf); + buf[cut] = saved; + } + go_buf_free(&b); + if (!first) { + return go_classify_std(ix, src, path, dir_out); + } + const go_module_t *m = go_pick_module(ix, first, src); + if (!m) { + return GO_PATH_AMBIGUOUS; + } + size_t used = strlen(m->path); + go_str_t rest = {path.s + used, path.n - used}; + if (rest.n > 0) { + rest.s++; /* the slash after the module path */ + rest.n--; + } + return go_module_pkg_dir(ix, m->dir, rest, dir_out); +} + +/* The package an import path means in `dir`: its only importable package. With + * several, the one called `hint` (the qualifier, or the path's last element). + * NULL for none; *ambiguous when several fit and nothing picks one. */ +static const go_pkg_t *go_dir_package(const go_dir_t *dir, go_str_t hint, bool *ambiguous) { + const go_pkg_t *only = NULL; + const go_pkg_t *named = NULL; + int count = 0; + for (int i = 0; i < dir->npkgs; i++) { + const go_pkg_t *p = dir->pkgs[i]; + if (!p->importable) { + continue; + } + count++; + only = p; + if (hint.s && go_str_eq(hint, p->name)) { + named = p; + } + } + *ambiguous = count > SKIP_ONE && !named; + return count > SKIP_ONE ? named : only; +} + +static bool go_name_byte(unsigned char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' || + c >= GO_ASCII_END; +} + +/* The last element of a slash-separated path. */ +static go_str_t go_path_base(go_str_t path) { + size_t start = path.n; + while (start > 0 && path.s[start - SKIP_ONE] != '/') { + start--; + } + return (go_str_t){path.s + start, path.n - start}; +} + +/* go/doc assumedPackageName: the last path element; for a major-version + * element (`v2`) the one before it; without a `go-` prefix; cut at the first + * byte that cannot be in an identifier. */ +static go_str_t go_assumed_name(go_str_t path) { + go_str_t base = go_path_base(path); + bool version = base.n >= GO_VERSION_MIN && base.s[0] == 'v'; + for (size_t i = SKIP_ONE; version && i < base.n; i++) { + version = base.s[i] >= '0' && base.s[i] <= '9'; + } + if (version && base.n < path.n) { + base = go_path_base((go_str_t){path.s, path.n - base.n - SKIP_ONE}); + } + if (base.n >= GO_GO_PREFIX && memcmp(base.s, "go-", GO_GO_PREFIX) == 0) { + base.s += GO_GO_PREFIX; + base.n -= GO_GO_PREFIX; + } + size_t n = 0; + while (n < base.n && go_name_byte((unsigned char)base.s[n])) { + n++; + } + return (go_str_t){base.s, n}; +} + +static int go_import_cmp(const void *a, const void *b) { + const go_import_t *x = (const go_import_t *)a; + const go_import_t *y = (const go_import_t *)b; + int c = strcmp(x->local, y->local); + return c ? c : strcmp(x->path, y->path); +} + +static int go_generic_cmp(const void *a, const void *b) { + const go_generic_t *x = (const go_generic_t *)a; + const go_generic_t *y = (const go_generic_t *)b; + return (x->line > y->line) - (x->line < y->line); +} + +/* The name an import without one is called by: the real package name of an + * in-repo directory, else go/doc's assumed name. */ +static go_str_t go_unnamed_import_name(const go_index_t *ix, const go_file_t *f, + const go_import_t *im) { + go_str_t path = go_str(im->path); + go_str_t name = go_assumed_name(path); + const go_dir_t *dir = NULL; + if (go_classify_path(ix, f, path, &dir) == GO_PATH_PACKAGE) { + bool ambiguous = false; + const go_pkg_t *pkg = go_dir_package(dir, name, &ambiguous); + if (pkg) { + return go_str(pkg->name); + } + } + return name; +} + +/* Name the unnamed imports of a file, sort its imports by name and mark the + * names two different paths claim. Its type-parameter records are sorted by + * line on the way. */ +static void go_finish_file(go_index_t *ix, go_file_t *f) { + if (f->ngenerics > SKIP_ONE) { + qsort(f->generics, (size_t)f->ngenerics, sizeof(f->generics[0]), go_generic_cmp); + } + for (int i = 0; i < f->nimports; i++) { + go_import_t *im = &f->imports[i]; + if (!im->local) { + go_str_t name = go_unnamed_import_name(ix, f, im); + im->local = ix_strndup(ix, name.s, name.n); + } + if (!im->local) { + f->nimports = 0; + return; + } + } + if (f->nimports > SKIP_ONE) { + qsort(f->imports, (size_t)f->nimports, sizeof(f->imports[0]), go_import_cmp); + } + int i = 0; + while (i < f->nimports) { + int end = i + SKIP_ONE; /* [i, end): the imports called alike, sorted by path */ + while (end < f->nimports && strcmp(f->imports[end].local, f->imports[i].local) == 0) { + end++; + } + bool conflict = strcmp(f->imports[i].path, f->imports[end - SKIP_ONE].path) != 0; + for (int k = i; k < end; k++) { + f->imports[k].conflict = conflict; + } + i = end; + } +} + +/* The file's import called `name`; NULL when it has none. */ +static const go_import_t *go_file_import(const go_file_t *f, go_str_t name) { + int lo = 0; + int hi = f->nimports - SKIP_ONE; + while (lo <= hi) { + int mid = lo + ((hi - lo) / PAIR_LEN); + const char *local = f->imports[mid].local; + size_t ll = strlen(local); + int c = memcmp(name.s, local, name.n < ll ? name.n : ll); + if (c == 0 && name.n != ll) { + c = name.n < ll ? -SKIP_ONE : SKIP_ONE; + } + if (c == 0) { + return &f->imports[mid]; + } + if (c < 0) { + hi = mid - SKIP_ONE; + } else { + lo = mid + SKIP_ONE; + } + } + return NULL; +} + +/* ── Declarations ────────────────────────────────────────────────── */ + +/* Count file `f` as a declarer of one name when it is built unconditionally. A + * file's records are read in one run, so its declarations of one name arrive + * together. */ +static void go_count_free(go_free_t *fr, const go_file_t *f) { + if (f->constrained || fr->last == f) { + return; + } + fr->last = f; + if (fr->files < GO_TWINS) { + fr->files++; + } +} + +static void go_add_sym(go_index_t *ix, go_file_t *f, int kind_bit, const char *name, + const char *alias) { + if (!f->pkg || f->ignored) { + return; + } + bool fresh = !go_lookup(ix->syms, f->pkg->idx, go_str(name), GO_NO_STR); + go_sym_t *s = + (go_sym_t *)ix_keyed(ix, ix->syms, f->pkg->idx, go_str(name), GO_NO_STR, sizeof(go_sym_t)); + if (!s) { + return; + } + s->kinds |= kind_bit; + s->prod = s->prod || !f->is_test; + go_count_free(&s->free, f); + bool is_alias = alias && strcmp(alias, "-") != 0; + if (fresh && is_alias) { + s->alias = ix_strndup(ix, alias, strlen(alias)); + s->alias_file = f; + } else if (!is_alias || !s->alias || strcmp(s->alias, alias) != 0) { + s->alias_mixed = true; /* declarations disagree: nothing to follow */ + } +} + +static void go_add_member(go_index_t *ix, go_file_t *f, int kind_bit, const char *type, + const char *name, bool has_node) { + go_fmem_t *fm = + (go_fmem_t *)ix_keyed(ix, ix->fmem, f->idx, go_str(type), go_str(name), sizeof(go_fmem_t)); + if (fm) { + fm->kinds |= kind_bit; + fm->node_kinds |= has_node ? kind_bit : 0; + } + if (!f->pkg || f->ignored) { + return; + } + go_mem_t *m = (go_mem_t *)ix_keyed(ix, ix->mems, f->pkg->idx, go_str(type), go_str(name), + sizeof(go_mem_t)); + if (m) { + m->kinds |= kind_bit; + m->prod = m->prod || !f->is_test; + go_count_free(&m->free, f); + } +} + +/* Record that file `f` declares `name` as `kind`, and what the declaration's + * w field says: `1` it keeps the file's flat node name ., `x` two + * declarations contest it, `0` a later one keeps it, `-` it has no node. + * `recv` is set for a method. */ +static void go_add_claim(go_index_t *ix, const go_file_t *f, int kind, const char *name, + const char *recv, const char *w) { + go_flat_t *fl = + (go_flat_t *)ix_keyed(ix, ix->flat, f->idx, go_str(name), GO_NO_STR, sizeof(go_flat_t)); + if (!fl) { + return; + } + fl->kinds |= kind; + if (w[0] == 'x') { + fl->contested = true; + } else if (w[0] == '1') { + fl->owner = kind; + fl->method_recv = recv ? ix_strndup(ix, recv, strlen(recv)) : NULL; + } +} + +/* Second pass over a file's scope: its declarations. */ +static void go_record_decl(go_index_t *ix, go_file_t *file, char **f, int nf) { + enum { + F_FIELDS = 3, + M_FIELDS = 4, + T_FIELDS = 5, + V_FIELDS = 3, + FIELD_FIELDS = 4, + IM_FIELDS = 3 + }; + enum { F_OWNER = 2, T_ALIAS = 3, T_OWNER = 4, V_OWNER = 2, M_OWNER = 3, FIELD_NODE = 3 }; + char kind = f[0][0]; + if (nf < IM_FIELDS || !go_name_ok(f[SKIP_ONE])) { + return; + } + if (kind == 'F' && nf == F_FIELDS) { + go_add_sym(ix, file, GO_KIND_FUNC, f[SKIP_ONE], NULL); + go_add_claim(ix, file, GO_KIND_FUNC, f[SKIP_ONE], NULL, f[F_OWNER]); + } else if (kind == 'T' && nf == T_FIELDS) { + go_add_sym(ix, file, GO_KIND_TYPE, f[SKIP_ONE], f[T_ALIAS]); + go_add_claim(ix, file, GO_KIND_TYPE, f[SKIP_ONE], NULL, f[T_OWNER]); + } else if (kind == 'V' && nf == V_FIELDS) { + go_add_sym(ix, file, GO_KIND_VALUE, f[SKIP_ONE], NULL); + go_add_claim(ix, file, GO_KIND_VALUE, f[SKIP_ONE], NULL, f[V_OWNER]); + } else if (kind == 'M' && nf == M_FIELDS && go_name_ok(f[PAIR_LEN])) { + if (strcmp(f[SKIP_ONE], "?") != 0) { + go_add_member(ix, file, GO_MEM_METHOD, f[SKIP_ONE], f[PAIR_LEN], true); + } + go_add_claim(ix, file, GO_KIND_METHOD, f[PAIR_LEN], f[SKIP_ONE], f[M_OWNER]); + } else if (kind == 'f' && nf == FIELD_FIELDS && go_name_ok(f[PAIR_LEN])) { + go_add_member(ix, file, GO_MEM_FIELD, f[SKIP_ONE], f[PAIR_LEN], f[FIELD_NODE][0] == '1'); + } else if (kind == 'm' && nf == IM_FIELDS && go_name_ok(f[PAIR_LEN])) { + go_add_member(ix, file, GO_MEM_IFACE, f[SKIP_ONE], f[PAIR_LEN], true); + } +} + +/* ── Index lifecycle ─────────────────────────────────────────────── */ + +static void go_destroy(void *index) { + go_index_t *ix = (go_index_t *)index; + if (!ix) { + return; + } + cbm_ht_free(ix->dir_by_path); + cbm_ht_free(ix->pkg_by_key); + cbm_ht_free(ix->file_by_path); + cbm_ht_free(ix->module_first); + cbm_ht_free(ix->module_at); + cbm_ht_free(ix->std_pkgs); + cbm_ht_free(ix->syms); + cbm_ht_free(ix->mems); + cbm_ht_free(ix->flat); + cbm_ht_free(ix->fmem); + cbm_free(CBM_MEM_CLASS_OTHER, ix->run_to_file); + cbm_arena_destroy(&ix->arena); + cbm_free(CBM_MEM_CLASS_OTHER, ix); +} + +static bool go_index_tables(go_index_t *ix, int file_count, int run_file_count) { + ix->dir_by_path = cbm_ht_create(CBM_SZ_1K); + ix->pkg_by_key = cbm_ht_create(CBM_SZ_1K); + ix->file_by_path = cbm_ht_create(CBM_SZ_4K); + ix->module_first = cbm_ht_create(CBM_SZ_64); + ix->module_at = cbm_ht_create(CBM_SZ_64); + ix->std_pkgs = cbm_ht_create(CBM_SZ_64); + ix->syms = cbm_ht_create(CBM_SZ_4K); + ix->mems = cbm_ht_create(CBM_SZ_4K); + ix->flat = cbm_ht_create(CBM_SZ_4K); + ix->fmem = cbm_ht_create(CBM_SZ_4K); + ix->nfiles = file_count; + ix->files = + (go_file_t *)ix_alloc(ix, (size_t)(file_count ? file_count : SKIP_ONE) * sizeof(go_file_t)); + ix->run_count = run_file_count; + ix->run_to_file = (go_file_t **)cbm_calloc( + CBM_MEM_CLASS_OTHER, + (size_t)(run_file_count ? run_file_count : SKIP_ONE) * sizeof(go_file_t *)); + return ix->dir_by_path && ix->pkg_by_key && ix->file_by_path && ix->module_first && + ix->module_at && ix->std_pkgs && ix->syms && ix->mems && ix->flat && ix->fmem && + ix->files && ix->run_to_file; +} + +/* True for a file below a directory called `testdata`: the go tool ignores + * such directories, so what they hold is fixture data, never product code. */ +static bool go_in_testdata(const char *rel_path) { + static const char dir[] = "testdata"; + const size_t dl = sizeof(dir) - SKIP_ONE; + for (const char *p = rel_path;;) { + const char *slash = strchr(p, '/'); + if (!slash) { + return false; /* the last element is the file's name */ + } + if ((size_t)(slash - p) == dl && memcmp(p, dir, dl) == 0) { + return true; + } + p = slash + SKIP_ONE; + } +} + +/* File i of the index: its directory, its test flags, the head of its scope. */ +static bool go_index_file(go_index_t *ix, int i, const cbm_doclink_file_t *src) { + go_file_t *f = &ix->files[i]; + memset(f, 0, sizeof(*f)); + f->idx = i; + f->rel_path = src->rel_path; + f->scope = src->scope; + f->is_test = cbm_is_test_file(src->rel_path, CBM_LANG_GO) || go_in_testdata(src->rel_path); + f->go_test = go_has_suffix(src->rel_path, "_test.go"); + const char *slash = strrchr(src->rel_path, '/'); + f->dir = + go_dir_get(ix, src->rel_path, slash ? (size_t)(slash - src->rel_path) : 0, src->rel_path); + if (!f->dir) { + ix->oom = true; + return false; + } + cbm_ht_set(ix->file_by_path, f->rel_path, f); + if (src->run_file >= 0 && src->run_file < ix->run_count) { + ix->run_to_file[src->run_file] = f; + } + return !f->scope || go_each_record(ix, f, f->scope, go_record_head); +} + +static void *go_build(const cbm_doclink_build_in_t *in) { + go_index_t *ix = (go_index_t *)cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*ix)); + if (!ix) { + return NULL; + } + cbm_arena_init(&ix->arena); + ix->project = in->ctx->project_name; + bool ok = go_index_tables(ix, in->file_count, in->run_file_count); + for (int i = 0; ok && i < in->file_count; i++) { + ok = go_index_file(ix, i, &in->files[i]); + } + if (ok) { + go_finish_modules(ix); + go_finish_dirs(ix); + } + for (int i = 0; ok && i < ix->nfiles; i++) { + go_finish_file(ix, &ix->files[i]); + ok = !ix->oom; + } + for (int i = 0; ok && i < ix->nfiles; i++) { + go_file_t *f = &ix->files[i]; + ok = !f->scope || go_each_record(ix, f, f->scope, go_record_decl); + f->scope = NULL; /* read: the index keeps no pointer into a blob */ + } + if (!ok || ix->oom) { + cbm_log_error("doc_links.go.error", "step", "index", "reason", "alloc"); + go_destroy(ix); + return NULL; + } + char b[CBM_SZ_4][CBM_SZ_32]; + snprintf(b[0], sizeof(b[0]), "%d", ix->nfiles); + snprintf(b[SKIP_ONE], sizeof(b[SKIP_ONE]), "%d", ix->npkgs); + snprintf(b[PAIR_LEN], sizeof(b[PAIR_LEN]), "%d", ix->nmodules + ix->nstd); + snprintf(b[CBM_SZ_3], sizeof(b[CBM_SZ_3]), "%u", cbm_ht_count(ix->syms)); + cbm_log_info("doc_links.go.index", "files", b[0], "packages", b[SKIP_ONE], "modules", + b[PAIR_LEN], "symbols", b[CBM_SZ_3]); + return ix; +} + +/* ── Resolution ──────────────────────────────────────────────────── */ + +typedef struct { + const go_index_t *ix; + const cbm_gbuf_t *g; + const go_file_t *src; + cbm_doclink_outcome_t *out; +} go_ctx_t; + +static void go_row(const go_ctx_t *c, int reason) { + c->out->kind = CBM_DOCLINK_UNRESOLVED; + c->out->reason = reason; +} + +/* Plain text for go/doc: neither an edge nor a row. */ +static void go_not_a_link(const go_ctx_t *c) { + c->out->kind = CBM_DOCLINK_LOCAL; +} + +static void go_edge(const go_ctx_t *c, const cbm_gbuf_node_t *node, bool exact) { + if (!node) { + go_row(c, CBM_DOCLINK_REASON_GRAPH_GAP); + return; + } + c->out->kind = CBM_DOCLINK_EDGE; + c->out->target = node; + c->out->exact = exact; +} + +/* A name the target package does not declare. Written bare it is plain text. + * Qualified it is a broken link; but in a package with a parse error the + * declaration may be hidden from the scan (a graph gap, not doc rot), and + * text with a non-ASCII byte may be no identifier at all, which this scan + * cannot tell (go/doc asks the Unicode letter classes): plain text. */ +static void go_undeclared(const go_ctx_t *c, const go_pkg_t *pkg, bool qualified, go_str_t recv, + go_str_t name) { + if (!qualified || go_has_non_ascii(recv) || go_has_non_ascii(name)) { + go_not_a_link(c); + } else { + go_row(c, pkg->incomplete ? CBM_DOCLINK_REASON_GRAPH_GAP : CBM_DOCLINK_REASON_MISSING); + } +} + +/* The node .[.] of the package's directory, or NULL. */ +static const cbm_gbuf_node_t *go_find_node(const go_ctx_t *c, const go_pkg_t *pkg, go_str_t a, + go_str_t b) { + size_t ql = strlen(pkg->dir->qn); + go_buf_t buf; + char *qn = go_buf_get(&buf, ql + a.n + b.n + CBM_SZ_3); + if (!qn) { + return NULL; + } + size_t w = ql; + memcpy(qn, pkg->dir->qn, ql); + qn[w++] = '.'; + memcpy(qn + w, a.s, a.n); + w += a.n; + if (b.s) { + qn[w++] = '.'; + memcpy(qn + w, b.s, b.n); + w += b.n; + } + qn[w] = '\0'; + const cbm_gbuf_node_t *n = cbm_gbuf_find_by_qn(c->g, qn); + go_buf_free(&buf); + return n; +} + +/* The file a node belongs to, when that is a file of package `pkg` whose + * declarations the reference may bind: not build-ignored, and no test code + * unless the reference itself is written in test code (a node kept by a test + * file's twin of a product declaration is no target for product code). */ +static const go_file_t *go_node_file(const go_ctx_t *c, const cbm_gbuf_node_t *n, + const go_pkg_t *pkg) { + if (!n || !n->file_path || !n->label) { + return NULL; + } + const go_file_t *f = (const go_file_t *)cbm_ht_get(c->ix->file_by_path, n->file_path); + if (!f || f->pkg != pkg || f->ignored || (f->is_test && !c->src->is_test)) { + return NULL; + } + return f; +} + +static bool go_label_is(const cbm_gbuf_node_t *n, const char *label) { + return strcmp(n->label, label) == 0; +} + +/* The declaration of the node's own file that keeps the flat name .; + * NULL when the node is no file's of package `pkg`, or two declarations + * contest the name there. */ +static const go_flat_t *go_flat_owner(const go_ctx_t *c, const cbm_gbuf_node_t *n, + const go_pkg_t *pkg, go_str_t name) { + const go_file_t *f = go_node_file(c, n, pkg); + const go_flat_t *fl = + f ? (const go_flat_t *)go_lookup(c->ix->flat, f->idx, name, GO_NO_STR) : NULL; + return (fl && !fl->contested) ? fl : NULL; +} + +/* The node of . when it IS the package's function, type or value + * of that name: the node's file keeps the name for a declaration of that + * kind, and the node has its label. */ +static const cbm_gbuf_node_t *go_symbol_node(const go_ctx_t *c, const go_pkg_t *pkg, int kind, + go_str_t name) { + const cbm_gbuf_node_t *n = go_find_node(c, pkg, name, GO_NO_STR); + const go_flat_t *fl = go_flat_owner(c, n, pkg, name); + if (!fl || fl->owner != kind) { + return NULL; + } + bool label_ok = (kind == GO_KIND_FUNC && go_label_is(n, "Function")) || + (kind == GO_KIND_TYPE && cbm_label_is_type_like(n->label)) || + (kind == GO_KIND_VALUE && go_label_is(n, "Variable")); + return label_ok ? n : NULL; +} + +/* The node of .. when it is that member of the package: a + * field or an interface method, or a method in a graph that names methods by + * their receiver. */ +static const cbm_gbuf_node_t *go_member_node(const go_ctx_t *c, const go_pkg_t *pkg, int kind, + go_str_t type, go_str_t name) { + const cbm_gbuf_node_t *n = go_find_node(c, pkg, type, name); + const go_file_t *f = go_node_file(c, n, pkg); + if (!f || !go_label_is(n, kind == GO_MEM_FIELD ? "Field" : "Method")) { + return NULL; + } + const go_fmem_t *fm = (const go_fmem_t *)go_lookup(c->ix->fmem, f->idx, type, name); + return (fm && (fm->node_kinds & kind)) ? n : NULL; +} + +/* The node of the method `recv.name`: its receiver-qualified node, else the + * flat node . when the method is the one that kept it (the last + * declaration of that name in the node's file). */ +static const cbm_gbuf_node_t *go_method_node(const go_ctx_t *c, const go_pkg_t *pkg, go_str_t recv, + go_str_t name) { + const cbm_gbuf_node_t *qualified = go_member_node(c, pkg, GO_MEM_METHOD, recv, name); + if (qualified) { + return qualified; + } + const cbm_gbuf_node_t *n = go_find_node(c, pkg, name, GO_NO_STR); + const go_flat_t *fl = go_flat_owner(c, n, pkg, name); + bool owns = fl && fl->owner == GO_KIND_METHOD && fl->method_recv && + go_str_eq(recv, fl->method_recv) && go_label_is(n, "Method"); + return owns ? n : NULL; +} + +static bool go_one_bit(int bits) { + return bits != 0 && (bits & (bits - SKIP_ONE)) == 0; +} + +/* Bind `n`, the node of a name declared once (build variants count as once). + * A name that two unconstrained files of the package declare (go_free_t) is + * two entities: written in a file that declares it itself (`declared_here`), + * the reference means that file's own declaration, which binds only when it + * is the one that has the node; written anywhere else, nothing picks one. */ +static void go_bind_node(const go_ctx_t *c, const cbm_gbuf_node_t *n, bool twins, + bool declared_here, bool exact) { + if (twins && !declared_here) { + go_row(c, CBM_DOCLINK_REASON_AMBIGUOUS); + return; + } + bool other_file = n && twins && strcmp(n->file_path, c->src->rel_path) != 0; + go_edge(c, other_file ? NULL : n, exact); +} + +/* [Name] / [pkg.Name] in package `pkg`. */ +static void go_bind_symbol(const go_ctx_t *c, const go_pkg_t *pkg, go_str_t name, bool qualified) { + const go_sym_t *s = (const go_sym_t *)go_lookup(c->ix->syms, pkg->idx, name, GO_NO_STR); + if (!s) { + go_undeclared(c, pkg, qualified, GO_NO_STR, name); + } else if (!go_one_bit(s->kinds)) { + go_row(c, CBM_DOCLINK_REASON_AMBIGUOUS); /* a function here, a type there */ + } else if (!s->prod && !c->src->is_test) { + go_row(c, CBM_DOCLINK_REASON_TEST_ONLY); + } else { + const go_flat_t *own = + pkg == c->src->pkg + ? (const go_flat_t *)go_lookup(c->ix->flat, c->src->idx, name, GO_NO_STR) + : NULL; + go_bind_node(c, go_symbol_node(c, pkg, s->kinds, name), s->free.files >= GO_TWINS, + own && (own->kinds & s->kinds), qualified); + } +} + +/* The package a qualifier of file `file` names through its imports. */ +static go_path_kind_t go_import_package(const go_index_t *ix, const go_file_t *file, + go_str_t qualifier, const go_pkg_t **pkg_out) { + const go_import_t *im = go_file_import(file, qualifier); + if (!im || im->conflict) { + return GO_PATH_NO_PACKAGE; + } + const go_dir_t *dir = NULL; + go_path_kind_t kind = go_classify_path(ix, file, go_str(im->path), &dir); + if (kind != GO_PATH_PACKAGE) { + return kind; + } + bool ambiguous = false; + *pkg_out = go_dir_package(dir, qualifier, &ambiguous); + if (ambiguous) { + return GO_PATH_AMBIGUOUS; + } + return *pkg_out ? GO_PATH_PACKAGE : GO_PATH_NO_PACKAGE; +} + +enum { GO_ALIAS_NONE = 0, GO_ALIAS_FOUND, GO_ALIAS_EXTERNAL, GO_ALIAS_TOO_DEEP }; + +/* [X.Member] where X is `type X = Y` or `= pkg.Y`: follow the alias (and an + * alias of an alias) to the type that declares Member; *pkg and *recv then + * name that type. */ +static int go_follow_alias(const go_ctx_t *c, const go_pkg_t **pkg, go_str_t *recv, go_str_t name) { + for (int depth = 0; depth < GO_ALIAS_DEPTH; depth++) { + const go_sym_t *s = (const go_sym_t *)go_lookup(c->ix->syms, (*pkg)->idx, *recv, GO_NO_STR); + if (!s || !s->alias || s->alias_mixed || s->kinds != GO_KIND_TYPE) { + return GO_ALIAS_NONE; + } + const char *dot = strchr(s->alias, '.'); + if (dot) { + go_str_t q = {s->alias, (size_t)(dot - s->alias)}; + const go_pkg_t *next = NULL; + go_path_kind_t kind = go_import_package(c->ix, s->alias_file, q, &next); + if (kind == GO_PATH_EXTERNAL) { + return GO_ALIAS_EXTERNAL; + } + if (kind != GO_PATH_PACKAGE) { + return GO_ALIAS_NONE; + } + *pkg = next; + } + *recv = dot ? go_str(dot + SKIP_ONE) : go_str(s->alias); + if (go_lookup(c->ix->mems, (*pkg)->idx, *recv, name)) { + return GO_ALIAS_FOUND; + } + } + return GO_ALIAS_TOO_DEEP; /* never an answer taken from half the chain */ +} + +/* The member `recv.name` of package `*pkg`, looked up through an alias when + * the type does not declare it itself (*pkg and *recv then name the type that + * does). NULL with *row false: not declared. NULL with *row true: a row was + * set (the alias leads out of the repository, or too deep). */ +static const go_mem_t *go_find_member(const go_ctx_t *c, const go_pkg_t **pkg, go_str_t *recv, + go_str_t name, bool *row) { + *row = false; + const go_mem_t *m = (const go_mem_t *)go_lookup(c->ix->mems, (*pkg)->idx, *recv, name); + if (m) { + return m; + } + int followed = go_follow_alias(c, pkg, recv, name); + if (followed == GO_ALIAS_EXTERNAL || followed == GO_ALIAS_TOO_DEEP) { + go_row(c, followed == GO_ALIAS_EXTERNAL ? CBM_DOCLINK_REASON_EXTERNAL + : CBM_DOCLINK_REASON_UNPARSEABLE); + *row = true; + return NULL; + } + return followed == GO_ALIAS_FOUND + ? (const go_mem_t *)go_lookup(c->ix->mems, (*pkg)->idx, *recv, name) + : NULL; +} + +/* [Recv.Name] / [pkg.Recv.Name] in package `pkg`. + * + * What binds: a member the named type declares ITSELF. That is a method + * declared with that receiver, a method of that interface type, or a field + * written in that type's own struct declaration; nothing promoted through an + * embedded type. A name the type does not declare is plain text when it is + * written without a package and a broken link (`missing`) when it is written + * with one. `type X = Y` is followed to the type that declares the member + * only for a reference written with a package. + * + * What go/doc does, as far as it is known here (no Go toolchain was at hand + * to check it; gopls sets its doc-comment parser up the same way): without a + * package, `Recv.Name` is a link only for a METHOD, so `[Type.Field]` inside + * its own package renders as plain text; with a package, go/doc links + * whatever follows it without looking. + * + * Binding the same-package field is a deliberate difference from go/doc. The + * brackets are doc-link syntax by the doc-comment specification ("[Name1. + * Name2] refers to exported identifiers in the current package"), and the + * field named is one declared entity, so nothing is guessed: its tier is + * `exact`. */ +static void go_bind_member(const go_ctx_t *c, const go_pkg_t *pkg, go_str_t recv, go_str_t name, + bool qualified) { + const go_pkg_t *named_pkg = pkg; + go_str_t named_recv = recv; + bool row = false; + const go_mem_t *m = qualified ? go_find_member(c, &pkg, &recv, name, &row) + : (const go_mem_t *)go_lookup(c->ix->mems, pkg->idx, recv, name); + if (row) { + return; + } + if (!m) { + go_undeclared(c, named_pkg, qualified, named_recv, name); + } else if (!go_one_bit(m->kinds)) { + go_row(c, CBM_DOCLINK_REASON_AMBIGUOUS); /* a method here, a field there */ + } else if (!m->prod && !c->src->is_test) { + go_row(c, CBM_DOCLINK_REASON_TEST_ONLY); + } else { + /* the member of one type: twins of the TYPE split its members too */ + const go_sym_t *ts = (const go_sym_t *)go_lookup(c->ix->syms, pkg->idx, recv, GO_NO_STR); + bool twins = m->free.files >= GO_TWINS || (ts && ts->free.files >= GO_TWINS); + const go_fmem_t *own = + pkg == c->src->pkg ? (const go_fmem_t *)go_lookup(c->ix->fmem, c->src->idx, recv, name) + : NULL; + const cbm_gbuf_node_t *n = m->kinds == GO_MEM_METHOD + ? go_method_node(c, pkg, recv, name) + : go_member_node(c, pkg, m->kinds, recv, name); + bool exact = qualified || m->kinds == GO_MEM_FIELD; + go_bind_node(c, n, twins, own && (own->kinds & m->kinds), exact); + } +} + +/* [pkg]: the package's directory node. The graph has a Folder node per + * directory and no node per Go package; the repository root is the Project + * node, which is no package. */ +static void go_bind_package(const go_ctx_t *c, const go_pkg_t *pkg) { + if (!pkg->has_prod && !c->src->is_test) { + go_row(c, CBM_DOCLINK_REASON_TEST_ONLY); + return; + } + const cbm_gbuf_node_t *n = cbm_gbuf_find_by_qn(c->g, pkg->dir->qn); + bool folder = n && n->label && strcmp(n->label, "Folder") == 0 && n->file_path && + strcmp(n->file_path, pkg->dir->path) == 0; + go_edge(c, folder ? n : NULL, true); +} + +/* Standard-library packages with a one-element import path (go/doc/comment + * stdPkgs): go/doc's own closed list, the only names it links without an + * import. A name of a later Go release that is missing here stays plain text; + * neither way is it an edge, unless the repository is the standard library + * itself (a `module std` tree), where the package is found by its path. */ +static bool go_is_std_single(go_str_t name) { + static const char *const std[] = { + "bufio", "bytes", "cmp", "context", "crypto", "embed", "encoding", + "errors", "expvar", "flag", "fmt", "hash", "html", "image", + "io", "iter", "log", "maps", "math", "mime", "net", + "os", "path", "plugin", "reflect", "regexp", "runtime", "slices", + "sort", "strconv", "strings", "structs", "sync", "syscall", "testing", + "time", "unicode", "unique", "unsafe", "uuid", "weak", + }; + return go_word_in(name, std, sizeof(std) / sizeof(std[0])); +} + +typedef enum { + GO_QUAL_NONE = 0, /* names no package: plain text */ + GO_QUAL_SELF, /* the documented file's own package */ + GO_QUAL_PATH, /* an import path */ +} go_qual_t; + +/* go/doc lookupPkg for a qualifier without a slash. + * + * go/doc itself knows only the UNION of the imports of all files of the + * package (one name with two paths is no name), then the package's own name, + * then its list of one-element standard-library paths. This resolver asks the + * documented FILE's own imports instead of the union: a strict subset of what + * go/doc links (a file can only disagree with the union by importing nothing + * under that name), so nothing is guessed, and a file's imports stay the + * file's own business -- with the union, every import edit would change its + * siblings' links and could not be repaired file by file. A qualifier that + * only a sibling file imports is therefore plain text here. + * + * A name two imports claim is no import name for go/doc, and it does not ask + * the package's own name then either: only the standard-library list is left. */ +static go_qual_t go_lookup_qualifier(const go_ctx_t *c, go_str_t q, go_str_t *path) { + const go_import_t *im = go_file_import(c->src, q); + if (im && !im->conflict) { + *path = go_str(im->path); + return GO_QUAL_PATH; + } + if (!im && go_str_eq(q, c->src->pkg->name)) { + return GO_QUAL_SELF; + } + if (go_is_std_single(q)) { + *path = q; + return GO_QUAL_PATH; + } + return GO_QUAL_NONE; +} + +/* True when the bare name is a type parameter in scope of the documented + * declaration: its own, or that of the generic type it is declared in (a + * field, an interface method). Generic declarations are top-level and do not + * overlap, so the one that starts last at or before the line is the only + * candidate. */ +static bool go_own_type_param(const go_file_t *f, uint32_t def_line, go_str_t name) { + int lo = 0; + int hi = f->ngenerics - SKIP_ONE; + const go_generic_t *hit = NULL; + while (lo <= hi) { + int mid = lo + ((hi - lo) / PAIR_LEN); + if (f->generics[mid].line <= def_line) { + hit = &f->generics[mid]; + lo = mid + SKIP_ONE; + } else { + hi = mid - SKIP_ONE; + } + } + if (hit && def_line > hit->last) { + hit = NULL; + } + for (const char *p = hit ? hit->names : ""; *p;) { + size_t n = strcspn(p, ","); + if (n == name.n && memcmp(p, name.s, n) == 0) { + return true; + } + p += n + (p[n] == ','); + } + return false; +} + +/* The package an import path means for the documented file; a row or the + * plain-text outcome is set, and NULL returned, when it means none. */ +static const go_pkg_t *go_path_package(const go_ctx_t *c, go_str_t path, go_str_t hint) { + const go_dir_t *dir = NULL; + go_path_kind_t kind = go_classify_path(c->ix, c->src, path, &dir); + bool ambiguous = kind == GO_PATH_AMBIGUOUS; + const go_pkg_t *pkg = kind == GO_PATH_PACKAGE ? go_dir_package(dir, hint, &ambiguous) : NULL; + if (kind == GO_PATH_EXTERNAL) { + go_row(c, CBM_DOCLINK_REASON_EXTERNAL); + } else if (kind == GO_PATH_UNKNOWN) { + go_row(c, CBM_DOCLINK_REASON_UNPARSEABLE); + } else if (ambiguous) { + go_row(c, CBM_DOCLINK_REASON_AMBIGUOUS); + } else if (!pkg) { + go_row(c, CBM_DOCLINK_REASON_MISSING); /* a module path without a package there */ + } + return ambiguous ? NULL : pkg; +} + +/* A reference qualified by `pkg` (an import path, or a name to look up). */ +static void go_resolve_qualified(const go_ctx_t *c, const CBMDocLinkGoRef *ref) { + go_str_t q = {ref->pkg, ref->pkg_len}; + go_str_t path = q; + go_str_t hint = q; + const go_pkg_t *pkg = c->src->pkg; + if (memchr(q.s, '/', q.n)) { + hint = go_path_base(q); + } else { + go_qual_t kind = go_lookup_qualifier(c, q, &path); + if (kind == GO_QUAL_NONE) { + go_not_a_link(c); + return; + } + if (kind == GO_QUAL_SELF) { + path.n = 0; + } + } + if (path.n > 0) { + pkg = go_path_package(c, path, hint); + if (!pkg) { + return; + } + } + go_str_t name = {ref->name, ref->name_len}; + go_str_t recv = {ref->recv, ref->recv_len}; + if (name.n == 0) { + go_bind_package(c, pkg); + } else if (recv.n == 0) { + go_bind_symbol(c, pkg, name, true); + } else { + go_bind_member(c, pkg, recv, name, true); + } +} + +/* True when the node the core takes for the edge's source IS the documented + * declaration: it sits in the file being resolved, at the declaration's line. + * Go method nodes share a flat name, so without this the doc of a method that + * lost the name (or of a build-tag twin) would be attributed to another + * declaration's node, in another file or on another line. That node's lines + * are known here: the file is one this run extracted. */ +static bool go_source_owns_node(const go_ctx_t *c, const CBMDocLink *link) { + if ((link->flags & CBM_DOCLINK_FLAG_FILE) || !link->source_qn) { + return true; /* the file's own doc: its File node is the source */ + } + const cbm_gbuf_node_t *src = cbm_gbuf_find_by_qn(c->g, link->source_qn); + if (!src) { + return true; /* no node at all: the core drops the reference */ + } + return src->file_path && strcmp(src->file_path, c->src->rel_path) == 0 && + src->start_line == (int)link->def_line; +} + +/* A well-formed reference of a file that has a package: by its shape. */ +static void go_resolve_ref(const go_ctx_t *c, const CBMDocLink *link, const CBMDocLinkGoRef *ref) { + go_str_t name = {ref->name, ref->name_len}; + go_str_t recv = {ref->recv, ref->recv_len}; + bool of_declaration = !(link->flags & CBM_DOCLINK_FLAG_FILE); + if (ref->pkg_len > 0) { + go_resolve_qualified(c, ref); + } else if (recv.n > 0) { + go_bind_member(c, c->src->pkg, recv, name, false); + } else if (of_declaration && go_own_type_param(c->src, link->def_line, name)) { + go_not_a_link(c); /* the declaration's own type parameter */ + } else { + go_bind_symbol(c, c->src->pkg, name, false); + } +} + +static void go_resolve(const void *index, void *state, int run_file, const CBMDocLink *link, + const cbm_gbuf_t *graph, cbm_doclink_outcome_t *out) { + (void)state; /* Go keeps no per-file working state */ + const go_index_t *ix = (const go_index_t *)index; + out->kind = CBM_DOCLINK_UNRESOLVED; + out->reason = CBM_DOCLINK_REASON_MISSING; + out->target = NULL; + out->exact = false; + if (!ix || !link->raw || run_file < 0 || run_file >= ix->run_count || + !ix->run_to_file[run_file]) { + return; + } + go_ctx_t c = {.ix = ix, .g = graph, .src = ix->run_to_file[run_file], .out = out}; + size_t raw_len = strlen(link->raw); + CBMDocLinkGoRef ref; + if (raw_len > CBM_DOCLINK_GO_REF_MAX) { + go_row(&c, CBM_DOCLINK_REASON_UNPARSEABLE); /* clipped at extraction: never resolved */ + return; + } + if (!cbm_doclink_go_parse_ref(link->raw, raw_len, &ref)) { + go_not_a_link(&c); /* go/doc knows no malformed link, only text */ + return; + } + if (!c.src->pkg) { + go_row(&c, CBM_DOCLINK_REASON_GRAPH_GAP); /* no package clause: nothing can be placed */ + return; + } + go_resolve_ref(&c, link, &ref); + if (out->kind == CBM_DOCLINK_EDGE && !go_source_owns_node(&c, link)) { + go_row(&c, CBM_DOCLINK_REASON_GRAPH_GAP); /* the documented declaration has no node */ + } +} + +/* ── Incremental scope rules ─────────────────────────────────────── */ + +/* The next record line of a blob; false at the end. */ +static bool go_delta_next(const char **cursor, go_str_t *line) { + const char *p = *cursor; + if (!p || !*p) { + return false; + } + const char *nl = strchr(p, '\n'); + line->s = p; + line->n = nl ? (size_t)(nl - p) : strlen(p); + *cursor = nl ? nl + SKIP_ONE : p + line->n; + return true; +} + +/* The first record line of a blob: what follows its tag line. */ +static const char *go_delta_start(const char *blob) { + const char *nl = strchr(blob, '\n'); + return nl ? nl + SKIP_ONE : NULL; +} + +static int go_line_cmp(go_str_t a, go_str_t b) { + int c = memcmp(a.s, b.s, a.n < b.n ? a.n : b.n); + if (c == 0 && a.n != b.n) { + c = a.n < b.n ? -SKIP_ONE : SKIP_ONE; + } + return c; +} + +/* Field `want` (0-based) of a record line; GO_NO_STR when it has fewer. */ +static go_str_t go_record_field(go_str_t line, int want) { + size_t start = 0; + int field = 0; + for (size_t i = 0; i <= line.n; i++) { + if (i < line.n && line.s[i] != '\t') { + continue; + } + if (field == want) { + return (go_str_t){line.s + start, i - start}; + } + field++; + start = i + SKIP_ONE; + } + return GO_NO_STR; +} + +/* The name a record declares: field 1 of F and V, field 2 of M, f and m. */ +static go_str_t go_record_name(go_str_t line) { + return go_record_field(line, (line.s[0] == 'F' || line.s[0] == 'V') ? SKIP_ONE : PAIR_LEN); +} + +/* `T name a pkg.Y w`: an alias whose target is read through the file's + * imports. */ +static bool go_is_qualified_alias(go_str_t line) { + enum { T_ALIAS_FIELD = 3 }; + if (line.n == 0 || line.s[0] != 'T') { + return false; + } + go_str_t alias = go_record_field(line, T_ALIAS_FIELD); + return alias.s && memchr(alias.s, '.', alias.n) != NULL; +} + +typedef struct { + cbm_doclink_name_fn removed; + void *ud; + bool imports_changed; + bool qualified_alias; + bool global; + bool failed; +} go_delta_t; + +/* A record only one of the two scopes has. */ +static void go_delta_record(go_delta_t *d, go_str_t line, bool stored_only) { + char kind = line.s[0]; + if (kind == 'I') { + d->imports_changed = true; + return; + } + if (kind == 'g') { + return; /* never persisted; a fresh blob may still carry them */ + } + bool removable = kind == 'F' || kind == 'M' || kind == 'V' || kind == 'f' || kind == 'm'; + if (!stored_only || !removable) { + d->global = true; /* a new or changed record, a package, type or module line */ + return; + } + go_str_t name = go_record_name(line); + if (!name.s || name.n == 0 || !d->removed || !d->removed(d->ud, name.s, name.n)) { + d->failed = true; + } +} + +/* One step of the merge of two sorted scopes: -1 a record only the stored + * scope has, 1 one only the fresh scope has, 0 the same record in both. */ +static int go_delta_step(go_delta_t *d, bool has_stored, go_str_t stored, bool has_fresh, + go_str_t fresh) { + int c = 0; + if (!has_stored) { + c = SKIP_ONE; + } else if (!has_fresh) { + c = -SKIP_ONE; + } else { + c = go_line_cmp(stored, fresh); + } + d->qualified_alias = d->qualified_alias || (c <= 0 && go_is_qualified_alias(stored)) || + (c >= 0 && go_is_qualified_alias(fresh)); + if (c < 0) { + go_delta_record(d, stored, true); + } else if (c > 0) { + go_delta_record(d, fresh, false); + } + return c; +} + +/* Both scopes are sorted, so one merge finds what each has alone. + * LOCAL nothing differs; or only the file's imports differ (they scope + * the file itself, which is re-extracted anyway); or declarations + * only LEFT: a function, method, value, field or interface method. + * Their names are reported: files with an edge into the changed + * file are dependents anyway, and files whose unresolved rows name + * one re-resolve. + * GLOBAL anything that appeared or changed (a name appearing can turn + * another file's plain text into a link or a graph gap; a w field + * flipping hands a node to another declaration), the package + * clause, a type (an alias may be followed through it), a module + * path, and the imports of a file with a qualified alias (other + * files follow `type X = pkg.Y` through them). */ +static int go_scope_delta(const char *stored, const char *fresh, cbm_doclink_name_fn removed, + void *ud) { + go_delta_t d = {.removed = removed, .ud = ud}; + const char *sp = go_delta_start(stored); + const char *fp = go_delta_start(fresh); + go_str_t sl = GO_NO_STR; + go_str_t fl = GO_NO_STR; + bool hs = go_delta_next(&sp, &sl); + bool hf = go_delta_next(&fp, &fl); + while ((hs || hf) && !d.global && !d.failed) { + int c = go_delta_step(&d, hs, sl, hf, fl); + if (c <= 0) { + hs = go_delta_next(&sp, &sl); + } + if (c >= 0) { + hf = go_delta_next(&fp, &fl); + } + } + if (d.failed) { + return CBM_NOT_FOUND; + } + bool global = d.global || (d.imports_changed && d.qualified_alias); + return global ? CBM_DOCLINK_DELTA_GLOBAL : CBM_DOCLINK_DELTA_LOCAL; +} + +const cbm_doclink_resolver_t cbm_doclink_go_resolver = { + .langs = {CBM_LANG_GO, CBM_LANG_GOMOD}, + .lang_count = 2, + .scope_tag = CBM_DOCLINK_GO_SCOPE_TAG, + .build = go_build, + .destroy = go_destroy, + .resolve = go_resolve, + .scope_delta = go_scope_delta, +}; diff --git a/tests/test_doc_mentions_go.c b/tests/test_doc_mentions_go.c new file mode 100644 index 000000000..f3d9b58b5 --- /dev/null +++ b/tests/test_doc_mentions_go.c @@ -0,0 +1,2112 @@ +/* + * test_doc_mentions_go.c — Go doc links ([Name], [pkg.Name], [Recv.Method]) + * -> MENTIONS edges. + * + * Extraction (doclink_go.c: the go/doc/comment text model, the candidate + * tokens, the scope blob) and the resolver (doc_links_go.c) through real + * fixtures indexed with cbm_pipeline_run. + */ +#include "test_framework.h" +#include "test_doc_mentions_helpers.h" + +/* ── helpers of this file ────────────────────────────────────────── */ + +static CBMFileResult *go_extract(const char *src, const char *rel) { + return dm_extract(src, CBM_LANG_GO, rel); +} + +/* The tokens of one doc comment put above `type T struct{}` in package p. */ +static CBMFileResult *go_doc(const char *comment) { + char src[8192]; + snprintf(src, sizeof(src), "package p\n\n%stype T struct{}\n", comment); + return go_extract(src, "pkg/t.go"); +} + +/* True when the scope blob holds exactly this record line. */ +static bool go_has_rec(const char *scope, const char *rec) { + size_t n = strlen(rec); + for (const char *p = scope; p && *p;) { + const char *nl = strchr(p, '\n'); + size_t len = nl ? (size_t)(nl - p) : strlen(p); + if (len == n && memcmp(p, rec, n) == 0) { + return true; + } + p = nl ? nl + 1 : NULL; + } + return false; +} + +static void go_w(const char *repo, const char *rel, const char *content) { + th_write_file(TH_PATH(repo, rel), content); +} + +/* Tier of the one MENTIONS edge src -> tgt (qualified-name suffixes): "exact" + * or "unique"; "" when there is no such edge; "many" for more than one. */ +static const char *go_tier(const char *db, const char *src, const char *tgt) { + char props[512]; + int n = 0; + dm_edge(db, src, tgt, props, sizeof(props), &n); + if (n == 0) { + return ""; + } + if (n != 1) { + return "many"; + } + if (strstr(props, "\"syntax\":\"doc_link\"") == NULL) { + return "other-syntax"; + } + return strstr(props, "\"tier\":\"exact\"") ? "exact" : "unique"; +} + +/* Reason of the unresolved row for `raw` in `rel`; "" when there is none. */ +static const char *go_reason(const char *db, const char *rel, const char *raw) { + static char reason[64]; + dm_row(db, rel, raw, reason, sizeof(reason), NULL, 0); + return reason; +} + +typedef struct { + const char *src; + const char *tgt; + const char *tier; /* "": there must be no edge */ +} go_want_edge_t; + +typedef struct { + const char *rel; + const char *raw; + const char *reason; /* "": there must be no row */ +} go_want_row_t; + +/* The number of edges that are not as wanted (each one printed). */ +static int go_bad_edges(const char *db, const go_want_edge_t *want, size_t n) { + int bad = 0; + for (size_t i = 0; i < n; i++) { + const char *tier = go_tier(db, want[i].src, want[i].tgt); + if (strcmp(tier, want[i].tier) != 0) { + printf(" edge %s -> %s: \"%s\", want \"%s\"\n", want[i].src, want[i].tgt, tier, + want[i].tier); + bad++; + } + } + return bad; +} + +/* The number of rows that are not as wanted (each one printed). */ +static int go_bad_rows(const char *db, const go_want_row_t *want, size_t n) { + int bad = 0; + for (size_t i = 0; i < n; i++) { + const char *reason = go_reason(db, want[i].rel, want[i].raw); + if (strcmp(reason, want[i].reason) != 0) { + printf(" row %s [%s]: \"%s\", want \"%s\"\n", want[i].rel, want[i].raw, reason, + want[i].reason); + bad++; + } + } + return bad; +} + +#define GO_COUNT(a) (sizeof(a) / sizeof((a)[0])) + +/* ── extraction: tokens ──────────────────────────────────────────── */ + +TEST(doc_mentions_go_tokens) { + const char *src = + "package p\n" /* 1 */ + "\n" /* 2 */ + "// Widget does things. See [Thing], [*Star], [Thing.Run],\n" /* 3 */ + "// [fmt.Println], [encoding/json.Marshal], [io] and [pkg.Recv.Name].\n" /* 4 */ + "// Not links: map[string]Value, a[i], x[Mid]y, [not a link], [lower.case],\n" /* 5 */ + "// [Plural]s and s[Sub].\n" /* 6 */ + "//\n" /* 7 */ + "//\tcode [InCode] here\n" /* 8 */ + "//\n" /* 9 */ + "// # Heading [InHeading]\n" /* 10 */ + "//\n" /* 11 */ + "// - item [InList] one\n" /* 12 */ + "// - [RFC 7159] defined below\n" /* 13 */ + "//\n" /* 14 */ + "// [RFC 7159]: https://tools.ietf.org/html/rfc7159\n" /* 15 */ + "type Widget struct{}\n"; /* 16 */ + CBMFileResult *r = go_extract(src, "pkg/w.go"); + ASSERT_NOT_NULL(r); + const CBMDocLink *thing = dm_find_token(r, "Thing"); + ASSERT_NOT_NULL(thing); + ASSERT_EQ(thing->line, 3); + ASSERT_EQ(thing->syntax, CBM_DOCLINK_GO_DOC_LINK); + ASSERT_STR_EQ(thing->source_qn, "p.pkg.Widget"); + ASSERT_EQ(thing->def_line, 16); + /* the star stays in the raw text; the resolver skips it */ + ASSERT_NOT_NULL(dm_find_token(r, "*Star")); + ASSERT_NOT_NULL(dm_find_token(r, "Thing.Run")); + const CBMDocLink *println = dm_find_token(r, "fmt.Println"); + ASSERT_NOT_NULL(println); + ASSERT_EQ(println->line, 4); + ASSERT_NOT_NULL(dm_find_token(r, "encoding/json.Marshal")); + ASSERT_NOT_NULL(dm_find_token(r, "io")); + ASSERT_NOT_NULL(dm_find_token(r, "pkg.Recv.Name")); + /* the border rule: a bracket pair touching a letter is an index or a + * type, never a link */ + ASSERT_NULL(dm_find_token(r, "string")); + ASSERT_NULL(dm_find_token(r, "i")); + ASSERT_NULL(dm_find_token(r, "Mid")); + ASSERT_NULL(dm_find_token(r, "Plural")); + ASSERT_NULL(dm_find_token(r, "Sub")); + /* no doc-link shape */ + ASSERT_NULL(dm_find_token(r, "not a link")); + ASSERT_NULL(dm_find_token(r, "lower.case")); + /* a code block and a heading hold no links; a list item does */ + ASSERT_NULL(dm_find_token(r, "InCode")); + ASSERT_NULL(dm_find_token(r, "InHeading")); + const CBMDocLink *in_list = dm_find_token(r, "InList"); + ASSERT_NOT_NULL(in_list); + ASSERT_EQ(in_list->line, 12); + /* a text with a link definition is a URL: one href token per use, none + * for the definition itself */ + const CBMDocLink *href = dm_find_token(r, "https://tools.ietf.org/html/rfc7159"); + ASSERT_NOT_NULL(href); + ASSERT_EQ(href->syntax, CBM_DOCLINK_HREF); + ASSERT_EQ(href->line, 13); + ASSERT_EQ(dm_count_tokens(r, "https://tools.ietf.org/html/rfc7159"), 1); + ASSERT_NULL(dm_find_token(r, "RFC 7159")); + ASSERT_EQ(r->doc_links.count, 9); + cbm_free_result(r); + + /* the family and its name */ + ASSERT_STR_EQ(cbm_doclink_syntax_name(CBM_DOCLINK_GO_DOC_LINK), "doc_link"); + const CBMDocLinkFamily *f = cbm_doclink_family(CBM_DOCLINK_GO_DOC_LINK); + ASSERT_NOT_NULL(f); + ASSERT_EQ(f->lang, CBM_LANG_GO); + /* held in production until its held-out audit passes; the suite ships it */ + cbm_doclink_test_reset_ships(); + bool ships_in_production = cbm_doclink_syntax_ships(CBM_DOCLINK_GO_DOC_LINK); + dm_ship_held_families(); + ASSERT_FALSE(ships_in_production); + ASSERT_TRUE(cbm_doclink_syntax_ships(CBM_DOCLINK_GO_DOC_LINK)); + PASS(); +} + +/* Every documented kind of definition is a source. */ +TEST(doc_mentions_go_sources) { + const char *src = "package p\n" + "\n" + "// F doc [RefF].\n" + "func F() {}\n" + "\n" + "// S doc [RefS].\n" + "type S struct {\n" + "\t// Field doc [RefField].\n" + "\tField int\n" + "}\n" + "\n" + "// M doc [RefM].\n" + "func (s *S) M() {}\n" + "\n" + "// I doc [RefI].\n" + "type I interface {\n" + "\t// Do doc [RefDo].\n" + "\tDo()\n" + "}\n" + "\n" + "// C doc [RefC].\n" + "const C = 1\n" + "\n" + "const (\n" + "\t// K doc [RefK].\n" + "\tK = iota\n" + ")\n" + "\n" + "// V doc [RefV].\n" + "var V = 2\n" + "\n" + "// A doc [RefA].\n" + "type A = S\n"; + CBMFileResult *r = go_extract(src, "pkg/s.go"); + ASSERT_NOT_NULL(r); + static const struct { + const char *raw; + const char *source; + } want[] = { + {"RefF", "p.pkg.F"}, {"RefS", "p.pkg.S"}, {"RefField", "p.pkg.S.Field"}, + {"RefM", "p.pkg.M"}, /* Go method nodes have a flat name */ + {"RefI", "p.pkg.I"}, {"RefDo", "p.pkg.I.Do"}, {"RefC", "p.pkg.C"}, + {"RefK", "p.pkg.K"}, {"RefV", "p.pkg.V"}, {"RefA", "p.pkg.A"}, + }; + for (size_t i = 0; i < GO_COUNT(want); i++) { + const CBMDocLink *t = dm_find_token(r, want[i].raw); + if (!t || strcmp(t->source_qn, want[i].source) != 0) { + printf(" %s: source %s, want %s\n", want[i].raw, t ? t->source_qn : "(no token)", + want[i].source); + FAIL("source of a doc link"); + } + ASSERT_EQ(dm_count_tokens(r, want[i].raw), 1); + ASSERT_EQ(t->flags & CBM_DOCLINK_FLAG_FILE, 0); + } + cbm_free_result(r); + PASS(); +} + +/* The package comment is a source of its own (the File node), harvested once: + * no definition's docstring is the package comment. */ +TEST(doc_mentions_go_package_comment_tokens) { + static const struct { + const char *what; + const char *src; + int file_tokens; /* tokens [Ref] flagged as written in the file's doc */ + int def_tokens; /* tokens [Ref] of a definition */ + uint32_t line; /* line of the file-level token */ + } cases[] = { + {"package comment, then a documented declaration", + "// Package p has a [Ref].\npackage p\n\n// F is.\nfunc F() {}\n", 1, 0, 1}, + {"declaration without a doc right after the clause", + "// Package p has a [Ref].\npackage p\nfunc F() {}\ntype T struct{}\nvar V = 1\n", 1, 0, + 1}, + {"block comment", "/*\nPackage p has a [Ref].\n*/\npackage p\n\nfunc F() {}\n", 1, 0, 2}, + {"a header comment is no package comment", + "// Copyright [Ref].\n\npackage p\n\nfunc F() {}\n", 0, 0, 0}, + {"header, then the package comment", + "// Copyright.\n\n// Package p has a [Ref].\npackage p\n\nfunc F() {}\n", 1, 0, 3}, + {"build line, then the package comment", + "//go:build linux\n\n// Package p has a [Ref].\npackage p\n", 1, 0, 3}, + {"the same link in the package comment and in a declaration's doc", + "// Package p has a [Ref].\npackage p\n\n// F has a [Ref].\nfunc F() {}\n", 1, 1, 1}, + }; + for (size_t i = 0; i < GO_COUNT(cases); i++) { + CBMFileResult *r = go_extract(cases[i].src, "pkg/doc.go"); + ASSERT_NOT_NULL(r); + int file_tokens = 0; + int def_tokens = 0; + uint32_t line = 0; + for (int k = 0; k < r->doc_links.count; k++) { + const CBMDocLink *t = &r->doc_links.items[k]; + if (strcmp(t->raw, "Ref") != 0) { + continue; + } + if (t->flags & CBM_DOCLINK_FLAG_FILE) { + file_tokens++; + line = t->line; + } else { + def_tokens++; + } + } + if (file_tokens != cases[i].file_tokens || def_tokens != cases[i].def_tokens || + line != cases[i].line) { + printf(" %s: %d file tokens (line %u), %d definition tokens\n", cases[i].what, + file_tokens, line, def_tokens); + cbm_free_result(r); + FAIL("package comment harvested once, as the file's"); + } + cbm_free_result(r); + } + PASS(); +} + +/* The go/doc/comment text model, case by case. */ +TEST(doc_mentions_go_text_model) { + static const struct { + const char *what; + const char *comment; + const char *present; /* '|'-separated raw texts that must be tokens */ + const char *absent; /* '|'-separated raw texts that must not be */ + } cases[] = { + {"block comment", "/*\nBlock [InBlock] text.\n\n\tcode [BlockCode]\n*/\n", "InBlock", + "BlockCode"}, + {"block comment on one line", "/* One [OneLine] line. */\n", "OneLine", ""}, + {"common indent is no code", "/*\n Indented [Para] text\n more text\n*/\n", "Para", + ""}, + {"no space after the slashes", "//[Tight] text\n", "Tight", ""}, + {"directive lines are no text", "// See [Kept].\n//go:generate echo [Gen]\n", "Kept", + "Gen"}, + {"carriage returns", "// Line [Cr] one.\r\n// Line two.\r\n", "Cr", ""}, + {"numbered list", "// Steps:\n// 1. first [NumOne]\n// 2) second [NumTwo]\n", + "NumOne|NumTwo", ""}, + {"list item continuation", "// List:\n// - item starts\n// and [Continued] here\n", + "Continued", ""}, + {"code after a paragraph without a blank line", + "// Text [BeforeCode] then\n//\tx := [NotThis]\n", "BeforeCode", "NotThis"}, + {"brace line joins the code block", + "// func main() {\n//\tuse([InBody])\n// }\n// After [AfterBrace].\n", "AfterBrace", + "InBody"}, + {"unindented list before its wrapped line", + "// - first [ListA]\n// - second\n// wrapped [ListB]\n", "ListA|ListB", ""}, + {"heading needs blank lines only towards text", + "// Intro.\n//\n// # Title [InTitle]\n//\n// Body [InBody2].\n", "InBody2", "InTitle"}, + {"hash line inside a paragraph is text", "// Intro\n// # not a heading [InPara]\n", + "InPara", ""}, + {"a pair restarts at every opening bracket", "// See [[Inner]] and [a [Second]].\n", + "Inner|Second", "a [Second"}, + {"brackets across a line break are no link", "// See [Split\n// Name] here.\n", "", + "Split\nName|Split Name"}, + {"unicode punctuation borders", + "// See \xE2\x80\x9C[Quoted]\xE2\x80\x9D \xE2\x80\x94[Dashed]\xE2\x80\xA6\n", + "Quoted|Dashed", ""}, + {"letters of any script are no border", + "// See \xC3\xA9[AfterLetter] and [BeforeLetter]\xD0\xB6.\n", "", + "AfterLetter|BeforeLetter"}, + {"symbols are no punctuation", "// See `[Ticked]` =[Equals] +[Plus]+ $[Dollar].\n", "", + "Ticked|Equals|Plus|Dollar"}, + {"punctuation on both sides", "// See ([Paren]), \"[Quote]\", [Comma], [Dot].\n", + "Paren|Quote|Comma|Dot", ""}, + {"paragraph edges", "// [First] word and last [Last]\n", "First|Last", ""}, + {"link definitions must fill their paragraph", + "// [Mixed]: https://example.com/a\n// and text [Other].\n", "Other|Mixed", ""}, + {"unknown scheme defines nothing", "// Use [Odd].\n//\n// [Odd]: gopher2://example.com\n", + "Odd", ""}, + /* go/doc looks a link text up from the FIRST opening bracket since the + * last closing one, without the brackets: `[a [Def]` asks for `a Def` */ + {"nested opening brackets: the pair's own text is not the looked-up text", + "// See [a [Def] here.\n//\n// [Def]: https://example.com/def\n", "Def", + "https://example.com/def"}, + {"nested opening brackets: the text since the first one is", + "// See [a [Def] here.\n//\n// [a Def]: https://example.com/adef\n", + "https://example.com/adef", "Def"}, + {"name shapes", "// [x.Y] [x.Y.Z] [X.Y] [a/b] [a/b.C] [a/b.C.D] [*a/b.C]\n", + "x.Y|x.Y.Z|X.Y|a/b|a/b.C|a/b.C.D|*a/b.C", ""}, + {"bad shapes", + "// [x.y.Z] [X.y] [1x] [X-Y] [a//b] [a/b/] [-a/b] [a/.b] [x.Y.Z.W] [**X] [] [*]\n", "", + "x.y.Z|X.y|X-Y|a//b|a/b/|-a/b|a/.b|x.Y.Z.W|**X|*"}, + {"a quirk kept from go/doc: a leading dot", "// See [.Dotted].\n", ".Dotted", ""}, + /* a Name starts with an upper-case letter of any script; the word + * before the dot is a package name whatever its letters */ + {"capital letters beyond ASCII", + "// See [\xC3\x9C" + "ber], [\xC3\xBC" + "ber.Name] and [X.\xC3\xBC" + "ber].\n", + "\xC3\x9C" + "ber|\xC3\xBC" + "ber.Name", + "X.\xC3\xBC" + "ber"}, + }; + for (size_t i = 0; i < GO_COUNT(cases); i++) { + CBMFileResult *r = go_doc(cases[i].comment); + ASSERT_NOT_NULL(r); + for (int pass = 0; pass < 2; pass++) { + char list[512]; + snprintf(list, sizeof(list), "%s", pass == 0 ? cases[i].present : cases[i].absent); + for (char *tok = list; tok && *tok;) { + char *bar = strchr(tok, '|'); + if (bar) { + *bar = '\0'; + } + bool found = dm_find_token(r, tok) != NULL; + if (found != (pass == 0)) { + printf(" %s: [%s] %s\n", cases[i].what, tok, + found ? "is a token" : "is no token"); + cbm_free_result(r); + FAIL("go/doc text model"); + } + tok = bar ? bar + 1 : NULL; + } + } + cbm_free_result(r); + } + PASS(); +} + +/* A reference to the documented function's own parameter is local. */ +TEST(doc_mentions_go_own_parameter) { + const char *src = "package p\n" + "\n" + "// Fetch reads from [url] with [Client] and reports to [Other].\n" + "func Fetch(url string, Client int) {}\n" + "\n" + "// Later mentions [url] and [Client], which are no parameters here.\n" + "func Later() {}\n"; + CBMFileResult *r = go_extract(src, "pkg/f.go"); + ASSERT_NOT_NULL(r); + ASSERT_NOT_NULL(dm_find_token(r, "Other")); + /* one token each: the one of Later */ + ASSERT_EQ(dm_count_tokens(r, "url"), 1); + ASSERT_EQ(dm_count_tokens(r, "Client"), 1); + ASSERT_STR_EQ(dm_find_token(r, "url")->source_qn, "p.pkg.Later"); + ASSERT_STR_EQ(dm_find_token(r, "Client")->source_qn, "p.pkg.Later"); + cbm_free_result(r); + PASS(); +} + +/* ── extraction: hostile input ───────────────────────────────────── */ + +TEST(doc_mentions_go_hostile_doc) { + /* an unterminated block comment, a bracket at the very end, a lone + * closing bracket, a bracket that never closes */ + CBMFileResult *r = go_doc("// ] [ [Ok] [\n"); + ASSERT_NOT_NULL(r); + ASSERT_EQ(r->doc_links.count, 1); + ASSERT_NOT_NULL(dm_find_token(r, "Ok")); + cbm_free_result(r); + + /* invalid UTF-8 next to a bracket is no border: no token, no crash */ + r = go_doc("// \xFF[Bad1] [Bad2]\xC3 [Bad3]\x80 \xE2\x80[Bad4] [Good].\n"); + ASSERT_NOT_NULL(r); + ASSERT_NULL(dm_find_token(r, "Bad1")); + ASSERT_NULL(dm_find_token(r, "Bad2")); + ASSERT_NULL(dm_find_token(r, "Bad3")); + ASSERT_NULL(dm_find_token(r, "Bad4")); + ASSERT_NOT_NULL(dm_find_token(r, "Good")); + cbm_free_result(r); + + /* an oversized reference is harvested clipped (one byte past the limit, + * so the resolver sees that it is too long and never resolves it); the + * next reference is read as usual */ + enum { BIG = 1 << 20 }; + char *src = malloc(BIG + 256); + ASSERT_NOT_NULL(src); + size_t w = (size_t)snprintf(src, 64, "package p\n\n// See ["); + memset(src + w, 'A', BIG); + w += BIG; + w += (size_t)snprintf(src + w, 128, "] and [Small].\ntype T struct{}\n"); + r = cbm_extract_file(src, (int)w, CBM_LANG_GO, "p", "pkg/big.go", 0, NULL, NULL); + ASSERT_NOT_NULL(r); + ASSERT_EQ(r->doc_links.count, 2); + ASSERT_NOT_NULL(dm_find_token(r, "Small")); + ASSERT_EQ(strlen(r->doc_links.items[0].raw), (size_t)CBM_DOCLINK_GO_REF_MAX + 1); + ASSERT_EQ(r->doc_links.items[0].syntax, CBM_DOCLINK_GO_DOC_LINK); + cbm_free_result(r); + + /* oversized text that is no link (it has spaces) is no token at all */ + w = (size_t)snprintf(src, 64, "package p\n\n// See ["); + memset(src + w, 'A', BIG); + src[w + 100] = ' '; + w += BIG; + w += (size_t)snprintf(src + w, 128, "] and [Small].\ntype T struct{}\n"); + r = cbm_extract_file(src, (int)w, CBM_LANG_GO, "p", "pkg/prose.go", 0, NULL, NULL); + ASSERT_NOT_NULL(r); + ASSERT_EQ(r->doc_links.count, 1); + ASSERT_NOT_NULL(dm_find_token(r, "Small")); + cbm_free_result(r); + + /* a megabyte of opening brackets, then one pair: linear, one token */ + w = (size_t)snprintf(src, 64, "package p\n\n// "); + memset(src + w, '[', BIG); + w += BIG; + w += (size_t)snprintf(src + w, 128, "Deep] end.\ntype T struct{}\n"); + r = cbm_extract_file(src, (int)w, CBM_LANG_GO, "p", "pkg/deep.go", 0, NULL, NULL); + ASSERT_NOT_NULL(r); + ASSERT_EQ(r->doc_links.count, 1); + ASSERT_NOT_NULL(dm_find_token(r, "Deep")); + cbm_free_result(r); + + /* thousands of link definitions and uses: every use resolves to its own + * definition (first definition wins), without a quadratic scan */ + enum { DEFS = 4000 }; + size_t cap = (size_t)DEFS * 96 + 256; + char *many = malloc(cap); + ASSERT_NOT_NULL(many); + w = (size_t)snprintf(many, cap, "package p\n\n"); + for (int i = 0; i < DEFS; i++) { + w += (size_t)snprintf(many + w, cap - w, "// Use [t%d].\n", i); + } + w += (size_t)snprintf(many + w, cap - w, "//\n"); + for (int i = 0; i < DEFS; i++) { + w += (size_t)snprintf(many + w, cap - w, "// [t%d]: https://example.com/%d\n", i, i); + } + w += (size_t)snprintf(many + w, cap - w, + "// [t7]: https://example.com/shadowed\n" + "type T struct{}\n"); + r = cbm_extract_file(many, (int)w, CBM_LANG_GO, "p", "pkg/many.go", 0, NULL, NULL); + ASSERT_NOT_NULL(r); + ASSERT_EQ(r->doc_links.count, DEFS); + ASSERT_EQ(dm_count_tokens(r, "https://example.com/7"), 1); + ASSERT_NULL(dm_find_token(r, "https://example.com/shadowed")); + cbm_free_result(r); + free(many); + + /* a NUL byte ends the comment text; what precedes it is still read */ + static const char nul_src[] = "package p\n\n// See [Before]\0[After].\ntype T struct{}\n"; + r = cbm_extract_file(nul_src, (int)sizeof(nul_src) - 1, CBM_LANG_GO, "p", "pkg/nul.go", 0, NULL, + NULL); + ASSERT_NOT_NULL(r); + ASSERT_NULL(dm_find_token(r, "After")); + cbm_free_result(r); + free(src); + PASS(); +} + +/* A fixed pseudo-random sequence: the same texts on every run and platform. */ +static uint32_t go_rand(uint32_t *state) { + *state = (*state * 1664525U) + 1013904223U; + return *state >> 8; +} + +/* Append one of `count` pieces, chosen by the sequence. */ +static size_t go_rand_piece(char *buf, size_t len, size_t cap, uint32_t *state, + const char *const *pieces, size_t count) { + const char *p = pieces[go_rand(state) % count]; + size_t pl = strlen(p); + if (len + pl >= cap) { + return len; + } + memcpy(buf + len, p, pl); + return len + pl; +} + +/* Comment text made of the pieces the text model reacts to, in every order: + * the parser must stay inside its buffers (the runner has the sanitizers on) + * and every token must keep the shape the resolver relies on. */ +TEST(doc_mentions_go_random_docs) { + static const char *const pieces[] = { + "[", + "]", + "[", + "]", + "*", + ".", + "/", + "Name", + "pkg", + "a/b", + " ", + " ", + "\n// ", + "\n//\t", + "\n//", + "\n//\n// ", + "# ", + "- ", + "1. ", + ": ", + "https://x", + "\xC3\xA9", + "\xFF", + "\xE2\x80", + "\xE2\x80\x9C", + "{", + "}", + "\\", + "\r", + "\t", + "T", + "x", + "]: ", + "[Name]", + "[p.Name]", + "[a/b.Name.M]", + "\n//\n// [Name]: https://x/y\n//\n// ", + "/*", + "*/", + "_", + "9", + }; + uint32_t state = 20261002U; + char src[4096]; + int doc_links = 0; + int hrefs = 0; + for (int round = 0; round < 1500; round++) { + size_t len = (size_t)snprintf(src, sizeof(src), "package p\n\n// "); + int n = 1 + (int)(go_rand(&state) % 120); + for (int i = 0; i < n; i++) { + len = go_rand_piece(src, len, sizeof(src) - 64, &state, pieces, GO_COUNT(pieces)); + } + len += (size_t)snprintf(src + len, sizeof(src) - len, "\nfunc F(Name int) {}\n"); + CBMFileResult *r = + cbm_extract_file(src, (int)len, CBM_LANG_GO, "p", "pkg/r.go", 0, NULL, NULL); + ASSERT_NOT_NULL(r); + for (int k = 0; k < r->doc_links.count; k++) { + const CBMDocLink *t = &r->doc_links.items[k]; + size_t raw_len = t->raw ? strlen(t->raw) : 0; + bool shape_ok = t->source_qn && raw_len > 0 && t->line >= 3 && + (t->syntax == CBM_DOCLINK_GO_DOC_LINK || t->syntax == CBM_DOCLINK_HREF); + CBMDocLinkGoRef ref; + if (shape_ok && t->syntax == CBM_DOCLINK_GO_DOC_LINK) { + /* a doc-link token is what the resolver can read back */ + shape_ok = raw_len <= CBM_DOCLINK_GO_REF_MAX && + cbm_doclink_go_parse_ref(t->raw, raw_len, &ref) && + memchr(t->raw, '\n', raw_len) == NULL; + } + if (!shape_ok) { + printf(" round %d: token [%s] syntax %d line %u\n", round, t->raw ? t->raw : "", + (int)t->syntax, t->line); + cbm_free_result(r); + FAIL("token shape"); + } + doc_links += t->syntax == CBM_DOCLINK_GO_DOC_LINK; + hrefs += t->syntax == CBM_DOCLINK_HREF; + } + cbm_free_result(r); + } + /* the texts do hold links of both families: the checks above ran */ + if (doc_links < 1000 || hrefs < 1) { + printf(" %d doc links, %d hrefs\n", doc_links, hrefs); + FAIL("random texts yield tokens"); + } + PASS(); +} + +/* Declarations in every order and state of repair: the scope scan must stay + * inside its buffers and write only well-formed records. */ +TEST(doc_mentions_go_random_scopes) { + static const char *const pieces[] = { + "func ", "type ", + "var ", "const ", + "import ", "package q\n", + "struct ", "interface ", + "(", ")", + "{", "}", + "[", "]", + "\n", "\n", + "\t", " ", + "=", ",", + ".", "*", + "A", "Bee", + "c", "T", + "int", "\"a/b\"", + "`x`", "//go:build ignore\n", + "// c\n", "/* c */", + "1", ";", + "any", "func (r *T[K]) ", + "Bee int", "A, Bee string", + }; + uint32_t state = 20261003U; + char src[4096]; + for (int round = 0; round < 1500; round++) { + size_t len = + (size_t)snprintf(src, sizeof(src), "%s", (round % 4 == 0) ? "" : "package p\n\n"); + int n = 1 + (int)(go_rand(&state) % 150); + for (int i = 0; i < n; i++) { + len = go_rand_piece(src, len, sizeof(src) - 8, &state, pieces, GO_COUNT(pieces)); + } + src[len] = '\0'; + CBMFileResult *r = + cbm_extract_file(src, (int)len, CBM_LANG_GO, "p", "pkg/r.go", 0, NULL, NULL); + ASSERT_NOT_NULL(r); + const char *s = r->doc_scope; + bool ok = s && strncmp(s, "go1", 3) == 0 && (s[3] == '\n' || s[3] == '\0'); + for (const char *p = ok ? strchr(s, '\n') : NULL; ok && p && p[1];) { + const char *line = p + 1; + ok = strchr("PIFMTVfmg", line[0]) != NULL && line[1] == '\t' && line[2] != '\0' && + line[2] != '\n'; + p = strchr(line, '\n'); + } + char *portable = ok ? cbm_doclink_portable_scope(s) : NULL; + ok = ok && portable && strstr(portable, "\ng\t") == NULL; + cbm_free(CBM_MEM_CLASS_OTHER, portable); + if (!ok) { + printf(" round %d: scope\n%s\nof\n%s\n", round, s ? s : "(null)", src); + cbm_free_result(r); + FAIL("scope records"); + } + cbm_free_result(r); + } + PASS(); +} + +/* ── extraction: scope blob ──────────────────────────────────────── */ + +TEST(doc_mentions_go_scope_blob) { + const char *src = "package alpha\n" + "\n" + "import (\n" + "\t\"fmt\"\n" + "\tal \"example.com/demo/pkg/alpha\"\n" + "\t. \"example.com/dot\"\n" + "\t_ \"example.com/blank\"\n" + "\tbad \"a\\tb\"\n" + ")\n" + "import \"io\"\n" + "\n" + "const Max = 10\n" + "const (\n" + "\tKindA, KindB = 1, 2\n" + ")\n" + "var Err = fmt.Errorf(\"x\")\n" + "var (\n" + "\tBlockVar = 1\n" + ")\n" + "var hidden = 1\n" + "\n" + "type Widget struct {\n" + "\tName, Size string\n" + "\tio.Reader\n" + "\t*Kind\n" + "\tsecret int\n" + "}\n" + "type Runner interface {\n" + "\tRun() error\n" + "\tstop()\n" + "\tio.Closer\n" + "}\n" + "type Kind int\n" + "type Alias = al.Widget\n" + "type Same = Kind\n" + "type List[T any, U comparable] struct{ items []T }\n" + "type lower struct{ Field int }\n" + "\n" + "func New() *Widget { return nil }\n" + "func (w *Widget) Run() error { return nil }\n" + "func (k Kind) Run() error { return nil }\n" + "func (l *List[A, B]) Push(v A) {}\n" + "func (x lower) Exported() {}\n" + "func Map[K comparable, V any](m map[K]V) {}\n" + "func helper() {}\n"; + CBMFileResult *r = go_extract(src, "pkg/alpha/a.go"); + ASSERT_NOT_NULL(r); + ASSERT_NOT_NULL(r->doc_scope); + const char *s = r->doc_scope; + ASSERT(strncmp(s, "go1\n", 4) == 0); + static const char *const want[] = { + "P\talpha\t-", + "I\t-\tfmt", + "I\t-\tio", + "I\tal\texample.com/demo/pkg/alpha", + "I\tbad\t?", /* a path this scan cannot record: the qualifier still exists */ + /* values: the first name of a constant spec or of a parenthesis-free + * var has a node (and keeps it: w 1); the others have none (w -) */ + "V\tMax\t1", + "V\tKindA\t1", + "V\tKindB\t-", + "V\tErr\t1", + "V\tBlockVar\t-", + /* types; fields (the first name of a declaration has a node, embedded + * fields and later names have none); interface methods */ + "T\tWidget\ts\t-\t1", + "f\tWidget\tName\t1", + "f\tWidget\tSize\t0", + "f\tWidget\tReader\t0", + "f\tWidget\tKind\t0", + "T\tRunner\ti\t-\t1", + "m\tRunner\tRun", + "T\tKind\to\t-\t1", + "T\tAlias\ta\tal.Widget\t1", + "T\tSame\ta\tKind\t1", + "T\tList\ts\t-\t1", + /* functions and methods; two methods named Run compete for one node + * and the later one keeps it */ + "F\tNew\t1", + "F\tMap\t1", + "M\tWidget\tRun\t0", + "M\tKind\tRun\t1", + "M\tList\tPush\t1", + "M\tlower\tExported\t1", + /* own type parameters, by the declaration's first and last line */ + "g\t36\t36\tT,U", + "g\t42\t42\tA,B", + "g\t44\t44\tK,V", + }; + for (size_t i = 0; i < GO_COUNT(want); i++) { + if (!go_has_rec(s, want[i])) { + printf(" no record \"%s\" in\n%s\n", want[i], s); + FAIL("scope record"); + } + } + ASSERT_NULL(strstr(s, "example.com/dot")); /* a dot import is no qualifier */ + ASSERT_NULL(strstr(s, "example.com/blank")); /* nor a blank one */ + ASSERT_NULL(strstr(s, "hidden")); + ASSERT_NULL(strstr(s, "secret")); + ASSERT_NULL(strstr(s, "stop")); + ASSERT_NULL(strstr(s, "\tlower\tField")); /* unexported type: nothing to name */ + ASSERT_NULL(strstr(s, "helper")); + /* records are sorted: the blob does not depend on declaration order */ + for (const char *p = strchr(s, '\n'); p && p[1];) { + const char *next = strchr(p + 1, '\n'); + if (next && next[1]) { + const char *after = strchr(next + 1, '\n'); + size_t la = (size_t)(next - p - 1); + size_t lb = after ? (size_t)(after - next - 1) : strlen(next + 1); + int c = memcmp(p + 1, next + 1, la < lb ? la : lb); + ASSERT(c < 0 || (c == 0 && la < lb)); + } + p = next; + } + /* the persisted scope has no line number: the g records are gone */ + char *portable = cbm_doclink_portable_scope(s); + ASSERT_NOT_NULL(portable); + ASSERT_NULL(strstr(portable, "\ng\t")); + ASSERT_TRUE(go_has_rec(portable, "M\tKind\tRun\t1")); + ASSERT(strncmp(portable, "go1\n", 4) == 0); + cbm_free(CBM_MEM_CLASS_OTHER, portable); + cbm_free_result(r); + + /* moving declarations changes nothing unless a node changes its owner */ + const char *a = "package p\n\nfunc (a A) Run() {}\nfunc (b B) Run() {}\nfunc F() {}\n"; + const char *moved = "package p\n\nfunc F() {}\n\n\nfunc (a A) Run() {}\nfunc (b B) Run() {}\n"; + const char *swapped = "package p\n\nfunc (b B) Run() {}\nfunc (a A) Run() {}\nfunc F() {}\n"; + CBMFileResult *ra = go_extract(a, "p/x.go"); + CBMFileResult *rm = go_extract(moved, "p/x.go"); + CBMFileResult *rs = go_extract(swapped, "p/x.go"); + ASSERT_NOT_NULL(ra); + ASSERT_NOT_NULL(rm); + ASSERT_NOT_NULL(rs); + ASSERT_STR_EQ(ra->doc_scope, rm->doc_scope); + ASSERT(strcmp(ra->doc_scope, rs->doc_scope) != 0); + ASSERT_TRUE(go_has_rec(rs->doc_scope, "M\tA\tRun\t1")); + ASSERT_TRUE(go_has_rec(rs->doc_scope, "M\tB\tRun\t0")); + cbm_free_result(ra); + cbm_free_result(rm); + cbm_free_result(rs); + + /* the node . is one for a function, a type, a value and the + * methods of that name: the last declaration keeps it, whatever its kind, + * and two on one line contest it */ + CBMFileResult *k1 = go_extract("package p\n\nfunc Open() {}\nfunc (t T) Open() {}\n", "p/k.go"); + CBMFileResult *k2 = go_extract("package p\n\nfunc (t T) Open() {}\nfunc Open() {}\n", "p/k.go"); + CBMFileResult *k3 = go_extract("package p\n\ntype Open int\nvar Open = 1\n" + "const Open = 2; func (t T) Open() {}\n", + "p/k.go"); + ASSERT_NOT_NULL(k1); + ASSERT_NOT_NULL(k2); + ASSERT_NOT_NULL(k3); + ASSERT_TRUE(go_has_rec(k1->doc_scope, "F\tOpen\t0")); + ASSERT_TRUE(go_has_rec(k1->doc_scope, "M\tT\tOpen\t1")); + ASSERT_TRUE(go_has_rec(k2->doc_scope, "F\tOpen\t1")); + ASSERT_TRUE(go_has_rec(k2->doc_scope, "M\tT\tOpen\t0")); + ASSERT_TRUE(go_has_rec(k3->doc_scope, "T\tOpen\to\t-\t0")); + ASSERT_TRUE(go_has_rec(k3->doc_scope, "V\tOpen\t0")); + ASSERT_TRUE(go_has_rec(k3->doc_scope, "V\tOpen\tx")); + ASSERT_TRUE(go_has_rec(k3->doc_scope, "M\tT\tOpen\tx")); + cbm_free_result(k1); + cbm_free_result(k2); + cbm_free_result(k3); + + /* a build-ignored file, a build-constrained file and a file with a parse + * error say so */ + CBMFileResult *ig = + go_extract("//go:build ignore\n\npackage main\n\nfunc Gen() {}\n", "p/gen.go"); + ASSERT_NOT_NULL(ig); + ASSERT_TRUE(go_has_rec(ig->doc_scope, "P\tmain\ti")); + cbm_free_result(ig); + static const char *const constrained[] = { + "//go:build linux && !cgo\n\npackage p\n", + "// Copyright.\n\n// +build windows\n\npackage p\n", + "//+build darwin\n\npackage p\n", + }; + for (size_t i = 0; i < GO_COUNT(constrained); i++) { + CBMFileResult *cr = go_extract(constrained[i], "p/c.go"); + ASSERT_NOT_NULL(cr); + ASSERT_STR_EQ(cr->doc_scope, "go1\nP\tp\tc"); + cbm_free_result(cr); + } + static const char *const unconstrained[] = { + "package p\n\n//go:build linux\n", /* after the package clause: a comment */ + "//go:buildx linux\n\npackage p\n", /* no build line */ + "/* //go:build linux */\n\npackage p\n", /* inside a block comment */ + }; + for (size_t i = 0; i < GO_COUNT(unconstrained); i++) { + CBMFileResult *fr = go_extract(unconstrained[i], "p/f.go"); + ASSERT_NOT_NULL(fr); + ASSERT_STR_EQ(fr->doc_scope, "go1\nP\tp\t-"); + cbm_free_result(fr); + } + CBMFileResult *bad = go_extract("package q\n\nfunc Broken( {\n\tx := \n}\n\nfunc After() {}\n" + "type T struct{}\nfunc (t T) M() {}\n", + "q/broken.go"); + ASSERT_NOT_NULL(bad); + ASSERT_TRUE(go_has_rec(bad->doc_scope, "P\tq\te")); + /* what the extractor still finds after the error, the scan finds too */ + for (int i = 0; i < bad->defs.count; i++) { + const CBMDefinition *d = &bad->defs.items[i]; + if (!d->name || d->name[0] < 'A' || d->name[0] > 'Z' || !d->label || + strcmp(d->label, "Module") == 0) { + continue; + } + char needle[128]; + snprintf(needle, sizeof(needle), "\t%s", d->name); + const char *hit = strstr(bad->doc_scope, needle); + char after = hit ? hit[strlen(needle)] : 'x'; + if (after != '\t' && after != '\n' && after != '\0') { + printf(" %s %s has a definition but no scope record\n", d->label, d->name); + FAIL("scope of a file with a parse error"); + } + } + cbm_free_result(bad); + + /* go.mod: the module path */ + const char *mod = "module example.com/demo\n\ngo 1.22\n\nrequire example.com/dep v1.0.0\n"; + CBMFileResult *gm = + cbm_extract_file(mod, (int)strlen(mod), CBM_LANG_GOMOD, "p", "go.mod", 0, NULL, NULL); + ASSERT_NOT_NULL(gm); + ASSERT_NOT_NULL(gm->doc_scope); + ASSERT_STR_EQ(gm->doc_scope, "go1\nG\texample.com/demo"); + cbm_free_result(gm); + PASS(); +} + +/* ── the resolver fixture ────────────────────────────────────────── */ + +static const char GO_ALPHA_A[] = + "package alpha\n" + "\n" + "import (\n" + "\t\"fmt\"\n" + "\n" + "\t\"example.com/demo/alpha/testdata/fixture\"\n" + "\t\"example.com/demo/beta\"\n" + "\tgam \"example.com/demo/gamma\"\n" + "\t\"example.com/demo/realname\"\n" + "\t\"example.com/demo/sub/deep\"\n" + "\t\"example.com/dep/ext\"\n" + "\t\"example.com/other/pkg\"\n" + "\tbad \"a\\tb\"\n" + ")\n" + "\n" + "// Widget is the main type. See [New], [Widget.Run], [Runner.Run], [Max],\n" + "// [Default] and [Block].\n" + "type Widget struct {\n" + "\t// Name names it; its size is [Widget.Size].\n" + "\tName string\n" + "\tSize, Weight int\n" + "\tbeta.Embedded\n" + "}\n" + "\n" + "// New makes a [Widget] through [beta.Make] and [gam.Fn]; see [other.Real],\n" + "// [deep.Deep], [pkg.Far], [beta.Thing.Do], [beta.Thing.Field],\n" + "// [beta.Thing.Second], [beta.Doer.Act], [beta.Gr\xC3\xB6\xC3\x9F" + "e], [beta.\xC3\x9C" + "ber], [beta] and [bad.T].\n" + "func New() *Widget { return nil }\n" + "\n" + "// Run runs. Plain text: [Make], [Thing], [realname.Real], [gamma.Fn],\n" + "// [Widget.Promoted], [Widget.Depth], [Widget.Unknown], [nosuch.Name], [Nope], " + "[beta.Fehlt_\xC3\x9F].\n" + "// Rows: [beta.Nope], [beta.Thing.Nope], [beta.InBlock], [ext.Thing],\n" + "// [fmt.Println], [encoding/json.Marshal], [io], [example.com/demo/nodir.X]\n" + "// and [example.com/demo/nested/pkg.Far]. Fields: [Widget.Size], [Widget.Weight], " + "[Widget.Embedded].\n" + "func (w *Widget) Run() {}\n" + "\n" + "// Runner runs.\n" + "type Runner interface {\n" + "\t// Run runs; see [Widget].\n" + "\tRun() error\n" + "}\n" + "\n" + "// Max is the maximum.\n" + "const Max = 10\n" + "\n" + "// Default is the default; tests have [Helper].\n" + "var Default = New()\n" + "\n" + "var (\n" + "\t// Block is in a block.\n" + "\tBlock = 1\n" + ")\n" + "\n" + "// T is a package-level type.\n" + "type T struct{}\n" + "\n" + "// Map shadows: [T] is its own type parameter, [Widget] is not.\n" + "func Map[T any](x T) {}\n" + "\n" + "// Plain has no type parameter: [T] is the type.\n" + "func Plain() {}\n" + "\n" + "// Spec follows the [spec].\n" + "//\n" + "// [spec]: https://example.com/spec\n" + "func Spec() {}\n" + "\n" + "// Box holds a value.\n" + "type Box[T any] struct {\n" + "\t// Value is of type [T], the type parameter of Box; [Widget] is a type.\n" + "\tValue T\n" + "}\n" + "\n" + "// Fixture names [fixture.Data], which is test data.\n" + "func Fixture() {}\n"; + +static const char GO_ALPHA_B[] = + "package alpha\n" + "\n" + "// Sibling imports nothing: [beta.Thing] and [gam.Fn] are plain text here,\n" + "// [Widget] is a link.\n" + "func Sibling() {}\n"; + +static const char GO_ALPHA_C[] = "package alpha\n" + "\n" + "// A is a.\n" + "type A struct{}\n" + "\n" + "// B is b.\n" + "type B struct{}\n" + "\n" + "// Close closes A; see [Target].\n" + "func (A) Close() {}\n" + "\n" + "// Close closes B; see [Target].\n" + "func (B) Close() {}\n" + "\n" + "// Target is a target; see [A.Close] and [B.Close].\n" + "func Target() {}\n" + "\n" + "// Open is a type that loses its node name to the method below.\n" + "type Open struct{}\n" + "\n" + "// Open opens: a method named like the type.\n" + "func (B) Open() {}\n" + "\n" + "// UsesOpen names [Open] and [B.Open].\n" + "func UsesOpen() {}\n"; + +static const char GO_ALPHA_D[] = "package alpha\n" + "\n" + "import \"example.com/demo/beta\"\n" + "\n" + "// Alias is beta's Thing.\n" + "type Alias = beta.Thing\n" + "\n" + "// Same is Alias again.\n" + "type Same = Alias\n" + "\n" + "// Bare names [Alias.Do]: plain text without a package.\n" + "func Bare() {}\n" + "\n" + "type L0 = L1\n" + "type L1 = L2\n" + "type L2 = L3\n" + "type L3 = L4\n" + "type L4 = L5\n" + "type L5 = L6\n" + "type L6 = L7\n" + "type L7 = L8\n" + "type L8 = L9\n" + "type L9 = Deep10\n" + "\n" + "// Deep10 ends the chain.\n" + "type Deep10 struct{}\n" + "\n" + "// End is its method.\n" + "func (Deep10) End() {}\n" + "\n" + "type CycA = CycB\n" + "type CycB = CycA\n"; + +static const char GO_BETA[] = "package beta\n" + "\n" + "// Thing is a thing.\n" + "type Thing struct {\n" + "\t// Field is a field.\n" + "\tField int\n" + "\tFirst, Second int\n" + "}\n" + "\n" + "// Do does.\n" + "func (t Thing) Do() {}\n" + "\n" + "// Make makes.\n" + "func Make() Thing { return Thing{} }\n" + "\n" + "// Embedded is embedded elsewhere.\n" + "type Embedded struct {\n" + "\t// Depth is a field of Embedded, promoted where it is embedded.\n" + "\tDepth int\n" + "}\n" + "\n" + "// Promoted is promoted through embedding.\n" + "func (Embedded) Promoted() {}\n" + "\n" + "// Gr\xC3\xB6\xC3\x9F" + "e has letters beyond ASCII.\n" + "type Gr\xC3\xB6\xC3\x9F" + "e struct{}\n" + "\n" + "// \xC3\x9C" + "ber starts with a capital letter beyond ASCII: exported.\n" + "type \xC3\x9C" + "ber struct{}\n" + "\n" + "var (\n" + "\t// InBlock has no node.\n" + "\tInBlock = 1\n" + ")\n" + "\n" + "// Doer does.\n" + "type Doer interface {\n" + "\t// Act acts.\n" + "\tAct()\n" + "}\n"; + +static const char GO_GAMMA_K[] = + "package gamma\n" + "\n" + "import \"example.com/demo/alpha\"\n" + "\n" + "// ViaAlias reaches [alpha.Alias.Do] and [alpha.Same.Field] through aliases;\n" + "// [alpha.Alias] is the alias itself, [alpha.Alias.Nope] is nothing,\n" + "// [alpha.Widget.Promoted] is promoted and not Widget's own,\n" + "// [alpha.Widget.Name] is a field, [alpha.Widget.Weight] a field without a\n" + "// node, [alpha.L3.End] is seven aliases away, [alpha.L0.End] ten and\n" + "// [alpha.CycA.End] never ends. [alpha.Helper] is test code.\n" + "func ViaAlias() {}\n"; + +/* The module example.com/demo with a nested module of its own path + * (sub/), a nested module of another path (nested/) and the packages the + * lookup tests need. */ +static void go_write_lookup_fixture(const char *repo) { + go_w(repo, "go.mod", "module example.com/demo\n\ngo 1.22\n"); + go_w(repo, "root.go", + "// Package demo is the root. See [alpha.Widget].\n" + "package demo\n" + "\n" + "import \"example.com/demo/alpha\"\n" + "\n" + "// Root names [example.com/demo], [alpha], [example.com/demo/mixed.F] and\n" + "// [example.com/demo/fmt.Println].\n" + "func Root() {}\n"); + go_w(repo, "alpha/doc.go", + "// Package alpha is documented here. It builds a [Widget]; printing is\n" + "// [fmt.Println] and [nosuch.Thing] is nothing.\n" + "// [beta.Thing] is no link here: this file imports nothing, a.go imports beta.\n" + "package alpha\n"); + go_w(repo, "alpha/a.go", GO_ALPHA_A); + go_w(repo, "alpha/b.go", GO_ALPHA_B); + go_w(repo, "alpha/c.go", GO_ALPHA_C); + go_w(repo, "alpha/d.go", GO_ALPHA_D); + go_w(repo, "alpha/a_test.go", + "package alpha\n" + "\n" + "// Helper exists only in tests.\n" + "type Helper struct{}\n" + "\n" + "// TestWidget uses [Helper] and [Widget].\n" + "func TestWidget() {}\n"); + go_w(repo, "alpha/x_test.go", + "package alpha_test\n" + "\n" + "import \"example.com/demo/alpha\"\n" + "\n" + "// Example uses [alpha.Widget]; [Widget] alone is plain text in this package.\n" + "func Example() {}\n"); + go_w(repo, "alpha/testdata/fixture/f.go", + "package fixture\n\n// Data is test data; it may name [Data] itself and [Twin].\n" + "var Data = 1\n\n// Twin is another value.\nvar Twin = 2\n"); + go_w(repo, "beta/b.go", GO_BETA); + go_w(repo, "gamma/g.go", "package gamma\n\n// Fn is a function.\nfunc Fn() {}\n"); + go_w(repo, "gamma/h.go", + "package gamma\n" + "\n" + "import \"example.com/demo/fmt\"\n" + "\n" + "// Local names the repository's own [fmt.Println].\n" + "func Local() {}\n"); + go_w(repo, "gamma/k.go", GO_GAMMA_K); + go_w(repo, "realname/r.go", "package other\n\n// Real is real.\nfunc Real() {}\n"); + go_w(repo, "fmt/f.go", + "package fmt\n\n// Println is the repository's own, not the standard library's.\n" + "func Println() {}\n"); + go_w(repo, "log/l.go", + "package log\n" + "\n" + "// Printf prints.\n" + "func Printf() {}\n" + "\n" + "// Use names its own package's [log.Printf], not the standard library's.\n" + "func Use() {}\n"); + go_w(repo, "strings/s.go", + "package strings\n" + "\n" + "import \"strings\"\n" + "\n" + "// Join joins.\n" + "func Join() {}\n" + "\n" + "// Use imports the standard library's strings: [strings.Join] is that one.\n" + "func Use() {}\n"); + go_w(repo, "sub/go.mod", "module example.com/demo/sub\n"); + go_w(repo, "sub/deep/d.go", "package deep\n\n// Deep is deep.\nfunc Deep() {}\n"); + go_w(repo, "nested/go.mod", "module example.com/other\n"); + go_w(repo, "nested/pkg/p.go", "package pkg\n\n// Far is far.\nfunc Far() {}\n"); + /* one import name, two packages: each file's own import decides */ + go_w(repo, "same/a.go", + "package same\n\nimport x \"example.com/demo/beta\"\n\n// A names [x.Thing].\nfunc A() " + "{}\n"); + go_w( + repo, "same/b.go", + "package same\n\nimport x \"example.com/demo/gamma\"\n\n// B names [x.Fn].\nfunc B() {}\n"); + go_w(repo, "same/doc.go", + "// Package same: [x.Thing] is no link in a file without imports.\npackage same\n"); + go_w(repo, "mixed/a.go", "package p1\n\n// F is p1's.\nfunc F() {}\n"); + go_w(repo, "mixed/b.go", "package p2\n\n// F is p2's.\nfunc F() {}\n"); + /* stand-alone programs in one directory: no package */ + go_w(repo, "progs/one.go", + "package main\n\n// Config is one program's.\ntype Config struct{}\n\n" + "// Load loads a [Config].\nfunc Load() {}\n"); + go_w(repo, "progs/two.go", + "package main\n\n// Config is another program's.\ntype Config struct{}\n\n" + "// Save saves a [Config].\nfunc Save() {}\n"); + go_w(repo, "progs/three.go", + "package main\n\n// Print prints a [Config], whichever.\nfunc Print() {}\n"); + /* build variants of one function */ + go_w(repo, "plat/impl_linux.go", "package plat\n\n// Open opens on Linux.\nfunc Open() {}\n"); + go_w(repo, "plat/impl_windows.go", + "package plat\n\n// Open opens on Windows.\nfunc Open() {}\n\n" + "// Helper calls [Open].\nfunc Helper() {}\n"); + go_w(repo, "plat/tagged_a.go", + "//go:build a\n\npackage plat\n\n// Tagged is one variant.\nfunc Tagged() {}\n"); + go_w(repo, "plat/tagged_b.go", + "//go:build !a\n\npackage plat\n\n// Tagged is the other variant.\nfunc Tagged() {}\n"); + go_w(repo, "plat/use.go", "package plat\n\n// Use calls [Open] and [Tagged].\nfunc Use() {}\n"); + /* a test file's variant keeps the node */ + go_w(repo, "tw/aa_test.go", + "//go:build integration\n\npackage tw\n\n// Shared is the test double.\n" + "func Shared() {}\n"); + go_w(repo, "tw/zz.go", + "package tw\n\n// Shared is the product one.\nfunc Shared() {}\n\n" + "// Caller calls [Shared].\nfunc Caller() {}\n"); + /* one name, two kinds */ + go_w(repo, "amb/x.go", "package amb\n\n// Dual is a function here.\nfunc Dual() {}\n"); + go_w(repo, "amb/y.go", + "//go:build other\n\npackage amb\n\n// Dual is a type there.\ntype Dual struct{}\n"); + go_w(repo, "amb/z.go", "package amb\n\n// Z names [Dual].\nfunc Z() {}\n"); + /* a reference longer than a reference can be */ + char *big = malloc(2048); + size_t w = (size_t)snprintf(big, 2048, "package long\n\n// L names ["); + memset(big + w, 'A', CBM_DOCLINK_GO_REF_MAX + 88); + w += CBM_DOCLINK_GO_REF_MAX + 88; + snprintf(big + w, 2048 - w, + "] and [Other].\nfunc L() {}\n\n// Other is other.\nfunc Other() {}\n"); + go_w(repo, "long/l.go", big); + free(big); +} + +/* Write the lookup fixture below `tmp` and index it into `db`. */ +static int go_index_lookup(char *tmp, size_t tmp_cap, char *db, size_t db_cap) { + snprintf(tmp, tmp_cap, "/tmp/cbm_dm_go_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + return -1; + } + char repo[400]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + go_write_lookup_fixture(repo); + snprintf(db, db_cap, "%s/go.db", tmp); + return dm_index(repo, db, NULL); +} + +/* ── resolver: every reference form, in its first lookup level ───── */ + +TEST(doc_mentions_go_forms) { + char tmp[256]; + char db[512]; + ASSERT_EQ(go_index_lookup(tmp, sizeof(tmp), db, sizeof(db)), 0); + static const go_want_edge_t edges[] = { + /* [Name], [Recv.Method], an interface's method, a constant, a variable */ + {"alpha.Widget", "alpha.New", "unique"}, + {"alpha.Widget", "alpha.Run", "unique"}, + {"alpha.Widget", "alpha.Runner.Run", "unique"}, + {"alpha.Widget", "alpha.Max", "unique"}, + {"alpha.Widget", "alpha.Default", "unique"}, + {"alpha.New", "alpha.Widget", "unique"}, + /* an interface method and a field are sources */ + {"alpha.Runner.Run", "alpha.Widget", "unique"}, + /* [pkg.Name]: the file's import, its renamed import, the real package + * name of an import whose directory is called otherwise, a nested + * module, a module with another path */ + {"alpha.New", "beta.Make", "exact"}, + {"alpha.New", "gamma.Fn", "exact"}, + {"alpha.New", "realname.Real", "exact"}, + {"alpha.New", "sub.deep.Deep", "exact"}, + {"alpha.New", "nested.pkg.Far", "exact"}, + /* [pkg.Recv.Name]: a method, a field, an interface method */ + {"alpha.New", "beta.Do", "exact"}, + {"alpha.New", "beta.Thing.Field", "exact"}, + {"alpha.New", "beta.Doer.Act", "exact"}, + /* an identifier with letters beyond ASCII */ + {"alpha.New", + "beta.Gr\xC3\xB6\xC3\x9F" + "e", + "exact"}, + {"alpha.New", + "beta.\xC3\x9C" + "ber", + "exact"}, + /* [pkg]: the package's directory node */ + {"alpha.New", "beta", "exact"}, + /* an import path */ + {"Root", "fmt.Println", "exact"}, + {"Root", "alpha", "exact"}, + /* test code may name test code; an external test package imports */ + {"alpha.TestWidget", "alpha.Helper", "unique"}, + {"alpha.TestWidget", "alpha.Widget", "unique"}, + {"alpha.Example", "alpha.Widget", "exact"}, + {"fixture.Data", "fixture.Twin", "unique"}, /* inside a testdata directory */ + }; + ASSERT_EQ(go_bad_edges(db, edges, GO_COUNT(edges)), 0); + /* the edge's properties, in full */ + char props[512]; + int n = 0; + dm_edge(db, "alpha.New", "beta.Make", props, sizeof(props), &n); + ASSERT_EQ(n, 1); + ASSERT_STR_EQ(props, "{\"via\":\"doc_comment\",\"syntax\":\"doc_link\",\"tier\":\"exact\"," + "\"line\":25,\"count\":1}"); + /* 12 targets; [bad.T] and [beta.Thing.Second] are rows */ + ASSERT_EQ(dm_mentions_from(db, "alpha.New"), 12); + /* a [Text] with a link definition is the href family: a row, never an edge */ + char reason[64]; + char syntax[64]; + dm_row(db, "alpha/a.go", "https://example.com/spec", reason, sizeof(reason), syntax, + sizeof(syntax)); + ASSERT_STR_EQ(reason, "external"); + ASSERT_STR_EQ(syntax, "href"); + ASSERT_EQ(dm_mentions_from(db, "alpha.Spec"), 0); + dm_unlink_db(db); + th_rmtree(tmp); + PASS(); +} + +/* ── resolver: text go/doc renders as text is neither edge nor row ── */ + +TEST(doc_mentions_go_plain_text) { + char tmp[256]; + char db[512]; + ASSERT_EQ(go_index_lookup(tmp, sizeof(tmp), db, sizeof(db)), 0); + const char *a = "alpha/a.go"; + static const go_want_row_t rows[] = { + /* a name the package does not declare, although an imported package + * or a sibling directory does */ + {"alpha/a.go", "Make", ""}, + {"alpha/a.go", "Thing", ""}, + {"alpha/a.go", "Nope", ""}, + /* a qualifier that is no import name of the file: the directory name + * of a package called otherwise, the path element of a renamed + * import, a word */ + {"alpha/a.go", "realname.Real", ""}, + {"alpha/a.go", "gamma.Fn", ""}, + {"alpha/a.go", "nosuch.Name", ""}, + /* members the type does not declare itself: a method and a field + * promoted through an embedded type, a name nobody declares */ + {"alpha/a.go", "Widget.Promoted", ""}, + {"alpha/a.go", "Widget.Depth", ""}, + {"alpha/a.go", "Widget.Unknown", ""}, + /* a qualified name with a byte this scan cannot classify */ + {"alpha/a.go", "beta.Fehlt_\xC3\x9F", ""}, + /* the file's own imports only: a sibling's import is no qualifier */ + {"alpha/b.go", "beta.Thing", ""}, + {"alpha/b.go", "gam.Fn", ""}, + /* a member through an alias, without a package */ + {"alpha/d.go", "Alias.Do", ""}, + /* another package clause in the same directory is another package */ + {"alpha/x_test.go", "Widget", ""}, + {"alpha/doc.go", "nosuch.Thing", ""}, + }; + ASSERT_EQ(go_bad_rows(db, rows, GO_COUNT(rows)), 0); + /* and none of them is an edge: Run's doc holds rows, plain text and one + * link, to the field Widget.Size */ + ASSERT_EQ(dm_mentions_from(db, "alpha.Run"), 1); + ASSERT_EQ(dm_mentions_from(db, "alpha.Bare"), 0); + static const go_want_edge_t edges[] = { + {"alpha.Sibling", "alpha.Widget", "unique"}, + {"alpha.Sibling", "beta.Thing", ""}, + {"alpha.Sibling", "gamma.Fn", ""}, + {"alpha.Example", "alpha.Widget", "exact"}, + }; + ASSERT_EQ(go_bad_edges(db, edges, GO_COUNT(edges)), 0); + ASSERT_EQ(dm_mentions_from(db, "alpha.Sibling"), 1); + ASSERT_STR_EQ(go_reason(db, a, "beta.Nope"), "missing"); /* the fixture is the one meant */ + dm_unlink_db(db); + th_rmtree(tmp); + PASS(); +} + +/* ── resolver: struct fields ─────────────────────────────────────── */ + +/* [Type.Field] binds the field written in that type's own struct declaration, + * with or without a package. (Without one, go/doc itself is believed to + * render it as plain text: a deliberate difference.) Nothing promoted, and + * nothing for a name the type does not declare. */ +TEST(doc_mentions_go_struct_fields) { + char tmp[256]; + char db[512]; + ASSERT_EQ(go_index_lookup(tmp, sizeof(tmp), db, sizeof(db)), 0); + static const go_want_edge_t edges[] = { + /* the same package: a field named from a method's doc and from a + * sibling field's doc */ + {"alpha.Run", "alpha.Widget.Size", "exact"}, + {"alpha.Widget.Name", "alpha.Widget.Size", "exact"}, + /* another package, by its import */ + {"alpha.New", "beta.Thing.Field", "exact"}, + {"gamma.ViaAlias", "alpha.Widget.Name", "exact"}, + /* Depth is a field of beta.Embedded, which Widget embeds: it is no + * field of Widget */ + {"alpha.Run", "beta.Embedded.Depth", ""}, + }; + ASSERT_EQ(go_bad_edges(db, edges, GO_COUNT(edges)), 0); + /* the promoted field exists as a node: the test above is not vacuous */ + ASSERT_EQ(dm_count(db, + "SELECT COUNT(*) FROM nodes WHERE label = 'Field' AND qualified_name LIKE " + "'%.beta.Embedded.Depth'"), + 1); + static const go_want_row_t rows[] = { + {"alpha/a.go", "Widget.Size", ""}, /* an edge, no row */ + {"alpha/a.go", "Widget.Depth", ""}, /* promoted: plain text */ + {"alpha/a.go", "Widget.Promoted", ""}, /* a promoted method: plain text */ + {"alpha/a.go", "Widget.Unknown", ""}, /* nobody's: plain text */ + /* a field the graph has no node for is a gap, not plain text: the + * second name of one declaration, and the embedded field itself */ + {"alpha/a.go", "Widget.Weight", "graph_gap"}, + {"alpha/a.go", "Widget.Embedded", "graph_gap"}, + {"gamma/k.go", "alpha.Widget.Promoted", "missing"}, /* with a package: a broken link */ + }; + ASSERT_EQ(go_bad_rows(db, rows, GO_COUNT(rows)), 0); + ASSERT_EQ(dm_mentions_from(db, "alpha.Widget.Name"), 1); + dm_unlink_db(db); + th_rmtree(tmp); + PASS(); +} + +/* ── resolver: the package of a qualifier is the FILE's import ───── */ + +/* The limit of this resolver against go/doc: a qualifier is looked up in the + * imports of the documented file only, not in the union of the imports of + * the package's files. A package comment in a file without imports (the + * usual doc.go) therefore cannot name what only a sibling file imports: no + * edge and no row. The other side of the same rule: where two files import + * different packages under one name, go/doc links neither, and here each + * file's links go through its own import. */ +TEST(doc_mentions_go_file_only_package_lookup) { + char tmp[256]; + char db[512]; + ASSERT_EQ(go_index_lookup(tmp, sizeof(tmp), db, sizeof(db)), 0); + static const go_want_row_t rows[] = { + {"alpha/doc.go", "beta.Thing", ""}, /* a.go imports beta, doc.go nothing */ + {"same/doc.go", "x.Thing", ""}, + {"alpha/b.go", "beta.Thing", ""}, + {"alpha/b.go", "gam.Fn", ""}, + }; + ASSERT_EQ(go_bad_rows(db, rows, GO_COUNT(rows)), 0); + static const go_want_edge_t edges[] = { + {"alpha.doc.go.__file__", "beta.Thing", ""}, + {"same.doc.go.__file__", "beta.Thing", ""}, + {"alpha.Sibling", "beta.Thing", ""}, + /* the file that does import it links it */ + {"alpha.New", "beta.Make", "exact"}, + /* one name, two packages: the file's own import decides */ + {"same.A", "beta.Thing", "exact"}, + {"same.B", "gamma.Fn", "exact"}, + {"same.A", "gamma.Fn", ""}, + {"same.B", "beta.Thing", ""}, + }; + ASSERT_EQ(go_bad_edges(db, edges, GO_COUNT(edges)), 0); + /* the package comment's one link is the one to its own package's type */ + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id " + "WHERE e.type = 'MENTIONS' AND s.label = 'File' " + "AND s.file_path IN ('alpha/doc.go', 'same/doc.go')"), + 1); + dm_unlink_db(db); + th_rmtree(tmp); + PASS(); +} + +/* ── resolver: the local rules ───────────────────────────────────── */ + +TEST(doc_mentions_go_local_rules) { + char tmp[256]; + char db[512]; + ASSERT_EQ(go_index_lookup(tmp, sizeof(tmp), db, sizeof(db)), 0); + static const go_want_edge_t edges[] = { + /* the function's own type parameter shadows the package's type T */ + {"alpha.Map", "alpha.T", ""}, + {"alpha.Map", "alpha.Widget", "unique"}, + /* a function without that type parameter means the type */ + {"alpha.Plain", "alpha.T", "unique"}, + /* a generic type's parameter is in scope of its fields' docs too */ + {"alpha.Box.Value", "alpha.T", ""}, + {"alpha.Box.Value", "alpha.Widget", "unique"}, + }; + ASSERT_EQ(go_bad_edges(db, edges, GO_COUNT(edges)), 0); + ASSERT_STR_EQ(go_reason(db, "alpha/a.go", "T"), ""); + ASSERT_EQ(dm_mentions_from(db, "alpha.Map"), 1); + ASSERT_EQ(dm_mentions_from(db, "alpha.Box.Value"), 1); + dm_unlink_db(db); + th_rmtree(tmp); + PASS(); +} + +/* ── resolver: one test per level boundary of the package lookup ─── */ + +TEST(doc_mentions_go_lookup_order) { + char tmp[256]; + char db[512]; + ASSERT_EQ(go_index_lookup(tmp, sizeof(tmp), db, sizeof(db)), 0); + /* 1. the file's import beats the package's own name: package strings + * imports the standard library's strings */ + ASSERT_STR_EQ(go_reason(db, "strings/s.go", "strings.Join"), "external"); + ASSERT_STR_EQ(go_tier(db, "strings.Use", "strings.Join"), ""); + /* 2. the package's own name beats the standard-library list: package log + * without an import of log */ + ASSERT_STR_EQ(go_tier(db, "log.Use", "log.Printf"), "exact"); + ASSERT_STR_EQ(go_reason(db, "log/l.go", "log.Printf"), ""); + /* 3. the standard-library list is the last level: [io] without an import */ + ASSERT_STR_EQ(go_reason(db, "alpha/a.go", "io"), "external"); + /* 4. an import decides by its PATH: fmt is the standard library's in a + * file that imports "fmt", the repository's own in a file that imports + * example.com/demo/fmt */ + ASSERT_STR_EQ(go_reason(db, "alpha/a.go", "fmt.Println"), "external"); + ASSERT_STR_EQ(go_tier(db, "alpha.Run", "fmt.Println"), ""); + ASSERT_STR_EQ(go_tier(db, "gamma.Local", "fmt.Println"), "exact"); + /* 5. the longest module path decides: a directory inside the root module + * that belongs to a nested module of another path is not reachable through + * the root module's path */ + ASSERT_STR_EQ(go_reason(db, "alpha/a.go", "example.com/demo/nested/pkg.Far"), "missing"); + ASSERT_STR_EQ(go_tier(db, "alpha.New", "nested.pkg.Far"), "exact"); + ASSERT_STR_EQ(go_tier(db, "alpha.New", "sub.deep.Deep"), "exact"); + /* 6. the bare name stops at the package: beta.Make is imported and still + * no [Make] */ + ASSERT_STR_EQ(go_tier(db, "alpha.Run", "beta.Make"), ""); + /* 7. members stop at the type: nothing promoted through embedding */ + ASSERT_STR_EQ(go_reason(db, "gamma/k.go", "alpha.Widget.Promoted"), "missing"); + ASSERT_STR_EQ(go_tier(db, "gamma.ViaAlias", "beta.Promoted"), ""); + /* 8. an alias is followed for a member written with its package, to the + * type that declares the member */ + static const go_want_edge_t alias[] = { + {"gamma.ViaAlias", "beta.Do", "exact"}, + {"gamma.ViaAlias", "beta.Thing.Field", "exact"}, + {"gamma.ViaAlias", "alpha.Alias", "exact"}, + {"gamma.ViaAlias", "alpha.End", "exact"}, /* seven aliases away */ + {"gamma.ViaAlias", "alpha.Widget.Name", "exact"}, + }; + ASSERT_EQ(go_bad_edges(db, alias, GO_COUNT(alias)), 0); + ASSERT_STR_EQ(go_reason(db, "gamma/k.go", "alpha.Alias.Nope"), "missing"); + dm_unlink_db(db); + th_rmtree(tmp); + PASS(); +} + +/* ── resolver: one test per reason ───────────────────────────────── */ + +TEST(doc_mentions_go_reasons) { + char tmp[256]; + char db[512]; + ASSERT_EQ(go_index_lookup(tmp, sizeof(tmp), db, sizeof(db)), 0); + char long_raw[CBM_DOCLINK_GO_REF_MAX + 2]; + memset(long_raw, 'A', CBM_DOCLINK_GO_REF_MAX + 1); + long_raw[CBM_DOCLINK_GO_REF_MAX + 1] = '\0'; + const go_want_row_t rows[] = { + /* external: the standard library by import, a dependency by import, + * the standard library by path */ + {"alpha/a.go", "fmt.Println", "external"}, + {"alpha/a.go", "ext.Thing", "external"}, + {"alpha/a.go", "encoding/json.Marshal", "external"}, + /* missing: a repository package that does not declare the name or the + * member; a path inside a module with no package there */ + {"alpha/a.go", "beta.Nope", "missing"}, + {"alpha/a.go", "beta.Thing.Nope", "missing"}, + {"alpha/a.go", "example.com/demo/nodir.X", "missing"}, + /* graph_gap: declared, but without a node of its own: a variable of a + * var block, a later name of a field declaration, the root package + * (the Project node is no package node) */ + {"alpha/a.go", "Block", "graph_gap"}, + {"alpha/a.go", "beta.InBlock", "graph_gap"}, + {"alpha/a.go", "beta.Thing.Second", "graph_gap"}, + {"gamma/k.go", "alpha.Widget.Weight", "graph_gap"}, + {"root.go", "example.com/demo", "graph_gap"}, + /* test_only_target: product code naming what only test files declare, + * or what a testdata directory holds */ + {"alpha/a.go", "Helper", "test_only_target"}, + {"gamma/k.go", "alpha.Helper", "test_only_target"}, + {"alpha/a.go", "fixture.Data", "test_only_target"}, + /* ambiguous: one name, a function here and a type there; one + * directory, two packages, and nothing that picks one */ + {"amb/z.go", "Dual", "ambiguous"}, + {"root.go", "example.com/demo/mixed.F", "ambiguous"}, + /* unparseable: an import path the scan could not record, an alias + * chain that is too long or never ends, a reference that is too long + * (never resolved by a prefix) */ + {"alpha/a.go", "bad.T", "unparseable"}, + {"gamma/k.go", "alpha.L0.End", "unparseable"}, + {"gamma/k.go", "alpha.CycA.End", "unparseable"}, + {"long/l.go", long_raw, "unparseable"}, + }; + ASSERT_EQ(go_bad_rows(db, rows, GO_COUNT(rows)), 0); + ASSERT_STR_EQ(go_tier(db, "long.L", "long.Other"), "unique"); + ASSERT_EQ(dm_mentions_from(db, "long.L"), 1); + /* every row of the fixture is a Go doc link or a URL */ + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM doc_link_unresolved WHERE syntax NOT IN " + "('doc_link','href')"), + 0); + dm_unlink_db(db); + th_rmtree(tmp); + PASS(); +} + +/* ── resolver: one node name, several declarations ───────────────── */ + +TEST(doc_mentions_go_node_owner) { + char tmp[256]; + char db[512]; + ASSERT_EQ(go_index_lookup(tmp, sizeof(tmp), db, sizeof(db)), 0); + /* Go method nodes have a flat name: A.Close and B.Close share + * alpha.Close, and the later declaration keeps it */ + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM nodes WHERE qualified_name LIKE '%.alpha.Close'"), + 1); + ASSERT_STR_EQ(go_tier(db, "alpha.Target", "alpha.Close"), "unique"); + ASSERT_STR_EQ(go_reason(db, "alpha/c.go", "A.Close"), "graph_gap"); + ASSERT_STR_EQ(go_reason(db, "alpha/c.go", "B.Close"), ""); + /* the doc of the method that lost the name is not attributed to the one + * that kept it: B.Close's [Target] is the edge, A.Close's a row */ + char props[512]; + int n = 0; + dm_edge(db, "alpha.Close", "alpha.Target", props, sizeof(props), &n); + ASSERT_EQ(n, 1); + ASSERT_NOT_NULL(strstr(props, "\"line\":12,\"count\":1")); + ASSERT_EQ(dm_count(db, "SELECT line FROM doc_link_unresolved WHERE rel_path = 'alpha/c.go' " + "AND raw = 'Target'"), + 9); + /* a type that lost its name to a method of that name is no target, the + * method is */ + ASSERT_STR_EQ(go_reason(db, "alpha/c.go", "Open"), "graph_gap"); + ASSERT_STR_EQ(go_tier(db, "alpha.UsesOpen", "alpha.Open"), "unique"); + ASSERT_EQ(dm_mentions_from(db, "alpha.UsesOpen"), 1); + + /* build variants are one entity: whichever file holds the node */ + static const go_want_edge_t variants[] = { + {"plat.Use", "plat.Open", "unique"}, + {"plat.Use", "plat.Tagged", "unique"}, + {"plat.Helper", "plat.Open", "unique"}, + }; + ASSERT_EQ(go_bad_edges(db, variants, GO_COUNT(variants)), 0); + + /* two unconstrained files that declare one name are no package: the name + * means the file's own declaration, and nothing from elsewhere */ + ASSERT_STR_EQ(go_tier(db, "progs.Load", "progs.Config"), "unique"); + ASSERT_STR_EQ(go_reason(db, "progs/two.go", "Config"), "graph_gap"); + ASSERT_STR_EQ(go_tier(db, "progs.Save", "progs.Config"), ""); + ASSERT_STR_EQ(go_reason(db, "progs/three.go", "Config"), "ambiguous"); + ASSERT_STR_EQ(go_tier(db, "progs.Print", "progs.Config"), ""); + + /* a node kept by a test file's variant is no target for product code */ + ASSERT_STR_EQ(go_reason(db, "tw/zz.go", "Shared"), "graph_gap"); + ASSERT_STR_EQ(go_tier(db, "tw.Caller", "tw.Shared"), ""); + dm_unlink_db(db); + th_rmtree(tmp); + PASS(); +} + +/* ── resolver: the package comment, from the File node ───────────── */ + +TEST(doc_mentions_go_package_comment) { + char tmp[256]; + char db[512]; + ASSERT_EQ(go_index_lookup(tmp, sizeof(tmp), db, sizeof(db)), 0); + /* a reference of the package comment is an edge from the File node */ + char props[512]; + int n = 0; + dm_edge(db, "alpha.doc.go.__file__", "alpha.Widget", props, sizeof(props), &n); + ASSERT_EQ(n, 1); + ASSERT_STR_EQ(props, "{\"via\":\"doc_comment\",\"syntax\":\"doc_link\",\"tier\":\"unique\"," + "\"line\":1,\"count\":1}"); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id " + "WHERE e.type = 'MENTIONS' AND s.label = 'File' " + "AND s.file_path = 'alpha/doc.go'"), + 1); + /* ... or a row with the file's path and the reference's line */ + ASSERT_STR_EQ(go_reason(db, "alpha/doc.go", "fmt.Println"), "external"); + ASSERT_EQ(dm_count(db, "SELECT line FROM doc_link_unresolved WHERE rel_path = 'alpha/doc.go' " + "AND raw = 'fmt.Println'"), + 2); + /* a file with imports: its package comment reads them */ + ASSERT_STR_EQ(go_tier(db, "root.go.__file__", "alpha.Widget"), "exact"); + dm_unlink_db(db); + th_rmtree(tmp); + PASS(); +} + +/* ── resolver: the repository's own standard library ─────────────── */ + +TEST(doc_mentions_go_std_module) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dm_gostd_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[400]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + go_w(repo, "src/go.mod", "module std\n\ngo 1.22\n"); + go_w(repo, "src/fmt/print.go", "package fmt\n\n// Println prints.\nfunc Println() {}\n"); + go_w(repo, "src/os/exec/exec.go", "package exec\n\n// Cmd is a command.\ntype Cmd struct{}\n"); + go_w(repo, "src/log/doc.go", + "// Package log links [os/exec.Cmd] by its path and [fmt.Println] by the\n" + "// standard library's own name.\n" + "package log\n"); + go_w(repo, "src/log/log.go", + "package log\n" + "\n" + "import (\n" + "\t\"fmt\"\n" + "\t\"os/exec\"\n" + ")\n" + "\n" + "// Print uses [fmt.Println], [exec.Cmd] and [fmt.Missing].\n" + "func Print() {}\n"); + char db[512]; + snprintf(db, sizeof(db), "%s/std.db", tmp); + ASSERT_EQ(dm_index(repo, db, NULL), 0); + /* a package of `module std` is imported by its bare directory path */ + static const go_want_edge_t edges[] = { + {"log.Print", "fmt.Println", "exact"}, + {"log.Print", "exec.Cmd", "exact"}, + {"log.doc.go.__file__", "exec.Cmd", "exact"}, + {"log.doc.go.__file__", "fmt.Println", "exact"}, + }; + ASSERT_EQ(go_bad_edges(db, edges, GO_COUNT(edges)), 0); + ASSERT_STR_EQ(go_reason(db, "src/log/log.go", "fmt.Missing"), "missing"); + dm_unlink_db(db); + th_rmtree(tmp); + PASS(); +} + +/* ── the ship gate ───────────────────────────────────────────────── */ + +TEST(doc_mentions_go_ship_gate) { + char tmp[256]; + char db[512]; + cbm_doclink_test_set_ships(CBM_DOCLINK_GO_DOC_LINK, false); + int rc = go_index_lookup(tmp, sizeof(tmp), db, sizeof(db)); + dm_ship_held_families(); + ASSERT_EQ(rc, 0); + /* a resolved doc link is a row with the gate's reason and no edge */ + ASSERT_STR_EQ(go_tier(db, "alpha.New", "beta.Make"), ""); + ASSERT_STR_EQ(go_reason(db, "alpha/a.go", "beta.Make"), "below_bar_tier"); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges WHERE type = 'MENTIONS'"), 0); + /* one that does not resolve keeps its own reason; plain text stays nothing */ + ASSERT_STR_EQ(go_reason(db, "alpha/a.go", "beta.Nope"), "missing"); + ASSERT_STR_EQ(go_reason(db, "alpha/a.go", "Make"), ""); + dm_unlink_db(db); + th_rmtree(tmp); + PASS(); +} + +/* ── incremental == full ─────────────────────────────────────────── */ + +#define GO_INC_A_HEAD "package a\n\nimport \"example.com/inc/b\"\n\n" +#define GO_INC_A_LOCAL "// Local is local.\nfunc Local() {}\n" +static const char GO_INC_A[] = GO_INC_A_HEAD + "// Use names [b.Target], [b.Gone], [b.T.M], [b.W.Name], [b.V.Name] and [Local].\n" + "func Use() {}\n\n" GO_INC_A_LOCAL; +static const char GO_INC_A_DOC_EDIT[] = + GO_INC_A_HEAD "// Use names [b.Target], [b.Gone], [b.T.M], [b.W.Name], [b.V.Name], [Local]\n" + "// and now [b.T].\n" + "func Use() {}\n\n" GO_INC_A_LOCAL; +static const char GO_INC_OTHER[] = "package a\n\n// Other names [b.Target].\nfunc Other() {}\n"; +static const char GO_INC_OTHER_IMPORT[] = + "package a\n\nimport \"example.com/inc/b\"\n\n// Other names [b.Target].\nfunc Other() {}\n"; +static const char GO_INC_DOC[] = "// Package a is documented. See [Local].\npackage a\n"; +static const char GO_INC_DOC_EDIT[] = + "// Package a is documented. See [Local] and [Use].\npackage a\n"; + +#define GO_INC_B_TARGET "// Target is the target.\nfunc Target() {}\n\n" +#define GO_INC_B_TARGET_BODY "// Target is the target.\nfunc Target() {\n\t_ = 1\n}\n\n" +#define GO_INC_B_RENAMED "// Renamed was the target.\nfunc Renamed() {}\n\n" +#define GO_INC_B_GONE "// Gone will go.\nfunc Gone() {}\n\n" +#define GO_INC_B_T "// T is a type.\ntype T struct{}\n\n" +#define GO_INC_B_TM "// M is a method of T.\nfunc (T) M() {}\n" +static const char GO_INC_B[] = "package b\n\n" GO_INC_B_TARGET GO_INC_B_GONE GO_INC_B_T GO_INC_B_TM; +static const char GO_INC_B_BODY[] = + "package b\n\n" GO_INC_B_TARGET_BODY GO_INC_B_GONE GO_INC_B_T GO_INC_B_TM; +static const char GO_INC_B_NO_GONE[] = "package b\n\n" GO_INC_B_TARGET_BODY GO_INC_B_T GO_INC_B_TM; +static const char GO_INC_B_NO_M[] = "package b\n\n" GO_INC_B_TARGET_BODY GO_INC_B_T; +static const char GO_INC_B_RENAME[] = "package b\n\n" GO_INC_B_RENAMED GO_INC_B_T; +static const char GO_INC_C[] = "package b\n\n// U is another type.\ntype U struct{}\n\n" + "// M is U's method; see [Target].\nfunc (U) M() {}\n"; +#define GO_INC_D_HEAD "package b\n\n// V is v.\ntype V struct{}\n\n// W is w.\ntype W struct{}\n\n" +#define GO_INC_D_V "// Name of V.\nfunc (V) Name() {}\n\n" +#define GO_INC_D_W "// Name of W.\nfunc (W) Name() {}\n\n" +static const char GO_INC_D[] = GO_INC_D_HEAD GO_INC_D_V GO_INC_D_W; +static const char GO_INC_D_SWAPPED[] = GO_INC_D_HEAD GO_INC_D_W GO_INC_D_V; + +TEST(doc_mentions_go_incremental_equals_full) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dm_goinc_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[400]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + go_w(repo, "go.mod", "module example.com/inc\n\ngo 1.22\n"); + go_w(repo, "a/a.go", GO_INC_A); + go_w(repo, "a/doc.go", GO_INC_DOC); + go_w(repo, "a/other.go", GO_INC_OTHER); + go_w(repo, "b/b.go", GO_INC_B); + go_w(repo, "b/c.go", GO_INC_C); + go_w(repo, "b/d.go", GO_INC_D); + char inc_db[512]; + char full_db[512]; + snprintf(inc_db, sizeof(inc_db), "%s/inc.db", tmp); + snprintf(full_db, sizeof(full_db), "%s/full.db", tmp); + ASSERT_EQ(dm_index(repo, inc_db, NULL), 0); + /* the preconditions the steps below move away from */ + static const go_want_edge_t before[] = { + {"a.Use", "b.Target", "exact"}, {"a.Use", "b.Gone", "exact"}, + {"a.Use", "b.M", "exact"}, /* T.M, in the first file of the directory, keeps b.M */ + {"a.Use", "b.Name", "exact"}, /* W.Name, the later declaration, keeps b.Name */ + {"a.Use", "a.Local", "unique"}, {"a.doc.go.__file__", "a.Local", "unique"}, + {"a.Other", "b.Target", ""}, /* other.go does not import b */ + }; + ASSERT_EQ(go_bad_edges(inc_db, before, GO_COUNT(before)), 0); + ASSERT_STR_EQ(go_reason(inc_db, "a/a.go", "b.V.Name"), "graph_gap"); + ASSERT_STR_EQ(go_reason(inc_db, "b/c.go", "Target"), "graph_gap"); /* U.M has no node */ + + /* a doc-comment edit re-extracts the file alone */ + go_w(repo, "a/a.go", GO_INC_A_DOC_EDIT); + ASSERT_EQ(dm_step(repo, inc_db, full_db, "doc edit", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0); + ASSERT_STR_EQ(go_tier(inc_db, "a.Use", "b.T"), "exact"); + + /* a body edit in a file other files' references point into: its scope is + * unchanged, and the edges into it stand */ + go_w(repo, "b/b.go", GO_INC_B_BODY); + ASSERT_EQ( + dm_step(repo, inc_db, full_db, "target body edit", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), + 0); + ASSERT_STR_EQ(go_tier(inc_db, "a.Use", "b.Target"), "exact"); + + /* a file's imports are its own business: an import edit repairs the file */ + go_w(repo, "a/other.go", GO_INC_OTHER_IMPORT); + ASSERT_EQ(dm_step(repo, inc_db, full_db, "import added", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), + 0); + ASSERT_STR_EQ(go_tier(inc_db, "a.Other", "b.Target"), "exact"); + + /* a reference added to the package comment: an edge from the File node */ + go_w(repo, "a/doc.go", GO_INC_DOC_EDIT); + ASSERT_EQ( + dm_step(repo, inc_db, full_db, "package comment", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0); + ASSERT_STR_EQ(go_tier(inc_db, "a.doc.go.__file__", "a.Use"), "unique"); + ASSERT_STR_EQ(go_tier(inc_db, "a.doc.go.__file__", "a.Local"), "unique"); + + /* a removed function: the files with an edge into the changed file + * re-resolve */ + go_w(repo, "b/b.go", GO_INC_B_NO_GONE); + ASSERT_EQ( + dm_step(repo, inc_db, full_db, "function removed", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), + 0); + ASSERT_STR_EQ(go_reason(inc_db, "a/a.go", "b.Gone"), "missing"); + + /* the method that kept a shared node name goes: the other file's method of + * that name takes the node over, and its doc becomes a source */ + go_w(repo, "b/b.go", GO_INC_B_NO_M); + ASSERT_EQ( + dm_step(repo, inc_db, full_db, "name keeper removed", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), + 0); + ASSERT_STR_EQ(go_reason(inc_db, "a/a.go", "b.T.M"), "missing"); + ASSERT_STR_EQ(go_tier(inc_db, "b.M", "b.Target"), "unique"); + ASSERT_STR_EQ(go_reason(inc_db, "b/c.go", "Target"), ""); + + /* declaration order decides which declaration keeps a shared name: + * swapping two methods moves the node, for files with no edge into it too */ + go_w(repo, "b/d.go", GO_INC_D_SWAPPED); + ASSERT_EQ(dm_step(repo, inc_db, full_db, "methods swapped", CBM_INCREMENTAL_ROUTE_FORCED_FULL), + 0); + ASSERT_STR_EQ(go_reason(inc_db, "a/a.go", "b.W.Name"), "graph_gap"); + ASSERT_STR_EQ(go_reason(inc_db, "a/a.go", "b.V.Name"), ""); + + /* renaming a target */ + go_w(repo, "b/b.go", GO_INC_B_RENAME); + ASSERT_EQ(dm_step(repo, inc_db, full_db, "rename", CBM_INCREMENTAL_ROUTE_FORCED_FULL), 0); + ASSERT_STR_EQ(go_reason(inc_db, "a/a.go", "b.Target"), "missing"); + ASSERT_STR_EQ(go_reason(inc_db, "a/other.go", "b.Target"), "missing"); + + /* a deleted file takes its declarations along */ + ASSERT_EQ(unlink(TH_PATH(repo, "b/d.go")), 0); + ASSERT_EQ(dm_step(repo, inc_db, full_db, "delete file", CBM_INCREMENTAL_ROUTE_FORCED_FULL), 0); + ASSERT_STR_EQ(go_reason(inc_db, "a/a.go", "b.V.Name"), "missing"); + + /* the module path names every package of the repository */ + go_w(repo, "go.mod", "module example.com/moved\n\ngo 1.22\n"); + ASSERT_EQ(dm_step(repo, inc_db, full_db, "module path", CBM_INCREMENTAL_ROUTE_FORCED_FULL), 0); + ASSERT_STR_EQ(go_reason(inc_db, "a/a.go", "b.T"), "external"); + + dm_unlink_db(inc_db); + dm_unlink_db(full_db); + th_rmtree(tmp); + PASS(); +} + +/* ── which scope changes are repairable file by file ─────────────── */ + +#define GO_DELTA_HEAD "package p\n\nimport \"example.com/m/q\"\n\n" +#define GO_DELTA_F "// F is.\nfunc F() {}\n\n" +#define GO_DELTA_G "// G is.\nfunc G() {}\n\n" +#define GO_DELTA_T "type T struct {\n\tA int\n\tB int\n}\n\n" +#define GO_DELTA_T_NO_B "type T struct {\n\tA int\n}\n\n" +#define GO_DELTA_M "func (T) M() {}\n\n" +#define GO_DELTA_I "type I interface {\n\tDo()\n\tUndo()\n}\n\n" +#define GO_DELTA_I_NO_UNDO "type I interface {\n\tDo()\n}\n\n" +#define GO_DELTA_C "const C = 1\n" +#define GO_DELTA_V "var V = 2\n" +#define GO_DELTA_REST GO_DELTA_T GO_DELTA_M GO_DELTA_I GO_DELTA_C GO_DELTA_V + +TEST(doc_mentions_go_scope_delta_rules) { + const char *base = GO_DELTA_HEAD GO_DELTA_F GO_DELTA_G GO_DELTA_REST; + const char *twins = "package p\n\nfunc Open() {}\nfunc (T) Open() {}\n"; + const char *alias = "package p\n\nimport \"example.com/m/q\"\n\ntype X = q.Y\n"; + const struct { + const char *what; + const char *before; + const char *after; + int want; + const char *names; + } cases[] = { + {"unchanged", base, base, CBM_DOCLINK_DELTA_LOCAL, ""}, + /* nothing of the persisted scope carries a line number or a body */ + {"lines moved", base, "// moved\n\n\n" GO_DELTA_HEAD GO_DELTA_F GO_DELTA_G GO_DELTA_REST, + CBM_DOCLINK_DELTA_LOCAL, ""}, + {"body and doc edited", base, + GO_DELTA_HEAD "// F is [G].\nfunc F() {\n\t_ = 1\n}\n\n" GO_DELTA_G GO_DELTA_REST, + CBM_DOCLINK_DELTA_LOCAL, ""}, + {"declarations reordered", base, GO_DELTA_HEAD GO_DELTA_G GO_DELTA_F GO_DELTA_REST, + CBM_DOCLINK_DELTA_LOCAL, ""}, + {"unexported declaration added", base, + GO_DELTA_HEAD GO_DELTA_F GO_DELTA_G "func helper() {}\n\n" GO_DELTA_REST, + CBM_DOCLINK_DELTA_LOCAL, ""}, + /* the file's imports scope the file itself */ + {"import added", base, + "package p\n\nimport (\n\t\"example.com/m/q\"\n\t\"example.com/m/r\"\n)\n\n" GO_DELTA_F + GO_DELTA_G GO_DELTA_REST, + CBM_DOCLINK_DELTA_LOCAL, ""}, + {"import removed", base, "package p\n\n" GO_DELTA_F GO_DELTA_G GO_DELTA_REST, + CBM_DOCLINK_DELTA_LOCAL, ""}, + {"import renamed", base, + "package p\n\nimport qq \"example.com/m/q\"\n\n" GO_DELTA_F GO_DELTA_G GO_DELTA_REST, + CBM_DOCLINK_DELTA_LOCAL, ""}, + /* a declaration that only LEFT: its name is reported */ + {"function removed", base, GO_DELTA_HEAD GO_DELTA_F GO_DELTA_REST, CBM_DOCLINK_DELTA_LOCAL, + "G"}, + {"method removed", base, + GO_DELTA_HEAD GO_DELTA_F GO_DELTA_G GO_DELTA_T GO_DELTA_I GO_DELTA_C GO_DELTA_V, + CBM_DOCLINK_DELTA_LOCAL, "M"}, + {"field removed", base, + GO_DELTA_HEAD GO_DELTA_F GO_DELTA_G GO_DELTA_T_NO_B GO_DELTA_M GO_DELTA_I GO_DELTA_C + GO_DELTA_V, + CBM_DOCLINK_DELTA_LOCAL, "B"}, + {"interface method removed", base, + GO_DELTA_HEAD GO_DELTA_F GO_DELTA_G GO_DELTA_T GO_DELTA_M GO_DELTA_I_NO_UNDO GO_DELTA_C + GO_DELTA_V, + CBM_DOCLINK_DELTA_LOCAL, "Undo"}, + {"constant removed", base, + GO_DELTA_HEAD GO_DELTA_F GO_DELTA_G GO_DELTA_T GO_DELTA_M GO_DELTA_I GO_DELTA_V, + CBM_DOCLINK_DELTA_LOCAL, "C"}, + /* anything that appears or changes can re-route references in files + * with no edge into this one */ + {"function added", base, + GO_DELTA_HEAD GO_DELTA_F GO_DELTA_G "func H() {}\n\n" GO_DELTA_REST, + CBM_DOCLINK_DELTA_GLOBAL, NULL}, + {"type removed", base, + GO_DELTA_HEAD GO_DELTA_F GO_DELTA_G GO_DELTA_T GO_DELTA_M GO_DELTA_C GO_DELTA_V, + CBM_DOCLINK_DELTA_GLOBAL, NULL}, + {"type changes its kind", base, + GO_DELTA_HEAD GO_DELTA_F GO_DELTA_G GO_DELTA_T GO_DELTA_M + "type I struct {\n\tDo int\n\tUndo int\n}\n\n" GO_DELTA_C GO_DELTA_V, + CBM_DOCLINK_DELTA_GLOBAL, NULL}, + {"package clause renamed", base, + "package other\n\nimport \"example.com/m/q\"\n\n" GO_DELTA_F GO_DELTA_G GO_DELTA_REST, + CBM_DOCLINK_DELTA_GLOBAL, NULL}, + {"build constraint added", base, + "//go:build linux\n\n" GO_DELTA_HEAD GO_DELTA_F GO_DELTA_G GO_DELTA_REST, + CBM_DOCLINK_DELTA_GLOBAL, NULL}, + {"variable moved into a block (it loses its node)", base, + GO_DELTA_HEAD GO_DELTA_F GO_DELTA_G GO_DELTA_T GO_DELTA_M GO_DELTA_I GO_DELTA_C + "var (\n\tV = 2\n)\n", + CBM_DOCLINK_DELTA_GLOBAL, NULL}, + /* the declaration that keeps a shared node name changes */ + {"same-named declarations swapped", twins, + "package p\n\nfunc (T) Open() {}\nfunc Open() {}\n", CBM_DOCLINK_DELTA_GLOBAL, NULL}, + {"one of two same-named declarations removed", twins, "package p\n\nfunc Open() {}\n", + CBM_DOCLINK_DELTA_GLOBAL, NULL}, + /* other files follow `type X = q.Y` through this file's imports */ + {"import changed under a qualified alias", alias, + "package p\n\nimport q \"example.com/m/other\"\n\ntype X = q.Y\n", + CBM_DOCLINK_DELTA_GLOBAL, NULL}, + /* a file that appears or goes */ + {"file deleted", base, NULL, CBM_DOCLINK_DELTA_GLOBAL, NULL}, + {"file added", NULL, base, CBM_DOCLINK_DELTA_GLOBAL, NULL}, + }; + for (size_t i = 0; i < GO_COUNT(cases); i++) { + char names[256]; + int got = dm_scope_delta(CBM_LANG_GO, "p/x.go", cases[i].before, cases[i].after, names, + sizeof(names)); + bool names_ok = !cases[i].names || strcmp(names, cases[i].names) == 0; + if (got != cases[i].want || !names_ok) { + printf(" %s: delta %d names \"%s\", want %d \"%s\"\n", cases[i].what, got, names, + cases[i].want, cases[i].names ? cases[i].names : "(any)"); + FAIL("scope delta"); + } + } + /* go.mod: the module path is everybody's scope; the rest of the file is not */ + char names[256]; + ASSERT_EQ(dm_scope_delta(CBM_LANG_GOMOD, "go.mod", "module example.com/a\n", + "module example.com/b\n", names, sizeof(names)), + CBM_DOCLINK_DELTA_GLOBAL); + ASSERT_EQ(dm_scope_delta(CBM_LANG_GOMOD, "go.mod", "module example.com/a\n\ngo 1.21\n", + "module example.com/a\n\ngo 1.22\n\nrequire example.com/x v1.0.0\n", + names, sizeof(names)), + CBM_DOCLINK_DELTA_LOCAL); + PASS(); +} + +/* ── one worker == several workers ───────────────────────────────── */ + +TEST(doc_mentions_go_workers_agree) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dm_gopar_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[400]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + go_w(repo, "go.mod", "module example.com/par\n\ngo 1.22\n"); + enum { PKGS = 12, FILES = 6 }; /* 72 files: past the parallel threshold */ + for (int p = 0; p < PKGS; p++) { + for (int f = 0; f < FILES; f++) { + char rel[64]; + char src[2048]; + int next = (p + 1) % PKGS; + snprintf(rel, sizeof(rel), "p%02d/f%d.go", p, f); + snprintf(src, sizeof(src), + "package p%02d\n" + "\n" + "import \"example.com/par/p%02d\"\n" + "\n" + "// T%d is a type; see [F%d], [T%d.Close], [p%02d.T%d], [p%02d.T%d.Close],\n" + "// [p%02d.Gone], [fmt.Println], [Shared] and [Nothing].\n" + "type T%d struct{}\n" + "\n" + "// Close closes T%d; see [T%d].\n" + "func (T%d) Close() {}\n" + "\n" + "// F%d is a function; see [p%02d].\n" + "func F%d() {}\n" + "\n" + "// Shared is declared by every file of the package.\n" + "func Shared() {}\n", + p, next, f, (f + 1) % FILES, (f + 2) % FILES, next, f, next, (f + 3) % FILES, + next, f, f, f, f, f, next, f); + go_w(repo, rel, src); + } + } + char par_db[512]; + char seq_db[512]; + snprintf(par_db, sizeof(par_db), "%s/par.db", tmp); + snprintf(seq_db, sizeof(seq_db), "%s/seq.db", tmp); + ASSERT_EQ(dm_workers_agree(repo, par_db, seq_db), 0); + /* and the fixture exercises what it should: edges, and rows of more than + * one reason */ + ASSERT_STR_EQ(go_tier(par_db, "p00.T0", "p00.F1"), "unique"); + ASSERT_STR_EQ(go_tier(par_db, "p00.T0", "p01.T0"), "exact"); + ASSERT_STR_EQ(go_reason(par_db, "p00/f0.go", "p01.Gone"), "missing"); + ASSERT_STR_EQ(go_reason(par_db, "p00/f0.go", "fmt.Println"), "external"); + ASSERT_STR_EQ(go_reason(par_db, "p00/f0.go", "p01.T3.Close"), "graph_gap"); + ASSERT_STR_EQ(go_reason(par_db, "p00/f3.go", "Shared"), "graph_gap"); + ASSERT_TRUE(dm_count(par_db, "SELECT COUNT(*) FROM edges WHERE type = 'MENTIONS'") > 100); + dm_unlink_db(par_db); + dm_unlink_db(seq_db); + th_rmtree(tmp); + PASS(); +} + +SUITE(doc_mentions_go) { + dm_ship_held_families(); + RUN_TEST(doc_mentions_go_tokens); + RUN_TEST(doc_mentions_go_sources); + RUN_TEST(doc_mentions_go_package_comment_tokens); + RUN_TEST(doc_mentions_go_text_model); + RUN_TEST(doc_mentions_go_own_parameter); + RUN_TEST(doc_mentions_go_hostile_doc); + RUN_TEST(doc_mentions_go_random_docs); + RUN_TEST(doc_mentions_go_random_scopes); + RUN_TEST(doc_mentions_go_scope_blob); + RUN_TEST(doc_mentions_go_forms); + RUN_TEST(doc_mentions_go_plain_text); + RUN_TEST(doc_mentions_go_struct_fields); + RUN_TEST(doc_mentions_go_file_only_package_lookup); + RUN_TEST(doc_mentions_go_local_rules); + RUN_TEST(doc_mentions_go_lookup_order); + RUN_TEST(doc_mentions_go_reasons); + RUN_TEST(doc_mentions_go_node_owner); + RUN_TEST(doc_mentions_go_package_comment); + RUN_TEST(doc_mentions_go_std_module); + RUN_TEST(doc_mentions_go_ship_gate); + RUN_TEST(doc_mentions_go_incremental_equals_full); + RUN_TEST(doc_mentions_go_scope_delta_rules); + RUN_TEST(doc_mentions_go_workers_agree); +} diff --git a/tests/test_doc_mentions_helpers.h b/tests/test_doc_mentions_helpers.h index 19714fad6..7756b4947 100644 --- a/tests/test_doc_mentions_helpers.h +++ b/tests/test_doc_mentions_helpers.h @@ -432,7 +432,7 @@ static inline int dm_reason_lines(const char *db, const char *block) { static const int DM_HELD_FAMILIES[] = { CBM_DOCLINK_MD_BARE_PATH, CBM_DOCLINK_MD_SUPERSEDES_PROSE, CBM_DOCLINK_RST_INCLUDE, CBM_DOCLINK_RST_KERNEL_DOC, CBM_DOCLINK_ADOC_ATTRIBUTE, CBM_DOCLINK_ADOC_CODE_PATH, - CBM_DOCLINK_ADOC_CODE_NAME, + CBM_DOCLINK_ADOC_CODE_NAME, CBM_DOCLINK_GO_DOC_LINK, }; static inline void dm_ship_held_families(void) { diff --git a/tests/test_main.c b/tests/test_main.c index 145e96ef6..d465ba98d 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -1825,6 +1825,7 @@ extern void suite_doc_links_md(void); extern void suite_doc_links_pdf(void); extern void suite_doc_links_rst(void); extern void suite_doc_links_adoc(void); +extern void suite_doc_mentions_go(void); extern void suite_infrascan(void); extern void suite_cli(void); extern void suite_agent_clients(void); @@ -2296,6 +2297,7 @@ int main(int argc, char **argv) { RUN_SELECTED_SUITE(doc_links_pdf); RUN_SELECTED_SUITE(doc_links_rst); RUN_SELECTED_SUITE(doc_links_adoc); + RUN_SELECTED_SUITE(doc_mentions_go); /* Infrastructure scanning */ RUN_SELECTED_SUITE(infrascan); From a1190afbb0f2dae89179a2862c1749cdfa1b13bd Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 10 Oct 2026 11:07:37 +0200 Subject: [PATCH 02/17] doc-links(java, kotlin): Javadoc and KDoc references become MENTIONS edges, held until their audit Java's {@link}, {@linkplain}, @see, @throws and {@value}, and Kotlin's [Name] links, @see, @throws and @sample, bound the way javadoc and Dokka search for a member (one resolver for both languages, so a Kotlin doc can name a Java type and back). The leg (doclink_jvm.c extracts the references and each file's scope; doc_links_jvm.c resolves them) is ported onto the restacked core: family rows carry their edge type, resolve() takes the per-file state (none here). All nine families are held (ships=false) until their held-out audit passes; the suite ships them through the test seam. The graph moved under the leg since it was written: a Java field's node is named by the field alone (no longer by its whole declarator, `MAX = 10`), and a field named like a method of its class has its own node, .#field, where the method used to take the name. The tests follow: the expected field names lose the initializer, a changed initializer is a local change repaired file by file (it renamed the node before, which forced a full run), and a field that shares its accessor's name is now an edge target and a source (doc_mentions_jvm_java_families_and_reasons, doc_mentions_jvm_node_ownership), where both were graph_gap rows. Measured with the leg's binary (1078036f) and this one on the same input: - elasticsearch subset (libs/ + server/src/main at f3dc4b0bd3a0, 5,956 files): no edge lost; 66 new edges and 7 edges moved onto a field's own #field node; 81 graph_gap rows gone, no new row, no reason changed. A seeded sample of 8 of the gone rows read against the source: every new edge is right (a field named like its accessor, enum constants, a field's own doc as the source). - Exposed (Kotlin, c419caaab61f): identical, 896 edges and 371 rows. Signed-off-by: Martin Vogel --- Makefile.cbm | 4 +- internal/cbm/doclink.c | 10 + internal/cbm/doclink.h | 34 + internal/cbm/doclink_jvm.c | 2983 +++++++++++++++++++ src/pipeline/doc_links.c | 1 + src/pipeline/doc_links.h | 1 + src/pipeline/doc_links_jvm.c | 4599 +++++++++++++++++++++++++++++ tests/test_doc_mentions_helpers.h | 10 +- tests/test_doc_mentions_jvm.c | 3669 +++++++++++++++++++++++ tests/test_main.c | 2 + 10 files changed, 11310 insertions(+), 3 deletions(-) create mode 100644 internal/cbm/doclink_jvm.c create mode 100644 src/pipeline/doc_links_jvm.c create mode 100644 tests/test_doc_mentions_jvm.c diff --git a/Makefile.cbm b/Makefile.cbm index bee0f531d..29f5896d0 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -387,6 +387,7 @@ EXTRACTION_SRCS = \ $(CBM_DIR)/pdf/pdf_tables.c \ $(CBM_DIR)/pdf/pdf_unicode.c \ $(CBM_DIR)/doclink_go.c \ + $(CBM_DIR)/doclink_jvm.c \ $(CBM_DIR)/sql_values.c \ $(CBM_DIR)/helpers.c \ $(CBM_DIR)/cpp_branch_views.c \ @@ -514,6 +515,7 @@ PIPELINE_SRCS = \ src/pipeline/doc_links_rst.c \ src/pipeline/doc_links_adoc.c \ src/pipeline/doc_links_go.c \ + src/pipeline/doc_links_jvm.c \ src/pipeline/doc_links_msbuild.c \ src/pipeline/pass_route_nodes.c \ src/pipeline/pass_enrichment.c \ @@ -776,7 +778,7 @@ TEST_DISCOVER_SRCS = \ TEST_GRAPH_BUFFER_SRCS = tests/test_graph_buffer.c -TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_pipeline_conventions.c tests/test_pipeline_frozen.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_doclinks.c tests/test_doc_mentions.c tests/test_doc_links_md.c tests/test_doc_links_pdf.c tests/test_doc_links_rst.c tests/test_doc_links_adoc.c tests/test_doc_mentions_go.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c +TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_pipeline_conventions.c tests/test_pipeline_frozen.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_doclinks.c tests/test_doc_mentions.c tests/test_doc_links_md.c tests/test_doc_links_pdf.c tests/test_doc_links_rst.c tests/test_doc_links_adoc.c tests/test_doc_mentions_go.c tests/test_doc_mentions_jvm.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c TEST_WATCHER_SRCS = tests/test_watcher.c diff --git a/internal/cbm/doclink.c b/internal/cbm/doclink.c index 0d8c1e753..6e58f34c2 100644 --- a/internal/cbm/doclink.c +++ b/internal/cbm/doclink.c @@ -190,6 +190,16 @@ static const doclink_lang_t DOCLINK_LANGS[] = { .scan_scope = cbm_doclink_gomod_scan_scope, .scope_tag = CBM_DOCLINK_GO_SCOPE_TAG, .portable_scope = cbm_doclink_go_portable_scope}, + {.lang = CBM_LANG_JAVA, + .parse_doc = cbm_doclink_java_parse_doc, + .scan_scope = cbm_doclink_java_scan_scope, + .scope_tag = CBM_DOCLINK_JVM_SCOPE_TAG, + .portable_scope = cbm_doclink_jvm_portable_scope}, + {.lang = CBM_LANG_KOTLIN, + .parse_doc = cbm_doclink_kotlin_parse_doc, + .scan_scope = cbm_doclink_kotlin_scan_scope, + .scope_tag = CBM_DOCLINK_JVM_SCOPE_TAG, + .portable_scope = cbm_doclink_jvm_portable_scope}, }; static const doclink_lang_t *doclink_lang(CBMLanguage lang) { diff --git a/internal/cbm/doclink.h b/internal/cbm/doclink.h index ab97ac0f4..c3a94cca3 100644 --- a/internal/cbm/doclink.h +++ b/internal/cbm/doclink.h @@ -85,6 +85,16 @@ X(CBM_DOCLINK_CS_INHERITDOC, CBM_LANG_CSHARP, "inheritdoc", false, true, "MENTIONS") \ /* go (doclink_go.c): [Name] doc links; held until its held-out audit passes */ \ X(CBM_DOCLINK_GO_DOC_LINK, CBM_LANG_GO, "doc_link", false, false, "MENTIONS") \ + /* java and kotlin (doclink_jvm.c): Javadoc and KDoc; held until their audit passes */ \ + X(CBM_DOCLINK_JAVA_LINK, CBM_LANG_JAVA, "link", false, false, "MENTIONS") \ + X(CBM_DOCLINK_JAVA_LINKPLAIN, CBM_LANG_JAVA, "linkplain", false, false, "MENTIONS") \ + X(CBM_DOCLINK_JAVA_SEE, CBM_LANG_JAVA, "see", false, false, "MENTIONS") \ + X(CBM_DOCLINK_JAVA_THROWS, CBM_LANG_JAVA, "throws", false, false, "MENTIONS") \ + X(CBM_DOCLINK_JAVA_VALUE, CBM_LANG_JAVA, "value", false, false, "MENTIONS") \ + X(CBM_DOCLINK_KT_KDOC_LINK, CBM_LANG_KOTLIN, "kdoc_link", false, false, "MENTIONS") \ + X(CBM_DOCLINK_KT_SEE, CBM_LANG_KOTLIN, "see", false, false, "MENTIONS") \ + X(CBM_DOCLINK_KT_THROWS, CBM_LANG_KOTLIN, "throws", false, false, "MENTIONS") \ + X(CBM_DOCLINK_KT_SAMPLE, CBM_LANG_KOTLIN, "sample", false, false, "MENTIONS") \ /* markdown (doclink_md.c): what a document's text names explicitly */ \ X(CBM_DOCLINK_MD_LINK, CBM_LANG_MARKDOWN, "link", false, true, "MENTIONS") \ X(CBM_DOCLINK_MD_PATH, CBM_LANG_MARKDOWN, "path", false, true, "MENTIONS") \ @@ -427,5 +437,29 @@ bool cbm_doclink_go_parse_ref(const char *raw, size_t len, CBMDocLinkGoRef *out) /* The Go scope blob (Go files and go.mod files) starts with this tag line. */ #define CBM_DOCLINK_GO_SCOPE_TAG "go1" +/* ── Java and Kotlin (doclink_jvm.c) ───────────────────────────────── */ + +void cbm_doclink_java_parse_doc(CBMExtractCtx *ctx, const CBMDefinition *def, const char *doc, + uint32_t doc_line); +void cbm_doclink_kotlin_parse_doc(CBMExtractCtx *ctx, const CBMDefinition *def, const char *doc, + uint32_t doc_line); +const char *cbm_doclink_java_scan_scope(CBMExtractCtx *ctx); +const char *cbm_doclink_kotlin_scan_scope(CBMExtractCtx *ctx); +char *cbm_doclink_jvm_portable_scope(const char *scope); + +/* Normalize one Java parameter type as written in a declaration or in a + * Javadoc `#member(...)` list: annotations, `final`, type arguments, package + * and outer-type qualifiers and a trailing parameter name are dropped, + * `...` becomes `[]`, array brackets stay. "?" when nothing is left. Writes a + * NUL-terminated string into `out` (truncated at cap) and returns its + * length. */ +size_t cbm_doclink_java_norm_type(const char *in, size_t len, char *out, size_t cap); + +/* Java and Kotlin files share one scope tag: one resolver indexes both. */ +#define CBM_DOCLINK_JVM_SCOPE_TAG "jvm1" + +/* The longest reference a Java or Kotlin token may carry: a longer one is cut + * just past this length at extraction and reported as unparseable. */ +#define CBM_DOCLINK_JVM_REF_MAX 1000 #endif /* CBM_DOCLINK_H */ diff --git a/internal/cbm/doclink_jvm.c b/internal/cbm/doclink_jvm.c new file mode 100644 index 000000000..b33d37f75 --- /dev/null +++ b/internal/cbm/doclink_jvm.c @@ -0,0 +1,2983 @@ +/* + * doclink_jvm.c — Java and Kotlin doc-comment references and the JVM doc-link + * scope. + * + * Tokens. + * Javadoc: `{@link R}` / `{@link R label}`, `{@linkplain R label}`, + * `{@value R#F}` (a bare `{@value}` names the documented field itself and is + * no token), `@see R` at the start of a block-tag line (`@see ` is + * a URL, `@see "text"` is prose), `@throws T` / `@exception T`. Nothing in + * `{@code}`, `{@literal}`, `{@snippet}` or an HTML comment is a reference. + * KDoc: `[Name]` and `[text][Name]` in the comment text (not `[text](url)`, + * not an image, not a `[label]: url` definition, not in a code span or a + * fenced or indented code block), `@see Name`, `@throws T` / `@exception T`, + * `@sample qualified.name`. The subject of `@param` / `@property` is the + * definition's own parameter, never a reference. + * Only block doc comments (slash, two stars) are read: that is what javadoc + * and Dokka read. + * + * Scope blob (one record per line, tab-separated, first line "jvm1"; records + * in document order, a member after its type): + * L lang j Java, k Kotlin + * P package "" when the file declares none + * I kind alias target import: s single (alias = the simple + * name it binds), w on-demand / star, + * m Java static single (alias = member, + * target = type), n Java static + * on-demand (target = type) + * T start end kind path tparams supers slot own pnames + * type: c class, i interface, e enum, + * r record, a annotation, o object, + * C companion object, t typealias; '!' + * after the kind when a parse error + * hides some of its members. path is + * the dotted LOGICAL path (Outer.Inner), + * tparams ','-joined, supers '|'-joined + * as written without type arguments + * ('?' for one that is no name) + * M start kind flags owner name tparams sig recv slot own pnames + * member: c callable, v value. flags: K + * constructor, p constructor property / + * record component, n enum constant, i + * implicit (declared by no text of its + * own). owner is the index of its T + * record ('-' for a Kotlin top-level + * declaration), sig the '|'-joined + * normalized parameter types ('-' for a + * value), recv the receiver type of a + * Kotlin extension + * Q name a type that is declared, but could not + * be placed (parse error) + * X from to lines whose declarations could not be + * placed + * slot / own say where the extractor put the declaration's node: slot is the + * qualified name of its definition relative to the file's module ('=' when it + * is the logical path, '-' when the declaration has no definition), own is 1 + * when that definition is the one the graph keeps for the qualified name + * inside this file. Both are read off the extractor's own definitions (same + * start line, name and kind), so this file models neither the qualified-name + * scheme nor its collisions: Kotlin flattens nested types and puts class + * properties at module level, Java names a field after its whole declarator + * (`MAX = 10`), overloads share one name, and whatever the scheme does next is + * followed without a change here. + * pnames (the parameter names of a callable, the primary-constructor + * parameters of a Kotlin class) and all line numbers serve the file's own + * references only: the persisted form drops them. + * + * Everything is a pure function of the file's bytes. Nesting is the tree's + * while the tree has no parse error; otherwise it is checked against the + * braces (see "Braces"). + */ +#include "doclink.h" + +#include "arena.h" +#include "foundation/constants.h" +#include "foundation/mem_core.h" +#include "helpers.h" +#include "tree_sitter/api.h" + +#include +#include +#include +#include + +enum { + JV_SB_INIT = 1024, + JV_MAX_DEPTH = 128, /* nested type declarations */ + JV_NAME_MAX = 255, /* one identifier */ + JV_TYPE_MAX = 512, /* one written type */ + JV_SLOT_MAX = 512, /* a definition's name relative to its module */ + JV_UINT_DIGITS = 16, + JV_ITEMS_INIT = 64, + JV_BRACES_INIT = 1024, + JV_CHAR_LITERAL_MAX = 12, /* a quoted unicode escape */ + JV_QUOTES = 3, /* """ */ + JV_UTF8_TAIL_MAX = 3, +}; + +/* ── Small arena string builder ──────────────────────────────────── */ + +typedef struct { + CBMArena *a; + char *buf; + size_t len; + size_t cap; + bool failed; +} jv_sb_t; + +static void jv_reserve(jv_sb_t *sb, size_t extra) { + if (sb->failed || sb->len + extra + SKIP_ONE <= sb->cap) { + return; + } + size_t ncap = sb->cap ? sb->cap : JV_SB_INIT; + while (ncap < sb->len + extra + SKIP_ONE) { + ncap *= PAIR_LEN; + } + char *grown = (char *)cbm_arena_alloc(sb->a, ncap); + if (!grown) { + sb->failed = true; + return; + } + if (sb->len > 0) { + memcpy(grown, sb->buf, sb->len); + } + sb->buf = grown; + sb->cap = ncap; +} + +static void jv_putn(jv_sb_t *sb, const char *s, size_t n) { + jv_reserve(sb, n); + if (sb->failed) { + return; + } + memcpy(sb->buf + sb->len, s, n); + sb->len += n; + sb->buf[sb->len] = '\0'; +} + +static void jv_puts(jv_sb_t *sb, const char *s) { + jv_putn(sb, s, strlen(s)); +} + +static void jv_putc(jv_sb_t *sb, char c) { + jv_putn(sb, &c, SKIP_ONE); +} + +static void jv_putu(jv_sb_t *sb, uint32_t v) { + char tmp[JV_UINT_DIGITS]; + int n = snprintf(tmp, sizeof(tmp), "%u", v); + if (n > 0) { + jv_putn(sb, tmp, (size_t)n); + } +} + +/* ── Characters ──────────────────────────────────────────────────── */ + +static bool jv_blank(char c) { + return c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '\f' || c == '\v'; +} + +/* Identifier characters: Java and Kotlin identifiers are Unicode, so every + * byte of a multi-byte character counts. */ +static bool jv_ident_start(char c) { + unsigned char u = (unsigned char)c; + return isalpha(u) || c == '_' || c == '$' || u >= 0x80; +} + +static bool jv_ident_char(char c) { + unsigned char u = (unsigned char)c; + return isalnum(u) || c == '_' || c == '$' || u >= 0x80; +} + +/* ── Doc text ────────────────────────────────────────────────────── */ + +/* The text of the block doc comments in `doc` (slash, two stars) without its + * decoration: every line break stays (a position's line is the doc's first + * line plus the breaks before it), each continuation line loses its leading + * blanks, its asterisks and one blank. Line comments and plain block comments + * between doc comments leave only their line breaks. `nesting`: block + * comments nest (Kotlin), so a comment written inside the doc is part of its + * text and its end is not the doc's. */ +static char *jv_clean_doc(CBMArena *a, const char *doc, bool nesting, size_t *out_len) { + size_t n = strlen(doc); + char *out = (char *)cbm_arena_alloc(a, n + SKIP_ONE); + if (!out) { + return NULL; + } + size_t w = 0; + size_t i = 0; + bool in_doc = false; + size_t inner = 0; /* comments open inside the doc */ + while (i < n) { + char c = doc[i]; + if (!in_doc) { + if (c == '/' && doc[i + 1] == '*' && doc[i + 2] == '*' && doc[i + 3] != '/') { + in_doc = true; + inner = 0; + i += 3; + while (doc[i] == '*' && doc[i + 1] != '/') { + i++; + } + continue; + } + if (c == '\n') { + out[w++] = '\n'; + } + i++; + continue; + } + if (nesting && c == '/' && doc[i + 1] == '*') { + inner++; + out[w++] = '/'; + out[w++] = '*'; + i += 2; + continue; + } + if (c == '*' && doc[i + 1] == '/') { + if (inner > 0) { + inner--; + out[w++] = '*'; + out[w++] = '/'; + } else { + in_doc = false; + } + i += 2; + continue; + } + if (c == '\n') { + out[w++] = '\n'; + i++; + while (doc[i] == ' ' || doc[i] == '\t' || doc[i] == '\r') { + i++; + } + if (doc[i] == '*' && doc[i + 1] != '/') { + while (doc[i] == '*' && doc[i + 1] != '/') { + i++; + } + if (doc[i] == ' ') { + i++; + } + } + continue; + } + if (c != '\r') { + out[w++] = c; + } + i++; + } + out[w] = '\0'; + *out_len = w; + return out; +} + +typedef struct { + CBMExtractCtx *ctx; + const CBMDefinition *def; + const char *t; /* the cleaned doc text */ + size_t n; +} jv_doc_t; + +/* Push the reference s[0, len) written on source line `line`: whitespace runs + * become one blank, control characters go, and a reference longer than any + * name is cut (on a character boundary) just past CBM_DOCLINK_JVM_REF_MAX, + * which is how the resolver knows it was oversized. */ +static void jv_push(jv_doc_t *d, const char *s, size_t len, uint32_t line, int syntax) { + size_t cap = (size_t)CBM_DOCLINK_JVM_REF_MAX + JV_UTF8_TAIL_MAX + SKIP_ONE; + size_t take = len < cap ? len : cap; + char *raw = (char *)cbm_arena_alloc(d->ctx->arena, take + SKIP_ONE); + if (!raw) { + return; + } + size_t w = 0; + bool space = false; + for (size_t i = 0; i < len && w < take; i++) { + unsigned char c = (unsigned char)s[i]; + if (c <= ' ') { + space = w > 0; + continue; + } + if (space && w + SKIP_ONE < take) { + raw[w++] = ' '; + } + space = false; + raw[w++] = (char)c; + } + if (w == take && len > take) { + /* cut: do not leave half a multi-byte character behind */ + while (w > 0 && ((unsigned char)raw[w - SKIP_ONE] & 0xC0) == 0x80) { + w--; + } + if (w > 0 && (unsigned char)raw[w - SKIP_ONE] >= 0xC0) { + w--; + } + } + raw[w] = '\0'; + if (w == 0) { + return; + } + CBMDocLink link = { + .source_qn = d->def->qualified_name, + .raw = raw, + .line = line, + .def_line = d->def->start_line, + .syntax = (uint16_t)syntax, + }; + cbm_doclinks_push(&d->ctx->result->doc_links, d->ctx->arena, link); +} + +/* Index just past the group opened at s[i] by `open` (nesting groups of the + * same kind), or n when it does not close. */ +static size_t jv_group_end(const char *s, size_t n, size_t i, char open, char close) { + int depth = 0; + for (size_t k = i; k < n; k++) { + if (s[k] == open) { + depth++; + } else if (s[k] == close) { + depth--; + if (depth == 0) { + return k + SKIP_ONE; + } + } + } + return n; +} + +static bool jv_word_is(const char *s, size_t len, const char *word) { + return strlen(word) == len && memcmp(s, word, len) == 0; +} + +/* ── Javadoc ─────────────────────────────────────────────────────── */ + +/* Length of the Javadoc reference at s: a (qualified) type, `#member`, + * `module/`, type arguments, array brackets and a parameter list. A parameter + * list that does not close takes the rest (the reference is malformed, and + * the resolver says so). */ +static size_t jv_java_ref_len(const char *s, size_t n) { + size_t j = 0; + while (j < n) { + char c = s[j]; + if (jv_ident_char(c) || c == '.' || c == '#' || c == '/') { + j++; + } else if (c == '<') { + size_t e = jv_group_end(s, n, j, '<', '>'); + if (e >= n && (n == 0 || s[n - SKIP_ONE] != '>')) { + break; + } + j = e; + } else if (c == '[' && j + SKIP_ONE < n && s[j + SKIP_ONE] == ']') { + j += PAIR_LEN; + } else { + break; + } + } + if (j < n && s[j] == '(') { + j = jv_group_end(s, n, j, '(', ')'); + } + return j; +} + +/* The reference at s[0, n): what the doc tool reads as one reference. Text + * glued to it (`Foo,`, `http://x`) makes the whole word the (malformed) + * reference. Returns its length, 0 when there is none. */ +static size_t jv_java_ref(const char *s, size_t n) { + size_t r = jv_java_ref_len(s, n); + if (r < n && !jv_blank(s[r])) { + while (r < n && !jv_blank(s[r])) { + r++; + } + } + return r; +} + +/* The href of an `` at s[0, n): pushed as a URL. */ +static void jv_java_href(jv_doc_t *d, const char *s, size_t n, uint32_t line) { + static const char key[] = "href"; + for (size_t i = 0; i + sizeof(key) <= n && s[i] != '>'; i++) { + if (memcmp(s + i, key, sizeof(key) - SKIP_ONE) != 0) { + continue; + } + size_t j = i + sizeof(key) - SKIP_ONE; + while (j < n && (s[j] == ' ' || s[j] == '\t')) { + j++; + } + if (j >= n || s[j] != '=') { + continue; + } + j++; + while (j < n && (s[j] == ' ' || s[j] == '\t')) { + j++; + } + if (j >= n || (s[j] != '"' && s[j] != '\'')) { + return; + } + char quote = s[j]; + size_t v = j + SKIP_ONE; + size_t e = v; + while (e < n && s[e] != quote) { + e++; + } + if (e < n && e > v) { + jv_push(d, s + v, e - v, line, CBM_DOCLINK_HREF); + } + return; + } +} + +/* `@see ...` / `@throws ...`: the rest of the tag's line starts at `i`. */ +static void jv_java_block_tag(jv_doc_t *d, size_t i, uint32_t line, bool see) { + const char *t = d->t; + while (i < d->n && (t[i] == ' ' || t[i] == '\t')) { + i++; + } + size_t eol = i; + while (eol < d->n && t[eol] != '\n') { + eol++; + } + if (i >= eol) { + return; + } + if (see && t[i] == '<') { + jv_java_href(d, t + i, eol - i, line); + return; + } + if (t[i] == '"' || t[i] == '{') { + return; /* a quoted title; an inline tag is found by the text scan */ + } + size_t r = jv_java_ref(t + i, eol - i); + if (r > 0) { + jv_push(d, t + i, r, line, see ? CBM_DOCLINK_JAVA_SEE : CBM_DOCLINK_JAVA_THROWS); + } +} + +/* An inline tag `{@name body}` whose body is t[b, e) and starts on `line`: + * its reference. */ +static void jv_java_inline(jv_doc_t *d, size_t b, size_t e, uint32_t line, int syntax) { + const char *t = d->t; + for (; b < e && jv_blank(t[b]); b++) { + line += t[b] == '\n'; + } + if (syntax == CBM_DOCLINK_JAVA_VALUE && b < e && t[b] == '"') { + /* {@value "format" ref}: the format string is not the reference */ + size_t q = b + SKIP_ONE; + while (q < e && t[q] != '"') { + line += t[q] == '\n'; + q++; + } + for (b = q < e ? q + SKIP_ONE : e; b < e && jv_blank(t[b]); b++) { + line += t[b] == '\n'; + } + } + size_t r = jv_java_ref(t + b, e - b); + if (r > 0) { + jv_push(d, t + b, r, line, syntax); + } +} + +/* An open inline tag. javadoc ends it at the brace that balances its own + * `{`; braces outside inline tags are plain text and balance nothing. */ +typedef struct { + int syntax; /* a link family, JV_TAG_OTHER, or JV_TAG_TEXT */ + size_t body; /* first byte after the tag's name */ + uint32_t line; /* the line of that byte */ + int depth; /* plain braces open inside it */ +} jv_tag_t; + +enum { + JV_TAG_OTHER = -1, /* another inline tag: its body is scanned */ + JV_TAG_TEXT = -2, /* code, literal, snippet: the body is text, not markup */ + JV_TAG_NEST_MAX = 32, +}; + +/* The kind of the inline tag `{@name`; `next` is the character after the + * name (a tag named `linkage` is not `link`). */ +static int jv_java_tag_kind(const char *name, size_t len, char next) { + if (jv_word_is(name, len, "code") || jv_word_is(name, len, "literal") || + jv_word_is(name, len, "snippet")) { + return JV_TAG_TEXT; + } + if (!jv_blank(next) && next != '}') { + return JV_TAG_OTHER; + } + if (jv_word_is(name, len, "link")) { + return CBM_DOCLINK_JAVA_LINK; + } + if (jv_word_is(name, len, "linkplain")) { + return CBM_DOCLINK_JAVA_LINKPLAIN; + } + if (jv_word_is(name, len, "value")) { + return CBM_DOCLINK_JAVA_VALUE; + } + return JV_TAG_OTHER; +} + +/* Past an HTML comment or a CDATA section that starts at t[i] (the doc tool + * reads no markup in either); i itself when none starts there. Line breaks + * passed are added to *line. One that never ends takes the rest. */ +static size_t jv_skip_hidden(const jv_doc_t *d, size_t i, uint32_t *line) { + static const struct { + const char *open; + const char *close; + } kinds[] = {{""}, {""}}; + for (size_t k = 0; k < sizeof(kinds) / sizeof(kinds[0]); k++) { + size_t ol = strlen(kinds[k].open); + if (d->n - i < ol || memcmp(d->t + i, kinds[k].open, ol) != 0) { + continue; + } + const char *close = strstr(d->t + i + ol, kinds[k].close); + size_t past = close ? (size_t)(close - d->t) + strlen(kinds[k].close) : d->n; + for (size_t p = i; p < past; p++) { + *line += d->t[p] == '\n'; + } + return past; + } + return i; +} + +void cbm_doclink_java_parse_doc(CBMExtractCtx *ctx, const CBMDefinition *def, const char *doc, + uint32_t doc_line) { + /* A field is also emitted as a package-level Variable with the same doc: + * the Field carries the references. A module declaration is no code + * definition (its "class" is named like a package). */ + if (!def->label || (strcmp(def->label, "Variable") == 0 && def->parent_class) || + (strcmp(def->label, "Class") == 0 && strchr(def->name, '.'))) { + return; + } + CBMArena *scratch = ctx->scratch ? ctx->scratch : ctx->arena; + jv_doc_t d = {.ctx = ctx, .def = def}; + d.t = jv_clean_doc(scratch, doc, false, &d.n); + if (!d.t) { + return; + } + const char *t = d.t; + size_t n = d.n; + jv_tag_t tags[JV_TAG_NEST_MAX]; + int sp = 0; /* open inline tags */ + uint32_t line = doc_line; + size_t i = 0; + bool bol = true; /* only blanks since the line began */ + while (i < n) { + char c = t[i]; + if (c == '\n') { + line++; + bol = true; + i++; + continue; + } + if (bol && (c == ' ' || c == '\t')) { + i++; + continue; + } + bool in_text = sp > 0 && tags[sp - SKIP_ONE].syntax == JV_TAG_TEXT; + if (bol && !in_text && c == '@' && i + SKIP_ONE < n && + isalpha((unsigned char)t[i + SKIP_ONE])) { + /* A block tag. javadoc ends an open {@link ..} (or any inline tag + * whose content is markup) at it: that tag never closed and is no + * link. Inside {@code}, {@literal} and {@snippet} an `@` at the + * start of a line is text (annotations in sample code). */ + size_t e = i + SKIP_ONE; + while (e < n && isalpha((unsigned char)t[e])) { + e++; + } + const char *name = t + i + SKIP_ONE; + size_t len = e - i - SKIP_ONE; + bool gap = e >= n || jv_blank(t[e]); + if (gap && jv_word_is(name, len, "see")) { + jv_java_block_tag(&d, e, line, true); + } else if (gap && + (jv_word_is(name, len, "throws") || jv_word_is(name, len, "exception"))) { + jv_java_block_tag(&d, e, line, false); + } + sp = 0; + bol = false; + i = e; + continue; + } + bol = false; + bool text = in_text; + if (c == '<' && sp == 0) { + size_t past = jv_skip_hidden(&d, i, &line); + if (past > i) { + i = past; + continue; + } + } + if (c == '{' && !text && i + SKIP_ONE < n && t[i + SKIP_ONE] == '@' && + sp < JV_TAG_NEST_MAX) { + size_t ne = i + PAIR_LEN; + while (ne < n && isalpha((unsigned char)t[ne])) { + ne++; + } + tags[sp++] = (jv_tag_t){.syntax = jv_java_tag_kind(t + i + PAIR_LEN, ne - i - PAIR_LEN, + ne < n ? t[ne] : '}'), + .body = ne, + .line = line}; + i = ne; + continue; + } + if (c == '{' && sp > 0) { + tags[sp - SKIP_ONE].depth++; + } else if (c == '}' && sp > 0) { + if (tags[sp - SKIP_ONE].depth > 0) { + tags[sp - SKIP_ONE].depth--; + } else { + jv_tag_t done = tags[--sp]; + if (done.syntax > CBM_DOCLINK_NONE) { + jv_java_inline(&d, done.body, i, done.line, done.syntax); + } + } + } + i++; + } +} + +/* ── KDoc ────────────────────────────────────────────────────────── */ + +/* Length of the KDoc link target at s[0, n): a qualified name whose segments + * are identifiers or backticked names. 0 when s is not exactly one. */ +static size_t jv_kdoc_target_len(const char *s, size_t n) { + size_t i = 0; + for (;;) { + if (i < n && s[i] == '`') { + size_t e = i + SKIP_ONE; + while (e < n && s[e] != '`' && s[e] != '\n') { + e++; + } + if (e >= n || s[e] != '`' || e == i + SKIP_ONE) { + return 0; + } + i = e + SKIP_ONE; + } else { + if (i >= n || !jv_ident_start(s[i]) || s[i] == '$') { + return 0; + } + while (i < n && jv_ident_char(s[i]) && s[i] != '$') { + i++; + } + } + if (i < n && s[i] == '.') { + i++; + continue; + } + return i == n ? i : 0; + } +} + +/* A `[...]` group opening at `i` on the line that ends at `eol`: the index of + * its `]`, or eol when it holds a bracket or does not close on the line. */ +static size_t jv_kdoc_bracket_close(const char *t, size_t i, size_t eol) { + for (size_t k = i + SKIP_ONE; k < eol; k++) { + if (t[k] == ']') { + return k; + } + if (t[k] == '[') { + return eol; + } + } + return eol; +} + +/* One line of KDoc text. */ +typedef struct { + size_t first; /* its first byte that is no blank */ + size_t eol; /* its end */ + uint32_t line; /* its source line */ +} jv_kline_t; + +/* `[Name]` / `[text][Name]` at `i`. Returns the index to continue from. */ +static size_t jv_kdoc_link(jv_doc_t *d, size_t i, const jv_kline_t *ln) { + const char *t = d->t; + size_t eol = ln->eol; + size_t close = jv_kdoc_bracket_close(t, i, eol); + if (close >= eol || close == i + SKIP_ONE) { + return i + SKIP_ONE; + } + size_t b = i + SKIP_ONE; + size_t e = close; + size_t next = close + SKIP_ONE; + if (next < eol && t[next] == '(') { + return next; /* [text](url): a Markdown link, not a code reference */ + } + if (next < eol && t[next] == '[') { + size_t close2 = jv_kdoc_bracket_close(t, next, eol); + if (close2 >= eol) { + return next; + } + if (close2 > next + SKIP_ONE) { + b = next + SKIP_ONE; /* [text][Name] */ + e = close2; + } + next = close2 + SKIP_ONE; + } else if (next < eol && t[next] == ':' && i == ln->first) { + return next; /* [label]: url -- a link definition */ + } + while (b < e && (t[b] == ' ' || t[b] == '\t')) { + b++; + } + while (e > b && (t[e - SKIP_ONE] == ' ' || t[e - SKIP_ONE] == '\t')) { + e--; + } + if (jv_kdoc_target_len(t + b, e - b) > 0) { + jv_push(d, t + b, e - b, ln->line, CBM_DOCLINK_KT_KDOC_LINK); + } + return next; +} + +/* The subject of `@see` / `@throws` / `@sample` starting at `i`: one word. + * Returns the index to continue the line's text scan from. */ +static size_t jv_kdoc_subject(jv_doc_t *d, size_t i, const jv_kline_t *ln, int syntax) { + const char *t = d->t; + size_t eol = ln->eol; + while (i < eol && (t[i] == ' ' || t[i] == '\t')) { + i++; + } + size_t e = i; + while (e < eol && t[e] != ' ' && t[e] != '\t') { + e++; + } + if (e == i) { + return i; + } + size_t b = i; + size_t w = e; + if (t[b] == '[') { + size_t close = jv_kdoc_bracket_close(t, b, eol); + if (close >= eol) { + return e; + } + b++; + w = close; + e = close + SKIP_ONE; + } else { + while (w > b && strchr(".,;:", t[w - SKIP_ONE])) { + w--; + } + } + bool url = + t[b] == '<' || (w - b > 4 && memcmp(t + b, "http", 4) == 0 && memchr(t + b, ':', w - b)); + if (w > b && !url) { + jv_push(d, t + b, w - b, ln->line, syntax); + } + return e; +} + +static int jv_kdoc_tag_syntax(const char *name, size_t len) { + if (jv_word_is(name, len, "see")) { + return CBM_DOCLINK_KT_SEE; + } + if (jv_word_is(name, len, "throws") || jv_word_is(name, len, "exception")) { + return CBM_DOCLINK_KT_THROWS; + } + if (jv_word_is(name, len, "sample")) { + return CBM_DOCLINK_KT_SAMPLE; + } + return CBM_DOCLINK_NONE; +} + +/* Just past the code span that the run of `len` backticks ending before `run` + * opens: the next run of exactly that many. 0 when the line has none (the + * backticks are then text). */ +static size_t jv_kdoc_span_end(const char *t, size_t run, size_t len, size_t eol) { + size_t k = run; + while (k < eol) { + if (t[k] != '`') { + k++; + continue; + } + size_t r = k; + while (r < eol && t[r] == '`') { + r++; + } + if (r - k == len) { + return r; + } + k = r; + } + return 0; +} + +/* The text of one KDoc line from `i`: code spans and HTML comments are + * skipped, bracket links pushed. *in_comment: an HTML comment is open (it may + * have begun on an earlier line). */ +static void jv_kdoc_text(jv_doc_t *d, size_t i, const jv_kline_t *ln, bool *in_comment) { + const char *t = d->t; + size_t eol = ln->eol; + /* A run of n backticks was looked for once and the rest of the line has no + * second one: a later run of n is text as well. Longer runs than a fence + * are not looked for (they would be searched once per length). */ + bool no_span[JV_QUOTES + SKIP_ONE] = {false}; + while (i < eol) { + char c = t[i]; + if (*in_comment) { + if (c == '-' && eol - i >= 3 && t[i + 1] == '-' && t[i + 2] == '>') { + *in_comment = false; + i += 3; + } else { + i++; + } + } else if (c == '\\') { + i += PAIR_LEN; + } else if (c == '`') { + size_t run = i; + while (run < eol && t[run] == '`') { + run++; + } + size_t len = run - i; + size_t past = + (len > JV_QUOTES || no_span[len]) ? 0 : jv_kdoc_span_end(t, run, len, eol); + if (past == 0 && len <= JV_QUOTES) { + no_span[len] = true; + } + i = past ? past : run; + } else if (c == '<' && eol - i >= 4 && memcmp(t + i, " \n" /* 8 */ + " * {@linkage NotATag} {@link Open\n" /* 9 */ + " * @see Gadget#spin()\n" /* 10 */ + " * @see site\n" /* 11 */ + " * @see \"The Book\"\n" /* 12 */ + " * @throws IllegalStateException never\n" /* 13 */ + " * @exception a.CustomError worse\n" /* 14 */ + " * @param the type {@link T}\n" /* 15 */ + " * @return {@value} nothing, {@value \"%x\" Helper#MASK}\n" /* 16 */ + " */\n" /* 17 */ + "public class Widget {\n" /* 18 */ + " /** Max {@value}. {@link Widget#MAX} */\n" /* 19 */ + " public static final int MAX = 10;\n" /* 20 */ + " /**\n" /* 21 */ + " * 
{@code\n"                                                    /* 22 */
+        "     * @Override {@link InSample}\n"                                     /* 23 */
+        "     * }
after {@link AfterSample}\n" /* 24 */ + " */\n" /* 25 */ + " void sample() {}\n" /* 26 */ + "}\n"; + CBMFileResult *r = dm_extract(src, CBM_LANG_JAVA, "a/Widget.java"); + ASSERT_NOT_NULL(r); + /* every family, with the line the reference is written on */ + const CBMDocLink *t = jt_token(r, "Gadget"); + ASSERT_EQ(t->syntax, CBM_DOCLINK_JAVA_LINK); + ASSERT_EQ(t->line, 4); + ASSERT_EQ(t->def_line, 18); + ASSERT_STR_EQ(t->source_qn, "p.a.Widget"); + ASSERT_EQ(dm_count_tokens(r, "Gadget#spin()"), 2); + ASSERT_EQ(jt_token(r, "Gadget#spin()")->syntax, CBM_DOCLINK_JAVA_LINKPLAIN); + /* the label is not part of the reference; blanks inside it are one */ + ASSERT_EQ(jt_token(r, "Helper#run(int, String)")->line, 5); + ASSERT_EQ(jt_token(r, "#MAX")->syntax, CBM_DOCLINK_JAVA_VALUE); + ASSERT_EQ(jt_token(r, "Wrapped#name")->line, 7); + ASSERT_EQ(jt_token(r, "IllegalStateException")->syntax, CBM_DOCLINK_JAVA_THROWS); + ASSERT_EQ(jt_token(r, "a.CustomError")->syntax, CBM_DOCLINK_JAVA_THROWS); + ASSERT_EQ(jt_token(r, "a.CustomError")->line, 14); + ASSERT_EQ(jt_token(r, "https://example.org/x")->syntax, CBM_DOCLINK_HREF); + ASSERT_EQ(jt_token(r, "https://example.org/x")->line, 11); + /* a format string is not the reference of {@value} */ + ASSERT_EQ(jt_token(r, "Helper#MASK")->syntax, CBM_DOCLINK_JAVA_VALUE); + /* a reference to the definition's own type parameter is still a token: + * the resolver, which knows the scope, calls it local */ + ASSERT_EQ(jt_token(r, "T")->line, 15); + /* not references: code, literal, HTML comment, CDATA, another tag whose + * name only starts like one, a quoted title, a bare {@value} */ + ASSERT_NULL(dm_find_token(r, "InCode")); + ASSERT_NULL(dm_find_token(r, "InLiteral")); + ASSERT_NULL(dm_find_token(r, "InComment")); + ASSERT_NULL(dm_find_token(r, "InCdata")); + ASSERT_NULL(dm_find_token(r, "NotATag")); + ASSERT_NULL(dm_find_token(r, "\"The")); + /* an inline tag that a block tag cuts off never closed: javadoc reports + * it, and it links nothing */ + ASSERT_NULL(dm_find_token(r, "Open")); + /* the field's doc is harvested once, on the Field (its package-level + * Variable twin carries the same doc) */ + ASSERT_EQ(dm_count_tokens(r, "Widget#MAX"), 1); + ASSERT_STR_EQ(jt_token(r, "Widget#MAX")->source_qn, "p.a.Widget.MAX"); + /* `@` at the start of a line inside {@code} is sample code, not a block + * tag, and what follows inside the block is still code */ + ASSERT_NULL(dm_find_token(r, "InSample")); + ASSERT_EQ(jt_token(r, "AfterSample")->line, 24); + cbm_free_result(r); + PASS(); +} + +TEST(doc_mentions_jvm_kotlin_tokens) { + const char *src = + "package k\n" /* 1 */ + "\n" /* 2 */ + "/**\n" /* 3 */ + " * A [Gadget], [the helper][Helper.run], [text](https://x.org), `[InCode]`.\n" /* 4 */ + " * ``double `tick` [InDouble]`` then [Outer.Inner] and [`quoted name`].\n" /* 5 */ + " * [label]: https://example.org\n" /* 6 */ + " * ![image][ImgLabel] \\[Escaped]\n" /* 7 */ + " * Nested /* comment [StillDoc] */ goes on to [AfterNested].\n" /* 8 */ + " *\n" /* 9 */ + " * [IndentedCode]\n" /* 10 */ + " *\n" /* 11 */ + " * ```\n" /* 12 */ + " * [Fenced]\n" /* 13 */ + " * ```\n" /* 14 */ + " * @param name the [name] of [T]\n" /* 15 */ + " * @property size the size\n" /* 16 */ + " * @see Gadget.spin\n" /* 17 */ + " * @see https://example.org\n" /* 18 */ + " * @throws IllegalStateException never\n" /* 19 */ + " * @exception CustomError worse\n" /* 20 */ + " * @sample k.samples.widgetSample\n" /* 21 */ + " * @return [Result] of [0] and [two words]\n" /* 22 */ + " */\n" /* 23 */ + "class Widget(val name: String, size: Int)\n"; /* 24 */ + CBMFileResult *r = dm_extract(src, CBM_LANG_KOTLIN, "k/Widget.kt"); + ASSERT_NOT_NULL(r); + const CBMDocLink *t = jt_token(r, "Gadget"); + ASSERT_EQ(t->syntax, CBM_DOCLINK_KT_KDOC_LINK); + ASSERT_EQ(t->line, 4); + ASSERT_EQ(t->def_line, 24); + ASSERT_STR_EQ(t->source_qn, "p.k.Widget.Widget"); + /* [text][Name]: the second bracket names the code */ + ASSERT_EQ(jt_token(r, "Helper.run")->syntax, CBM_DOCLINK_KT_KDOC_LINK); + ASSERT_NULL(dm_find_token(r, "the helper")); + ASSERT_EQ(jt_token(r, "Outer.Inner")->line, 5); + ASSERT_EQ(jt_token(r, "`quoted name`")->line, 5); + /* a block comment inside the doc is part of its text: Kotlin comments + * nest, and the doc goes on after it */ + ASSERT_EQ(jt_token(r, "StillDoc")->line, 8); + ASSERT_EQ(jt_token(r, "AfterNested")->line, 8); + ASSERT_EQ(jt_token(r, "Gadget.spin")->syntax, CBM_DOCLINK_KT_SEE); + ASSERT_EQ(jt_token(r, "IllegalStateException")->syntax, CBM_DOCLINK_KT_THROWS); + ASSERT_EQ(jt_token(r, "CustomError")->syntax, CBM_DOCLINK_KT_THROWS); + ASSERT_EQ(jt_token(r, "CustomError")->line, 20); + ASSERT_EQ(jt_token(r, "k.samples.widgetSample")->syntax, CBM_DOCLINK_KT_SAMPLE); + ASSERT_EQ(jt_token(r, "Result")->line, 22); + /* own parameter and type parameter: tokens, local to the resolver */ + ASSERT_EQ(jt_token(r, "name")->line, 15); + ASSERT_EQ(jt_token(r, "T")->line, 15); + /* not references: a Markdown link, code spans, a link definition, an + * HTML comment, an image's label, an escaped bracket, an indented and a + * fenced code block, a URL, and brackets that hold no name */ + ASSERT_NULL(dm_find_token(r, "text")); + ASSERT_NULL(dm_find_token(r, "InCode")); + ASSERT_NULL(dm_find_token(r, "InDouble")); + ASSERT_NULL(dm_find_token(r, "label")); + ASSERT_NULL(dm_find_token(r, "InComment")); + ASSERT_NULL(dm_find_token(r, "image")); + ASSERT_NULL(dm_find_token(r, "ImgLabel")); + ASSERT_NULL(dm_find_token(r, "Escaped")); + ASSERT_NULL(dm_find_token(r, "IndentedCode")); + ASSERT_NULL(dm_find_token(r, "Fenced")); + ASSERT_NULL(dm_find_token(r, "https://example.org")); + ASSERT_NULL(dm_find_token(r, "0")); + ASSERT_NULL(dm_find_token(r, "two words")); + ASSERT_NULL(dm_find_token(r, "size")); + cbm_free_result(r); + PASS(); +} + +/* ── extraction: hostile input ───────────────────────────────────── */ + +enum { JT_BIG = 200000, JT_MEGA = 1100000 }; + +/* Extract `head` + `count` copies of `unit` + `tail` as one doc comment on a + * class; the result, or NULL when memory ran out. */ +static CBMFileResult *jt_extract_repeated(CBMLanguage lang, const char *head, const char *unit, + int count, const char *tail) { + size_t ul = strlen(unit); + size_t cap = strlen(head) + (ul * (size_t)count) + strlen(tail) + 64; + char *src = malloc(cap); + if (!src) { + return NULL; + } + size_t w = (size_t)snprintf(src, cap, "/**\n * %s", head); + for (int i = 0; i < count; i++) { + memcpy(src + w, unit, ul); + w += ul; + } + snprintf(src + w, cap - w, "%s\n */\n%s", tail, + lang == CBM_LANG_JAVA ? "public class C {}\n" : "class C\n"); + CBMFileResult *r = dm_extract(src, lang, lang == CBM_LANG_JAVA ? "C.java" : "C.kt"); + free(src); + return r; +} + +TEST(doc_mentions_jvm_malformed_and_oversized) { + /* a reference longer than any name (here: over a megabyte) is kept just + * past the limit, which is how the resolver knows it was cut: it never + * resolves a prefix */ + CBMFileResult *r = jt_extract_repeated(CBM_LANG_JAVA, "{@link ", "A", JT_MEGA, "}"); + ASSERT_NOT_NULL(r); + ASSERT_EQ(r->doc_links.count, 1); + ASSERT_GT(strlen(r->doc_links.items[0].raw), (size_t)CBM_DOCLINK_JVM_REF_MAX); + ASSERT(strlen(r->doc_links.items[0].raw) < (size_t)CBM_DOCLINK_JVM_REF_MAX + 16); + cbm_free_result(r); + r = jt_extract_repeated(CBM_LANG_KOTLIN, "[", "A", JT_MEGA, "]"); + ASSERT_NOT_NULL(r); + ASSERT_EQ(r->doc_links.count, 1); + ASSERT_GT(strlen(r->doc_links.items[0].raw), (size_t)CBM_DOCLINK_JVM_REF_MAX); + ASSERT(strlen(r->doc_links.items[0].raw) < (size_t)CBM_DOCLINK_JVM_REF_MAX + 16); + cbm_free_result(r); + /* tags that never close, nested deeper than any text nests */ + r = jt_extract_repeated(CBM_LANG_JAVA, "", "{@link A ", JT_BIG, ""); + ASSERT_NOT_NULL(r); + ASSERT_EQ(r->doc_links.count, 0); + cbm_free_result(r); + /* ... and closed again: the innermost ones are references, no more than + * the parser keeps open at once, and nothing overflows */ + r = jt_extract_repeated(CBM_LANG_JAVA, "", "{@link A ", 100, "}}}}}}}}}}"); + ASSERT_NOT_NULL(r); + ASSERT(r->doc_links.count <= 10); + cbm_free_result(r); + /* brackets, backticks and comment openers by the hundred thousand on one + * line: one pass, no token */ + r = jt_extract_repeated(CBM_LANG_KOTLIN, "", "[", JT_BIG, ""); + ASSERT_NOT_NULL(r); + ASSERT_EQ(r->doc_links.count, 0); + cbm_free_result(r); + r = jt_extract_repeated(CBM_LANG_KOTLIN, "", "` `` ", JT_BIG, " [After]"); + ASSERT_NOT_NULL(r); + cbm_free_result(r); + r = jt_extract_repeated(CBM_LANG_KOTLIN, "[Before] ", " */ + RS_HTML_PI = 3, /* until ?> */ + RS_HTML_DECL = 4, /* until > */ + RS_HTML_CDATA = 5, /* until ]]> */ + RS_HTML_BLOCK = 6, /* a block tag: until a blank line */ + RS_HTML_ANY = 7, /* any single tag alone on its line: until a blank line */ +}; + +typedef struct { + const char *label; /* normalized: trimmed, whitespace collapsed, ASCII lower case */ + const char *dest; + uint32_t dest_len; + int order; +} rs_def_t; + +typedef struct { + CBMArena *a; + rs_def_t *v; + int n; + int cap; + bool failed; +} rs_defs_t; + +typedef struct { + bool fence; + char fence_ch; + uint32_t fence_len; + int html; /* RS_HTML_* of the open HTML block */ + bool para; + bool quoted; /* the open paragraph is inside a block quote */ + CBMArena *a; + int *list; /* content column of every open list item */ + int nlist; + int list_cap; + bool failed; /* out of memory: the doc text gets no tokens */ +} rs_md_t; + +/* A cursor in one line: byte position and the column it stands at. */ +typedef struct { + const char *p; + uint32_t n; + uint32_t pos; + int col; +} rs_cur_t; + +static void rs_cur_skip_ws(rs_cur_t *c) { + while (c->pos < c->n && (c->p[c->pos] == ' ' || c->p[c->pos] == '\t')) { + c->col += c->p[c->pos] == '\t' ? RS_MD_TAB - (c->col % RS_MD_TAB) : SKIP_ONE; + c->pos++; + } +} + +static bool rs_cur_eol(const rs_cur_t *c) { + return rs_blank(c->p + c->pos, c->n - c->pos); +} + +/* An opening code fence at the cursor: its character and length. */ +static bool rs_md_fence_open(const rs_cur_t *c, char *ch, uint32_t *len) { + if (c->pos >= c->n || (c->p[c->pos] != '`' && c->p[c->pos] != '~')) { + return false; + } + char f = c->p[c->pos]; + uint32_t k = c->pos; + while (k < c->n && c->p[k] == f) { + k++; + } + if (k - c->pos < RS_MD_FENCE_MIN) { + return false; + } + if (f == '`' && memchr(c->p + k, '`', c->n - k)) { + return false; /* the info string of a backtick fence holds no backtick */ + } + *ch = f; + *len = k - c->pos; + return true; +} + +static bool rs_md_fence_close(const rs_line_t *ln, const rs_md_t *st) { + rs_cur_t c = {ln->p, ln->len, 0, 0}; + rs_cur_skip_ws(&c); + uint32_t k = c.pos; + while (k < c.n && c.p[k] == st->fence_ch) { + k++; + } + return k - c.pos >= st->fence_len && rs_blank(c.p + k, c.n - k); +} + +/* `---`, `***`, `___` (three or more, spaces allowed between). */ +static bool rs_md_thematic(const rs_cur_t *c) { + if (c->pos >= c->n) { + return false; + } + char m = c->p[c->pos]; + if (m != '-' && m != '*' && m != '_') { + return false; + } + int count = 0; + for (uint32_t k = c->pos; k < c->n; k++) { + char ch = c->p[k]; + if (ch == m) { + count++; + } else if (ch != ' ' && ch != '\t' && ch != '\r') { + return false; + } + } + return count >= RS_MD_FENCE_MIN; +} + +/* `# heading`: the cursor moves to the heading's text. */ +static bool rs_md_atx(rs_cur_t *c) { + uint32_t k = c->pos; + while (k < c->n && c->p[k] == '#') { + k++; + } + uint32_t hashes = k - c->pos; + if (hashes == 0 || hashes > RS_MD_ATX_MAX || (k < c->n && c->p[k] != ' ' && c->p[k] != '\t')) { + return false; + } + c->col += (int)hashes; + c->pos = k; + return true; +} + +/* A list marker at the cursor: its width in bytes, 0 when there is none. + * *first is false for an ordered marker other than `1`, which cannot + * interrupt a paragraph. */ +static uint32_t rs_md_list_marker(const rs_cur_t *c, bool *first) { + *first = true; + uint32_t k = c->pos; + if (k >= c->n) { + return 0; + } + char m = c->p[k]; + if (m == '-' || m == '+' || m == '*') { + k++; + } else if (isdigit((unsigned char)m)) { + while (k < c->n && isdigit((unsigned char)c->p[k]) && k - c->pos < RS_MD_ORDERED_DIGITS) { + k++; + } + if (k >= c->n || (c->p[k] != '.' && c->p[k] != ')')) { + return 0; + } + *first = k - c->pos == SKIP_ONE && m == '1'; + k++; + } else { + return 0; + } + if (k < c->n && c->p[k] != ' ' && c->p[k] != '\t' && c->p[k] != '\r') { + return 0; + } + return k - c->pos; +} + +static bool rs_ci_prefix(const char *p, uint32_t n, const char *word) { + size_t wl = strlen(word); + if (n < wl) { + return false; + } + for (size_t i = 0; i < wl; i++) { + if (tolower((unsigned char)p[i]) != word[i]) { + return false; + } + } + return true; +} + +static bool rs_ci_find(const char *p, uint32_t n, const char *word) { + size_t wl = strlen(word); + for (uint32_t i = 0; i + wl <= n; i++) { + if (rs_ci_prefix(p + i, n - i, word)) { + return true; + } + } + return false; +} + +static bool rs_html_block_tag(const char *name, size_t len) { + static const char *const tags[] = { + "address", "article", "aside", "base", "basefont", "blockquote", "body", + "caption", "center", "col", "colgroup", "dd", "details", "dialog", + "dir", "div", "dl", "dt", "fieldset", "figcaption", "figure", + "footer", "form", "frame", "frameset", "h1", "h2", "h3", + "h4", "h5", "h6", "head", "header", "hr", "html", + "iframe", "legend", "li", "link", "main", "menu", "menuitem", + "nav", "noframes", "ol", "optgroup", "option", "p", "param", + "search", "section", "summary", "table", "tbody", "td", "tfoot", + "th", "thead", "title", "tr", "track", "ul", + }; + char low[RS_MD_TAG_NAME_MAX]; + if (len == 0 || len >= sizeof(low)) { + return false; + } + for (size_t i = 0; i < len; i++) { + low[i] = (char)tolower((unsigned char)name[i]); + } + low[len] = '\0'; + for (size_t i = 0; i < sizeof(tags) / sizeof(tags[0]); i++) { + if (strcmp(tags[i], low) == 0) { + return true; + } + } + return false; +} + +/* End of an HTML tag that starts at p[0] == '<' (a quoted attribute value may + * hold '>'): index just past its '>', or 0 when the line has none. */ +static uint32_t rs_html_tag_end(const char *p, uint32_t n) { + char quote = 0; + for (uint32_t i = SKIP_ONE; i < n; i++) { + char c = p[i]; + if (quote) { + quote = c == quote ? 0 : quote; + } else if (c == '"' || c == '\'') { + quote = c; + } else if (c == '>') { + return i + SKIP_ONE; + } else if (c == '<') { + return 0; + } + } + return 0; +} + +/* The HTML block that starts at the cursor (RS_HTML_*), none inside a + * paragraph for the kind that cannot interrupt one. */ +static int rs_md_html_start(const rs_cur_t *c, bool in_para) { + const char *p = c->p + c->pos; + uint32_t n = c->n - c->pos; + if (n < PAIR_LEN || p[0] != '<') { + return RS_HTML_NONE; + } + static const char *const raw[] = {"' || p[rl] == '\r')) { + return RS_HTML_RAW; + } + } + if (rs_ci_prefix(p, n, ""); + case RS_HTML_PI: + return rs_ci_find(ln->p, ln->len, "?>"); + case RS_HTML_DECL: + return memchr(ln->p, '>', ln->len) != NULL; + case RS_HTML_CDATA: + return rs_ci_find(ln->p, ln->len, "]]>"); + default: + return false; + } +} + +/* ── Link reference definitions ──────────────────────────────────── */ + +/* A link label, normalized into the arena: trimmed, inner whitespace + * collapsed, ASCII letters lower-cased. NULL when it holds no text. */ +static const char *rs_label_norm(CBMArena *a, const char *s, size_t n) { + char *out = (char *)cbm_arena_alloc(a, n + SKIP_ONE); + if (!out) { + return NULL; + } + size_t w = 0; + bool space = false; + for (size_t i = 0; i < n; i++) { + unsigned char c = (unsigned char)s[i]; + if (isspace(c)) { + space = w > 0; + continue; + } + if (space) { + out[w++] = ' '; + space = false; + } + out[w++] = (char)tolower(c); + } + out[w] = '\0'; + return w > 0 ? out : NULL; +} + +/* End of the label that starts after the '[' at s[from - 1]: the index of its + * closing ']', or UINT32_MAX when the text is no label (an unescaped bracket + * inside, too long, or unclosed). */ +static uint32_t rs_label_end(const char *s, uint32_t n, uint32_t from) { + for (uint32_t i = from; i < n && i - from <= RS_MD_LABEL_MAX; i++) { + char c = s[i]; + if (c == '\\' && i + SKIP_ONE < n) { + i++; + } else if (c == '[') { + return UINT32_MAX; + } else if (c == ']') { + return i; + } + } + return UINT32_MAX; +} + +/* True when s[0..n) can be a link label: some text, no unescaped bracket, not + * longer than CommonMark allows. */ +static bool rs_label_clean(const char *s, uint32_t n) { + if (n == 0 || n > RS_MD_LABEL_MAX || rs_blank(s, n)) { + return false; + } + for (uint32_t i = 0; i < n; i++) { + if (s[i] == '\\' && i + SKIP_ONE < n) { + i++; + } else if (s[i] == '[' || s[i] == ']') { + return false; + } + } + return true; +} + +/* A link destination at s[*pos]: `<...>` or a run without spaces and with + * balanced parentheses. On success *pos moves past it. */ +static bool rs_link_dest(const char *s, uint32_t n, uint32_t *pos, const char **dest, + uint32_t *dest_len) { + uint32_t i = *pos; + if (i < n && s[i] == '<') { + uint32_t k = i + SKIP_ONE; + while (k < n && s[k] != '>' && s[k] != '<' && s[k] != '\n') { + k += (s[k] == '\\' && k + SKIP_ONE < n) ? PAIR_LEN : SKIP_ONE; + } + if (k >= n || s[k] != '>') { + return false; + } + *dest = s + i + SKIP_ONE; + *dest_len = k - i - SKIP_ONE; + *pos = k + SKIP_ONE; + return true; + } + int depth = 0; + uint32_t k = i; + while (k < n) { + unsigned char c = (unsigned char)s[k]; + if (c == '\\' && k + SKIP_ONE < n) { + k += PAIR_LEN; + continue; + } + if (c <= ' ') { + break; + } + if (c == '(') { + depth++; + } else if (c == ')') { + if (depth == 0) { + break; + } + depth--; + } + k++; + } + if (depth != 0) { + return false; + } + *dest = s + i; + *dest_len = k - i; + *pos = k; + return true; +} + +/* An optional link title at s[*pos] (`"..."`, `'...'`, `(...)`): *pos moves + * past it. false when a title starts and does not end. */ +static bool rs_link_title(const char *s, uint32_t n, uint32_t *pos) { + uint32_t i = *pos; + if (i >= n || (s[i] != '"' && s[i] != '\'' && s[i] != '(')) { + return true; + } + char close = s[i] == '(' ? ')' : s[i]; + for (uint32_t k = i + SKIP_ONE; k < n; k++) { + if (s[k] == '\\' && k + SKIP_ONE < n) { + k++; + } else if (s[k] == close) { + *pos = k + SKIP_ONE; + return true; + } + } + return false; +} + +static void rs_defs_push(rs_defs_t *d, rs_def_t def) { + if (d->n >= d->cap) { + int ncap = d->cap ? d->cap * PAIR_LEN : CBM_SZ_8; + rs_def_t *grown = (rs_def_t *)cbm_arena_alloc(d->a, (size_t)ncap * sizeof(rs_def_t)); + if (!grown) { + d->failed = true; + return; + } + if (d->n > 0) { + memcpy(grown, d->v, (size_t)d->n * sizeof(rs_def_t)); + } + d->v = grown; + d->cap = ncap; + } + def.order = d->n; + d->v[d->n++] = def; +} + +static uint32_t rs_skip_spaces(const char *s, uint32_t n, uint32_t i) { + while (i < n && (s[i] == ' ' || s[i] == '\t' || s[i] == '\r')) { + i++; + } + return i; +} + +/* A link reference definition `[label]: dest "title"` that starts at the + * cursor of lines[i]; the destination may stand on the next line. Returns the + * number of lines it takes (0: this is no definition). */ +static int rs_md_definition(rs_line_t *lines, int n, int i, const rs_cur_t *c, rs_defs_t *defs) { + const char *s = c->p; + uint32_t len = c->n; + uint32_t pos = c->pos; + if (pos >= len || s[pos] != '[' || (pos + SKIP_ONE < len && s[pos + SKIP_ONE] == '^')) { + return 0; + } + uint32_t close = rs_label_end(s, len, pos + SKIP_ONE); + if (close == UINT32_MAX || close + SKIP_ONE >= len || s[close + SKIP_ONE] != ':') { + return 0; + } + const char *label = rs_label_norm(defs->a, s + pos + SKIP_ONE, close - pos - SKIP_ONE); + if (!label) { + return 0; + } + int taken = SKIP_ONE; + uint32_t k = rs_skip_spaces(s, len, close + PAIR_LEN); + if (k >= len) { + /* the destination is on the next line */ + if (i + SKIP_ONE >= n || rs_blank(lines[i + SKIP_ONE].p, lines[i + SKIP_ONE].len)) { + return 0; + } + s = lines[i + SKIP_ONE].p; + len = lines[i + SKIP_ONE].len; + k = rs_skip_spaces(s, len, 0); + taken = PAIR_LEN; + } + const char *dest = NULL; + uint32_t dest_len = 0; + if (!rs_link_dest(s, len, &k, &dest, &dest_len) || dest_len == 0) { + return 0; + } + uint32_t after = rs_skip_spaces(s, len, k); + if (after < len && + (after == k || !rs_link_title(s, len, &after) || rs_skip_spaces(s, len, after) < len)) { + return 0; /* something that is no title follows the destination */ + } + rs_defs_push(defs, (rs_def_t){.label = label, .dest = dest, .dest_len = dest_len}); + return taken; +} + +static int rs_def_cmp(const void *a, const void *b) { + const rs_def_t *x = (const rs_def_t *)a; + const rs_def_t *y = (const rs_def_t *)b; + int c = strcmp(x->label, y->label); + return c ? c : (x->order > y->order) - (x->order < y->order); +} + +/* The first definition of `label` (normalized), or NULL. */ +static const rs_def_t *rs_def_find(const rs_defs_t *d, const char *label) { + int lo = 0; + int hi = d->n; + while (lo < hi) { + int mid = lo + ((hi - lo) / PAIR_LEN); + if (strcmp(d->v[mid].label, label) < 0) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + return (lo < d->n && strcmp(d->v[lo].label, label) == 0) ? &d->v[lo] : NULL; +} + +/* ── Markdown blocks: one line ───────────────────────────────────── */ + +static void rs_md_pop_lists(rs_md_t *st, int col) { + while (st->nlist > 0 && st->list[st->nlist - SKIP_ONE] > col) { + st->nlist--; + } +} + +/* Column the text of a line is measured from: the content column of the + * innermost open list item it reaches. The stack is strictly increasing (an + * item opens to the right of everything still open), so this is a binary + * search. */ +static int rs_md_base(const rs_md_t *st, int col) { + int lo = 0; + int hi = st->nlist; + while (lo < hi) { + int mid = lo + ((hi - lo) / PAIR_LEN); + if (st->list[mid] <= col) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + return lo > 0 ? st->list[lo - SKIP_ONE] : 0; +} + +static void rs_md_push_list(rs_md_t *st, int content) { + if (st->nlist >= st->list_cap) { + int ncap = st->list_cap ? st->list_cap * PAIR_LEN : RS_MD_LIST_INIT; + int *grown = (int *)cbm_arena_alloc(st->a, (size_t)ncap * sizeof(int)); + if (!grown) { + st->failed = true; + return; + } + if (st->nlist > 0) { + memcpy(grown, st->list, (size_t)st->nlist * sizeof(int)); + } + st->list = grown; + st->list_cap = ncap; + } + st->list[st->nlist++] = content; +} + +/* Block-quote markers at the cursor; true when there was one. */ +static bool rs_md_quote(rs_cur_t *c) { + bool quoted = false; + while (c->pos < c->n && c->p[c->pos] == '>') { + quoted = true; + c->pos++; + c->col++; + if (c->pos < c->n && c->p[c->pos] == ' ') { + c->pos++; + c->col++; + } + rs_cur_skip_ws(c); + } + return quoted; +} + +/* Open a list item whose marker is `width` bytes at the cursor; the cursor + * moves to the item's content (past a task marker). */ +static void rs_md_open_item(rs_md_t *st, rs_cur_t *c, uint32_t width) { + c->pos += width; + c->col += (int)width; + int marker_end = c->col; + rs_cur_skip_ws(c); + int content = c->col; + if (rs_cur_eol(c) || content - marker_end > RS_MD_CODE_INDENT) { + content = marker_end + SKIP_ONE; + } + rs_md_push_list(st, content); + /* a task marker `[ ]` / `[x]` is no link */ + const char *p = c->p + c->pos; + uint32_t left = c->n - c->pos; + if (left > RS_MARK_LEN && p[0] == '[' && (p[1] == ' ' || p[1] == 'x' || p[1] == 'X') && + p[PAIR_LEN] == ']' && (p[RS_MARK_LEN] == ' ' || p[RS_MARK_LEN] == '\t')) { + c->pos += RS_MARK_LEN; + c->col += RS_MARK_LEN; + } +} + +/* What starts at the cursor. */ +enum { RS_BLK_TEXT = 0, RS_BLK_FENCE, RS_BLK_HTML, RS_BLK_RULE, RS_BLK_HEADING, RS_BLK_ITEM }; + +typedef struct { + int kind; + char fence_ch; + uint32_t fence_len; + int html; + uint32_t marker; /* width of a list marker */ + bool interrupts; /* may end an open paragraph */ +} rs_blk_t; + +static rs_blk_t rs_md_block_at(const rs_cur_t *c, bool in_para) { + rs_blk_t b = {.kind = RS_BLK_TEXT, .interrupts = true}; + if (rs_md_fence_open(c, &b.fence_ch, &b.fence_len)) { + b.kind = RS_BLK_FENCE; + return b; + } + b.html = rs_md_html_start(c, in_para); + if (b.html != RS_HTML_NONE) { + b.kind = RS_BLK_HTML; + return b; + } + if (rs_md_thematic(c)) { + b.kind = RS_BLK_RULE; + return b; + } + rs_cur_t probe = *c; + if (rs_md_atx(&probe)) { + b.kind = RS_BLK_HEADING; + return b; + } + bool first = true; + b.marker = rs_md_list_marker(c, &first); + if (b.marker > 0) { + b.kind = RS_BLK_ITEM; + /* an empty item, or an ordered one not numbered 1, continues a paragraph */ + rs_cur_t rest = *c; + rest.pos += b.marker; + b.interrupts = first && !rs_cur_eol(&rest); + } + return b; +} + +/* A line of an open fenced block or HTML block, or a blank line: classified + * here. false for any other line. */ +static bool rs_md_line_in_block(rs_md_t *st, rs_line_t *ln) { + if (st->fence) { + st->fence = !rs_md_fence_close(ln, st); + return true; + } + bool blank = rs_blank(ln->p, ln->len); + if (st->html != RS_HTML_NONE) { + if (st->html >= RS_HTML_BLOCK ? blank : rs_md_html_end(ln, st->html)) { + st->html = RS_HTML_NONE; + } + ln->kind = blank ? RS_L_BLANK : RS_L_CODE; + return true; + } + if (blank) { + ln->kind = RS_L_BLANK; + st->para = false; + } + return blank; +} + +/* The line is inline text from the cursor on. */ +static void rs_md_text(rs_line_t *ln, const rs_cur_t *c, bool para_start) { + ln->kind = RS_L_TEXT; + ln->para_start = para_start; + ln->len -= c->pos; + ln->p += c->pos; +} + +/* What is no list item starts at the cursor of lines[i]. Returns the lines it + * takes, or 0 when the rest of the line is inline text (a heading's, or a + * paragraph's first line). */ +static int rs_md_start_block(rs_md_t *st, rs_line_t *lines, int n, int i, rs_cur_t *c, + const rs_blk_t *b, bool quoted, rs_defs_t *defs) { + rs_line_t *ln = &lines[i]; + if (b->kind == RS_BLK_FENCE) { + st->fence = true; + st->fence_ch = b->fence_ch; + st->fence_len = b->fence_len; + return SKIP_ONE; + } + if (b->kind == RS_BLK_HTML) { + bool closed = b->html < RS_HTML_BLOCK && rs_md_html_end(ln, b->html); + st->html = closed ? RS_HTML_NONE : b->html; + return SKIP_ONE; + } + if (b->kind == RS_BLK_RULE) { + ln->kind = RS_L_BLANK; + return SKIP_ONE; + } + if (b->kind == RS_BLK_HEADING) { + (void)rs_md_atx(c); + return 0; + } + int taken = rs_md_definition(lines, n, i, c, defs); + if (taken > 0) { + ln->kind = RS_L_DEF; + if (taken > SKIP_ONE) { + lines[i + SKIP_ONE].kind = RS_L_DEF; + } + return taken; + } + st->para = true; + st->quoted = quoted; + return 0; +} + +/* Classify lines[i] and strip its block markers. Returns how many lines were + * consumed (2 when a link definition's destination is on the next line). */ +static int rs_md_line(rs_md_t *st, rs_line_t *lines, int n, int i, rs_defs_t *defs) { + rs_line_t *ln = &lines[i]; + ln->kind = RS_L_CODE; + if (rs_md_line_in_block(st, ln)) { + return SKIP_ONE; + } + rs_cur_t c = {ln->p, ln->len, 0, 0}; + rs_cur_skip_ws(&c); + bool quoted = rs_md_quote(&c); + if (quoted && !st->quoted) { + st->para = false; /* a block quote interrupts a paragraph */ + } + /* every round opens one list item, which moves the cursor on */ + for (;;) { + rs_blk_t b = rs_md_block_at(&c, st->para); + if (st->para) { + bool breaks = b.kind != RS_BLK_TEXT && b.interrupts && + c.col - rs_md_base(st, c.col) < RS_MD_CODE_INDENT; + if (!breaks) { + rs_md_text(ln, &c, false); /* the paragraph goes on, however indented */ + return SKIP_ONE; + } + st->para = false; + } + rs_md_pop_lists(st, c.col); + if (c.col - rs_md_base(st, c.col) >= RS_MD_CODE_INDENT) { + return SKIP_ONE; /* an indented code block */ + } + if (b.kind != RS_BLK_ITEM) { + int taken = rs_md_start_block(st, lines, n, i, &c, &b, quoted, defs); + if (taken > 0) { + return taken; + } + break; + } + rs_md_open_item(st, &c, b.marker); + if (rs_cur_eol(&c)) { + ln->kind = RS_L_BLANK; + return SKIP_ONE; + } + } + rs_md_text(ln, &c, true); + return SKIP_ONE; +} + +/* ── Inline links ────────────────────────────────────────────────── */ + +enum { RS_LINK_INLINE = 0, RS_LINK_REFERENCE, RS_LINK_COLLAPSED, RS_LINK_SHORTCUT }; + +typedef struct { + uint32_t pos; + bool image; +} rs_opener_t; + +/* A maximal run of backticks. */ +typedef struct { + uint32_t pos; + uint32_t len; +} rs_run_t; + +/* What a found link is handed to. */ +typedef struct { + CBMExtractCtx *ctx; + const char *source_qn; + uint32_t def_line; + const rs_defs_t *defs; + const rs_line_t *lines; /* the paragraph's lines */ + int nlines; + int line_cursor; +} rs_sink_t; + +/* Inline state of one paragraph. The arrays are allocated once per doc text, + * large enough for any paragraph of it (one slot per '[' and per backtick), so + * nothing here has a limit a long doc could hit. */ +typedef struct { + const char *s; /* the paragraph */ + uint32_t n; + rs_opener_t *open; /* the '[' not yet closed */ + int nopen; + int dead; /* openers below this index are inside a link: they open none */ + rs_run_t *runs; /* backtick runs in text order */ + rs_run_t *by_len; /* the same, sorted by (length, position) */ + int nruns; + int run_cursor; /* first run that ends after the scan position */ + bool no_comment_end; /* no later `-->` / `]]>` / `?>`: searched once */ + bool no_cdata_end; + bool no_pi_end; + rs_sink_t *sink; +} rs_inline_t; + +/* Source line of paragraph offset `off`. Links are reported in the order of + * their opening brackets, so the cursor rarely moves back. */ +static uint32_t rs_sink_line(rs_sink_t *k, uint32_t off) { + if (k->lines[k->line_cursor].boff > off) { + k->line_cursor = 0; + } + while (k->line_cursor + SKIP_ONE < k->nlines && + k->lines[k->line_cursor + SKIP_ONE].boff <= off) { + k->line_cursor++; + } + return k->lines[k->line_cursor].line; +} + +static bool rs_is_url(const char *s, size_t n) { + static const char mailto[] = "mailto:"; + static const char sep[] = "://"; + if (rs_ci_prefix(s, (uint32_t)n, mailto)) { + return true; + } + size_t i = 0; + if (n == 0 || !isalpha((unsigned char)s[0])) { + return false; + } + while (i < n && (isalnum((unsigned char)s[i]) || s[i] == '+' || s[i] == '.' || s[i] == '-')) { + i++; + } + return i + sizeof(sep) - SKIP_ONE <= n && memcmp(s + i, sep, sizeof(sep) - SKIP_ONE) == 0; +} + +static void rs_push_token(rs_sink_t *k, const char *raw, size_t n, int syntax, uint32_t line) { + static const char cut[] = "..."; + char buf[CBM_DOCLINK_RS_RAW_CAP]; + if (n >= sizeof(buf) - sizeof(cut)) { + /* oversized: a prefix that can never parse as a path */ + n = sizeof(buf) - sizeof(cut) - SKIP_ONE; + memcpy(buf, raw, n); + memcpy(buf + n, cut, sizeof(cut)); + n += sizeof(cut) - SKIP_ONE; + } else { + memcpy(buf, raw, n); + buf[n] = '\0'; + } + for (size_t i = 0; i < n; i++) { + if (buf[i] == '\n' || buf[i] == '\t' || buf[i] == '\r' || buf[i] == '\0') { + buf[i] = ' '; + } + } + const char *copy = cbm_arena_strndup(k->ctx->arena, buf, n); + if (!copy) { + return; + } + CBMDocLink link = { + .source_qn = k->source_qn, + .raw = copy, + .line = line, + .def_line = k->def_line, + .syntax = (uint16_t)syntax, + }; + cbm_doclinks_push(&k->ctx->result->doc_links, k->ctx->arena, link); +} + +/* A found link: decide whether rustdoc treats its destination as an intra-doc + * link, and push the token. `defined`: the destination was written (inline) + * or comes from a definition, so it may be an ordinary URL. */ +static void rs_emit_link(rs_sink_t *k, int kind, const char *dest, uint32_t dest_len, bool from_def, + uint32_t off) { + while (dest_len > 0 && isspace((unsigned char)dest[0])) { + dest++; + dest_len--; + } + while (dest_len > 0 && isspace((unsigned char)dest[dest_len - SKIP_ONE])) { + dest_len--; + } + if (dest_len == 0) { + return; + } + uint32_t line = rs_sink_line(k, off); + bool written = kind == RS_LINK_INLINE || from_def; /* rustdoc's can_be_url */ + int family = from_def ? CBM_DOCLINK_RS_LINK_DEF : CBM_DOCLINK_RS_INTRA_DOC; + if (written && memchr(dest, '/', dest_len)) { + if (rs_is_url(dest, dest_len)) { + rs_push_token(k, dest, dest_len, CBM_DOCLINK_HREF, line); + } + return; /* a relative file is not harvested */ + } + /* backticks go, the fragment is cut */ + char clean[CBM_DOCLINK_RS_RAW_CAP]; + size_t w = 0; + int hashes = 0; + bool backtick = false; + for (uint32_t i = 0; i < dest_len; i++) { + if (dest[i] == '`') { + backtick = true; + continue; + } + hashes += dest[i] == '#'; + if (w + SKIP_ONE < sizeof(clean)) { + clean[w++] = dest[i]; + } + } + clean[w] = '\0'; + if (w + SKIP_ONE >= sizeof(clean)) { + rs_push_token(k, clean, w, family, line); /* oversized */ + return; + } + /* rustdoc looks at the text before the first '#' first: empty means an + * anchor of this page, whatever follows. A second '#' is then an error. */ + size_t path_len = hashes > 0 ? (size_t)((const char *)memchr(clean, '#', w) - clean) : w; + while (path_len > 0 && isspace((unsigned char)clean[path_len - SKIP_ONE])) { + path_len--; + } + if (path_len == 0) { + return; /* an anchor of this page */ + } + if (hashes > SKIP_ONE) { + path_len = w; /* the row shows the whole destination */ + } + bool lenient = written || (kind == RS_LINK_SHORTCUT && !backtick); + CBMDocLinkRsPath path; + CBMDocLinkRsPathStatus st = hashes > SKIP_ONE + ? CBM_DOCLINK_RS_PATH_MALFORMED + : cbm_doclink_rs_parse_path(clean, path_len, lenient, &path); + if (st == CBM_DOCLINK_RS_PATH_NOT_LINK) { + if (written && rs_is_url(dest, dest_len)) { + rs_push_token(k, dest, dest_len, CBM_DOCLINK_HREF, line); + } + return; + } + rs_push_token(k, clean, path_len, family, line); +} + +/* A reference-style link whose label is s[0..n): its destination is the + * definition's, or (rustdoc's broken-link callback) the label itself. */ +static void rs_emit_reference(rs_sink_t *k, int kind, const char *label, uint32_t n, uint32_t off) { + const char *norm = rs_label_norm(rs_scratch(k->ctx), label, n); + if (!norm || norm[0] == '^') { + return; /* empty, or a footnote */ + } + const rs_def_t *def = rs_def_find(k->defs, norm); + if (def) { + rs_emit_link(k, kind, def->dest, def->dest_len, true, off); + } else { + rs_emit_link(k, kind, label, n, false, off); + } +} + +static int rs_run_len_cmp(const void *a, const void *b) { + const rs_run_t *x = (const rs_run_t *)a; + const rs_run_t *y = (const rs_run_t *)b; + if (x->len != y->len) { + return x->len < y->len ? -1 : 1; + } + return (x->pos > y->pos) - (x->pos < y->pos); +} + +/* Record the paragraph's backtick runs, in text order and by length: a code + * span closes at the first later run of its opener's length, which is then a + * binary search instead of a scan of the rest of the paragraph. */ +static void rs_inline_runs(rs_inline_t *in) { + in->nruns = 0; + in->run_cursor = 0; + for (uint32_t i = 0; i < in->n;) { + if (in->s[i] != '`') { + i++; + continue; + } + uint32_t e = i; + while (e < in->n && in->s[e] == '`') { + e++; + } + in->runs[in->nruns++] = (rs_run_t){.pos = i, .len = e - i}; + i = e; + } + if (in->nruns > 0) { + memcpy(in->by_len, in->runs, (size_t)in->nruns * sizeof(rs_run_t)); + qsort(in->by_len, (size_t)in->nruns, sizeof(rs_run_t), rs_run_len_cmp); + } +} + +/* Past the code span that opens at the backtick s[i], or past its run alone + * when no later run has the opener's length. The opener is what is left of + * its run from s[i] on (a backslash before the run takes its first backtick). */ +static uint32_t rs_inline_backticks(rs_inline_t *in, uint32_t i) { + while (in->run_cursor < in->nruns && + in->runs[in->run_cursor].pos + in->runs[in->run_cursor].len <= i) { + in->run_cursor++; + } + if (in->run_cursor >= in->nruns) { + return i + SKIP_ONE; + } + uint32_t run_end = in->runs[in->run_cursor].pos + in->runs[in->run_cursor].len; + rs_run_t want = {.pos = run_end, .len = run_end - i}; + int lo = 0; + int hi = in->nruns; + while (lo < hi) { + int mid = lo + ((hi - lo) / PAIR_LEN); + if (rs_run_len_cmp(&in->by_len[mid], &want) < 0) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + if (lo < in->nruns && in->by_len[lo].len == want.len) { + return in->by_len[lo].pos + want.len; + } + return run_end; +} + +/* Index just past the first `mark` at or after s[from], or 0. */ +static uint32_t rs_find_after(const rs_inline_t *in, uint32_t from, const char *mark) { + size_t ml = strlen(mark); + for (uint32_t k = from; k + ml <= in->n; k++) { + if (memcmp(in->s + k, mark, ml) == 0) { + return k + (uint32_t)ml; + } + } + return 0; +} + +/* Past the HTML comment, CDATA section or processing instruction that starts + * at s[i]; 0 when s[i] starts none of them. One whose end marker is missing is + * plain text, and so is every later one of its kind: the search for the marker + * runs once per paragraph. */ +static uint32_t rs_inline_raw_html(rs_inline_t *in, uint32_t i, bool *is_raw) { + static const char comment[] = ""; + none = &in->no_comment_end; + skip = sizeof(comment) - SKIP_ONE; + } else if (n >= sizeof(cdata) - SKIP_ONE && memcmp(p, cdata, sizeof(cdata) - SKIP_ONE) == 0) { + end_mark = "]]>"; + none = &in->no_cdata_end; + skip = sizeof(cdata) - SKIP_ONE; + } else if (n >= sizeof(pi) - SKIP_ONE && memcmp(p, pi, sizeof(pi) - SKIP_ONE) == 0) { + end_mark = "?>"; + none = &in->no_pi_end; + skip = sizeof(pi) - SKIP_ONE; + } + *is_raw = end_mark != NULL; + if (!end_mark || *none) { + return 0; + } + uint32_t end = rs_find_after(in, i + skip, end_mark); + *none = end == 0; + return end; +} + +/* Past the autolink or HTML construct at s[i] == '<', or i + 1 when it is + * neither. */ +static uint32_t rs_inline_angle(rs_inline_t *in, uint32_t i) { + const char *p = in->s + i; + uint32_t n = in->n - i; + if (n < PAIR_LEN) { + return i + SKIP_ONE; + } + bool is_raw = false; + uint32_t raw_end = rs_inline_raw_html(in, i, &is_raw); + if (is_raw) { + return raw_end ? raw_end : i + SKIP_ONE; + } + char c = p[1]; + if (!(isalpha((unsigned char)c) || c == '/' || c == '!')) { + return i + SKIP_ONE; + } + uint32_t end = rs_html_tag_end(p, n); + if (end == 0) { + return i + SKIP_ONE; + } + if (isalpha((unsigned char)c)) { + /* `` is a tag; `` and `` are autolinks; text + * with a space before any of that is neither */ + uint32_t k = SKIP_ONE; + while (k < end && + (isalnum((unsigned char)p[k]) || p[k] == '-' || p[k] == '+' || p[k] == '.')) { + k++; + } + bool tag = p[k] == '>' || p[k] == ' ' || p[k] == '\t' || p[k] == '\n' || p[k] == '/'; + bool autolink = (p[k] == ':' || p[k] == '@') && !memchr(p, ' ', end); + if (!tag && !autolink) { + return i + SKIP_ONE; + } + } + return i + end; +} + +/* The inline link `(dest "title")` at s[i] == '('. On success *end is past the + * closing parenthesis. */ +static bool rs_inline_dest(const rs_inline_t *in, uint32_t i, const char **dest, uint32_t *dest_len, + uint32_t *end) { + const char *s = in->s; + uint32_t n = in->n; + uint32_t k = i + SKIP_ONE; + while (k < n && isspace((unsigned char)s[k])) { + k++; + } + if (!rs_link_dest(s, n, &k, dest, dest_len)) { + return false; + } + uint32_t after = k; + while (after < n && isspace((unsigned char)s[after])) { + after++; + } + if (after > k && !rs_link_title(s, n, &after)) { + return false; + } + while (after < n && isspace((unsigned char)s[after])) { + after++; + } + if (after >= n || s[after] != ')') { + return false; + } + *end = after + SKIP_ONE; + return true; +} + +/* The reference-style link the ']' at s[i] ends, when the text after it or + * the bracket text itself is a label: full `[text][label]`, collapsed + * `[label][]`, shortcut `[label]`. *made is set when it is one; returns where + * scanning continues. */ +static uint32_t rs_inline_reference(rs_inline_t *in, const rs_opener_t *op, uint32_t i, + bool *made) { + const char *s = in->s; + const char *text = s + op->pos + SKIP_ONE; + uint32_t text_len = i - op->pos - SKIP_ONE; + uint32_t after = i + SKIP_ONE; + bool text_is_label = rs_label_clean(text, text_len); + if (after < in->n && s[after] == '[') { + uint32_t close = rs_label_end(s, in->n, after + SKIP_ONE); + if (close != UINT32_MAX && close > after + SKIP_ONE) { + *made = true; + if (!op->image) { + rs_emit_reference(in->sink, RS_LINK_REFERENCE, s + after + SKIP_ONE, + close - after - SKIP_ONE, op->pos); + } + return close + SKIP_ONE; + } + if (close == after + SKIP_ONE && text_is_label) { + *made = true; + if (!op->image) { + rs_emit_reference(in->sink, RS_LINK_COLLAPSED, text, text_len, op->pos); + } + return close + SKIP_ONE; + } + } + if (text_is_label) { + *made = true; + if (!op->image) { + rs_emit_reference(in->sink, RS_LINK_SHORTCUT, text, text_len, op->pos); + } + } + return i + SKIP_ONE; +} + +/* The ']' at s[i]: make a link of the bracket pair it closes, when it is one. + * Returns where scanning continues. */ +static uint32_t rs_inline_close(rs_inline_t *in, uint32_t i) { + if (in->nopen == 0) { + return i + SKIP_ONE; + } + int idx = --in->nopen; + rs_opener_t op = in->open[idx]; + bool dead = idx < in->dead && !op.image; /* inside a link: links do not nest */ + if (in->dead > in->nopen) { + in->dead = in->nopen; + } + if (dead) { + return i + SKIP_ONE; + } + uint32_t after = i + SKIP_ONE; + const char *dest = NULL; + uint32_t dest_len = 0; + uint32_t end = 0; + bool made = false; + if (after < in->n && in->s[after] == '(' && rs_inline_dest(in, after, &dest, &dest_len, &end)) { + made = true; + if (!op.image) { + rs_emit_link(in->sink, RS_LINK_INLINE, dest, dest_len, false, op.pos); + } + } else { + end = rs_inline_reference(in, &op, i, &made); + } + if (made && !op.image) { + in->dead = in->nopen; /* every '[' still open encloses this link */ + } + return end; +} + +/* Find the links of one paragraph. */ +static void rs_inline_scan(rs_inline_t *in) { + uint32_t i = 0; + while (i < in->n) { + char c = in->s[i]; + if (c == '\\' && i + SKIP_ONE < in->n && ispunct((unsigned char)in->s[i + SKIP_ONE])) { + i += PAIR_LEN; + } else if (c == '`') { + i = rs_inline_backticks(in, i); + } else if (c == '<') { + i = rs_inline_angle(in, i); + } else if (c == '[') { + bool image = i > 0 && in->s[i - SKIP_ONE] == '!' && + !(i > SKIP_ONE && in->s[i - PAIR_LEN] == '\\'); + in->open[in->nopen++] = (rs_opener_t){.pos = i, .image = image}; + i++; + } else if (c == ']') { + i = rs_inline_close(in, i); + } else { + i++; + } + } +} + +/* ── Tokens of one doc text ──────────────────────────────────────── */ + +/* Scan the paragraph lines[first..last] (RS_L_TEXT lines): their contents are + * joined by newlines in `buf`. */ +static void rs_scan_paragraph(rs_inline_t *in, rs_sink_t *sink, rs_line_t *lines, int first, + int last, char *buf) { + uint32_t w = 0; + for (int i = first; i <= last; i++) { + lines[i].boff = w; + memcpy(buf + w, lines[i].p, lines[i].len); + w += lines[i].len; + buf[w++] = '\n'; + } + in->s = buf; + in->n = w; + in->nopen = 0; + in->dead = 0; + in->no_comment_end = false; + in->no_cdata_end = false; + in->no_pi_end = false; + in->sink = sink; + sink->lines = lines + first; + sink->nlines = last - first + SKIP_ONE; + sink->line_cursor = 0; + rs_inline_runs(in); + rs_inline_scan(in); +} + +/* The inline state for a doc text whose inline lines are lines[0..n): arrays + * with one slot per '[' and per backtick of the whole text, so no paragraph can + * outgrow them. NULL when there is no inline text or memory ran out; *buf_out + * is the paragraph buffer. */ +static rs_inline_t *rs_inline_new(CBMArena *tmp, const rs_line_t *lines, int n, char **buf_out) { + size_t text_bytes = 0; + size_t brackets = 0; + size_t backticks = 0; + for (int i = 0; i < n; i++) { + if (lines[i].kind != RS_L_TEXT) { + continue; + } + text_bytes += (size_t)lines[i].len + SKIP_ONE; + for (uint32_t k = 0; k < lines[i].len; k++) { + brackets += lines[i].p[k] == '['; + backticks += lines[i].p[k] == '`'; + } + } + if (text_bytes == 0 || brackets == 0) { + return NULL; /* no bracket, no link */ + } + rs_inline_t *in = (rs_inline_t *)cbm_arena_alloc(tmp, sizeof(*in)); + char *buf = (char *)cbm_arena_alloc(tmp, text_bytes + SKIP_ONE); + if (!in || !buf) { + return NULL; + } + memset(in, 0, sizeof(*in)); + in->open = (rs_opener_t *)cbm_arena_alloc(tmp, brackets * sizeof(rs_opener_t)); + in->runs = (rs_run_t *)cbm_arena_alloc(tmp, (backticks + SKIP_ONE) * sizeof(rs_run_t)); + in->by_len = (rs_run_t *)cbm_arena_alloc(tmp, (backticks + SKIP_ONE) * sizeof(rs_run_t)); + if (!in->open || !in->runs || !in->by_len) { + return NULL; + } + *buf_out = buf; + return in; +} + +/* The references of one doc text, pushed as tokens of the node `source_qn`. + * `file_doc`: it is the file's own doc, whose lines need not be contiguous. */ +static void rs_parse_text(CBMExtractCtx *ctx, const char *source_qn, uint32_t def_line, + const char *doc, uint32_t doc_line, bool file_doc) { + if (!doc || !source_qn || !strchr(doc, '[')) { + return; /* no bracket, no link */ + } + CBMArena *tmp = rs_scratch(ctx); + int n = 0; + rs_line_t *lines = rs_doc_lines(tmp, doc, doc_line, &n); + if (!lines || n == 0) { + return; + } + if (file_doc) { + rs_lines_resync(ctx, lines, n, doc, doc_line); + } + rs_defs_t defs = {.a = tmp}; + rs_md_t st; + memset(&st, 0, sizeof(st)); + st.a = tmp; + for (int i = 0; i < n;) { + i += rs_md_line(&st, lines, n, i, &defs); + } + char *buf = NULL; + rs_inline_t *in = (defs.failed || st.failed) ? NULL : rs_inline_new(tmp, lines, n, &buf); + if (!in) { + return; + } + if (defs.n > SKIP_ONE) { + qsort(defs.v, (size_t)defs.n, sizeof(rs_def_t), rs_def_cmp); + } + rs_sink_t sink = {.ctx = ctx, .source_qn = source_qn, .def_line = def_line, .defs = &defs}; + int first = CBM_NOT_FOUND; + for (int i = 0; i <= n; i++) { + bool text = i < n && lines[i].kind == RS_L_TEXT; + if (first >= 0 && (!text || lines[i].para_start)) { + rs_scan_paragraph(in, &sink, lines, first, i - SKIP_ONE, buf); + first = CBM_NOT_FOUND; + } + if (text && first < 0) { + first = i; + } + } +} + +void cbm_doclink_rs_parse_doc(CBMExtractCtx *ctx, const CBMDefinition *def, const char *doc, + uint32_t doc_line) { + if (!ctx || !ctx->result || !def) { + return; + } + /* the driver's stand-in definition for the file's own doc is labelled File */ + bool file_doc = def->label && strcmp(def->label, "File") == 0; + rs_parse_text(ctx, def->qualified_name, def->start_line, doc, doc_line, file_doc); +} + +/* ── Scope scan: state ───────────────────────────────────────────── */ + +enum { + RS_SCAN_NAME_CAP = 128, /* bytes of one identifier, NUL included */ + RS_SCAN_PATH_CAP = 512, /* bytes of one written path, NUL included */ + RS_SCAN_ATTR_CAP = 1024, /* bytes of an attribute that are read */ + RS_SCAN_DERIVES_CAP = 256, /* bytes of the derive names of one item, NUL included */ + RS_SCAN_INIT = 16, +}; + +/* A block whose items are still to be read: the file, an inline module's + * body, an extern block's body. Blocks are queued, never recursed into. */ +typedef struct { + TSNode node; + int mod; /* module the items belong to */ +} rs_block_t; + +/* A definition of the file, keyed for the items the scan meets. */ +typedef struct { + uint32_t line; + const char *name; + int def; +} rs_defkey_t; + +/* What the attributes above an item say about it. */ +typedef struct { + bool test; /* #[cfg(test)], #[test], #[bench], #[tokio::test] ... */ + bool macro_export; /* #[macro_export] */ + bool cond_path; /* #[cfg_attr(.., path = ..)]: where the module file is depends on cfg */ + bool derives_more; /* derive names that did not fit, or were not read */ + char macro_kind; /* P #[proc_macro], B #[proc_macro_attribute], V #[proc_macro_derive] */ + char path[RS_SCAN_PATH_CAP]; /* #[path = "..."] */ + char derive[RS_SCAN_NAME_CAP]; /* the derive macro's name */ + char derives[RS_SCAN_DERIVES_CAP]; /* #[derive(A, b::C)] -> "A,C": impls not in the tree */ +} rs_attrs_t; + +typedef struct { + CBMExtractCtx *ctx; + CBMArena *tmp; /* nothing of it outlives the scan */ + const char *src; + uint32_t src_len; + rs_sb_t sb; + bool failed; /* out of memory */ + bool broken; /* something the blob cannot represent, or a parse error */ + uint32_t err_byte; /* first byte that is not trusted; UINT32_MAX: none */ + int next_mod; + rs_block_t *blocks; + int nblocks; + int blocks_cap; + const char *item_prefix; /* "." */ + size_t item_prefix_len; + rs_defkey_t *defkeys; + int ndefkeys; + unsigned char *owner; /* per definition: 1 when it owns its graph node */ + int *owner_of; /* per definition: the definition that owns its qualified name */ + int *def_mod; /* per definition: 1 + the module the scan met it in, 0: not met */ + uint32_t *ref_lines; /* start lines of the definitions that carry references, sorted */ + int nref_lines; + int last_use_mod; /* the V record written last: a run of */ + char last_use[RS_SCAN_NAME_CAP]; /* invocations of one macro is written once */ +} rs_scan_t; + +static bool rs_kind(TSNode n, const char *kind) { + return strcmp(ts_node_type(n), kind) == 0; +} + +static TSNode rs_field(TSNode n, const char *field) { + return ts_node_child_by_field_name(n, field, (uint32_t)strlen(field)); +} + +static uint32_t rs_line_of(TSNode n) { + return ts_node_start_point(n).row + TS_LINE_OFFSET; +} + +static uint32_t rs_end_line_of(TSNode n) { + return ts_node_end_point(n).row + TS_LINE_OFFSET; +} + +static void *rs_tmp_alloc(rs_scan_t *s, size_t n) { + void *p = cbm_arena_alloc(s->tmp, n ? n : SKIP_ONE); + s->failed = s->failed || !p; + return p; +} + +/* The node's text as a name: copied into `out` when it is an identifier that + * fits (raw `r#name` keeps its marker, as the extractor's names do). false + * for anything else -- error recovery can hand back a "name" spanning code. */ +static bool rs_name_of(const rs_scan_t *s, TSNode n, char *out, size_t cap) { + if (ts_node_is_null(n)) { + return false; + } + uint32_t a = ts_node_start_byte(n); + uint32_t b = ts_node_end_byte(n); + if (b <= a || b > s->src_len || (size_t)(b - a) >= cap || !rs_is_segment(s->src + a, b - a)) { + return false; + } + memcpy(out, s->src + a, b - a); + out[b - a] = '\0'; + return true; +} + +/* A written path (`a::b::C`, `crate::x`, `::std::fmt`) of the node's text, + * whitespace removed: copied into `out` when it holds nothing but segments. + * false for a type that is no path (a reference, a tuple, generics inside). */ +static bool rs_path_of(const rs_scan_t *s, TSNode n, char *out, size_t cap) { + if (ts_node_is_null(n)) { + return false; + } + uint32_t a = ts_node_start_byte(n); + uint32_t b = ts_node_end_byte(n); + if (b <= a || b > s->src_len) { + return false; + } + size_t w = 0; + for (uint32_t i = a; i < b; i++) { + char c = s->src[i]; + if (isspace((unsigned char)c)) { + continue; + } + if (w + SKIP_ONE >= cap) { + return false; + } + out[w++] = c; + } + out[w] = '\0'; + CBMDocLinkRsPath path; + return w > 0 && !memchr(out, '<', w) && !memchr(out, '@', w) && !memchr(out, '!', w) && + !memchr(out, '(', w) && + cbm_doclink_rs_parse_path(out, w, false, &path) == CBM_DOCLINK_RS_PATH_OK; +} + +/* ── Scope scan: definitions ─────────────────────────────────────── */ + +static int rs_defkey_cmp(const void *a, const void *b) { + const rs_defkey_t *x = (const rs_defkey_t *)a; + const rs_defkey_t *y = (const rs_defkey_t *)b; + if (x->line != y->line) { + return x->line < y->line ? -1 : 1; + } + int c = strcmp(x->name, y->name); + return c ? c : (x->def > y->def) - (x->def < y->def); +} + +/* The order in which the graph buffer lets definitions of one qualified name + * replace each other (cbm_gbuf_upsert_node, same file): start line, then + * name, then label; the last one owns the node. */ +static int rs_defqn_cmp_ctx(const CBMDefinition *x, const CBMDefinition *y) { + int c = strcmp(x->qualified_name, y->qualified_name); + if (c) { + return c; + } + if (x->start_line != y->start_line) { + return x->start_line < y->start_line ? -1 : 1; + } + c = strcmp(x->name, y->name); + return c ? c : strcmp(x->label ? x->label : "", y->label ? y->label : ""); +} + +typedef struct { + const CBMDefinition *def; + int idx; +} rs_defref_t; + +static int rs_defref_cmp(const void *a, const void *b) { + const rs_defref_t *x = (const rs_defref_t *)a; + const rs_defref_t *y = (const rs_defref_t *)b; + int c = rs_defqn_cmp_ctx(x->def, y->def); + return c ? c : (x->idx > y->idx) - (x->idx < y->idx); +} + +/* Index the file's definitions: by (line, name) for the items the scan meets, + * and which definition of every qualified name owns the node. */ +static void rs_scan_defs(rs_scan_t *s) { + const CBMDefArray *defs = &s->ctx->result->defs; + int n = defs->count; + s->defkeys = (rs_defkey_t *)rs_tmp_alloc(s, (size_t)n * sizeof(rs_defkey_t)); + s->owner = (unsigned char *)rs_tmp_alloc(s, (size_t)n + SKIP_ONE); + s->owner_of = (int *)rs_tmp_alloc(s, ((size_t)n + SKIP_ONE) * sizeof(int)); + s->def_mod = (int *)rs_tmp_alloc(s, ((size_t)n + SKIP_ONE) * sizeof(int)); + rs_defref_t *refs = (rs_defref_t *)rs_tmp_alloc(s, (size_t)n * sizeof(rs_defref_t)); + if (s->failed) { + return; + } + memset(s->owner, 0, (size_t)n + SKIP_ONE); + memset(s->def_mod, 0, ((size_t)n + SKIP_ONE) * sizeof(int)); + for (int i = 0; i <= n; i++) { + s->owner_of[i] = CBM_NOT_FOUND; + } + int k = 0; + for (int i = 0; i < n; i++) { + const CBMDefinition *d = &defs->items[i]; + if (!d->name || !d->qualified_name) { + continue; + } + s->defkeys[k] = (rs_defkey_t){.line = d->start_line, .name = d->name, .def = i}; + refs[k] = (rs_defref_t){.def = d, .idx = i}; + k++; + } + s->ndefkeys = k; + if (k == 0) { + return; + } + qsort(s->defkeys, (size_t)k, sizeof(rs_defkey_t), rs_defkey_cmp); + qsort(refs, (size_t)k, sizeof(rs_defref_t), rs_defref_cmp); + int owner = CBM_NOT_FOUND; /* of the run of equal qualified names being walked, last first */ + for (int i = k - SKIP_ONE; i >= 0; i--) { + bool last = i + SKIP_ONE >= k || strcmp(refs[i + SKIP_ONE].def->qualified_name, + refs[i].def->qualified_name) != 0; + owner = last ? refs[i].idx : owner; + s->owner[refs[i].idx] = last ? SKIP_ONE : 0; + s->owner_of[refs[i].idx] = owner; + } +} + +/* The definition the extractor made of an item of module `mod`: same start + * line, name and label, declared by `parent` (the qualified name of its type, + * NULL for a module-level item). CBM_NOT_FOUND when there is none. The module + * of a definition that is found is remembered. */ +static int rs_scan_def(rs_scan_t *s, int mod, uint32_t line, const char *name, const char *label, + const char *parent) { + const CBMDefArray *defs = &s->ctx->result->defs; + int lo = 0; + int hi = s->ndefkeys; + while (lo < hi) { + int mid = lo + ((hi - lo) / PAIR_LEN); + const rs_defkey_t *key = &s->defkeys[mid]; + bool less = key->line < line || (key->line == line && strcmp(key->name, name) < 0); + if (less) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + for (int i = lo; i < s->ndefkeys; i++) { + const rs_defkey_t *key = &s->defkeys[i]; + if (key->line != line || strcmp(key->name, name) != 0) { + break; + } + const CBMDefinition *d = &defs->items[key->def]; + bool same_parent = parent ? (d->parent_class && strcmp(d->parent_class, parent) == 0) + : d->parent_class == NULL; + if (d->label && strcmp(d->label, label) == 0 && same_parent) { + s->def_mod[key->def] = mod + SKIP_ONE; + return key->def; + } + } + return CBM_NOT_FOUND; +} + +static char rs_label_code(const char *label) { + static const struct { + const char *label; + char code; + } table[] = { + {"Struct", 'S'}, {"Enum", 'E'}, {"Class", 'C'}, {"Interface", 'I'}, {"Type", 'T'}, + {"Function", 'F'}, {"Variable", 'V'}, {"Method", 'M'}, {"Field", 'D'}, + }; + for (size_t i = 0; i < sizeof(table) / sizeof(table[0]); i++) { + if (strcmp(table[i].label, label) == 0) { + return table[i].code; + } + } + return 0; +} + +/* Write an item's node field: '!' when definition `def` does not exist, does + * not own its node, or has a qualified name the blob cannot carry; else the + * label code and the qualified name's part after the file's item prefix. */ +static void rs_put_node(rs_scan_t *s, int def, const char *name) { + if (def < 0) { + rs_sb_putc(&s->sb, '!'); + return; + } + const CBMDefinition *d = &s->ctx->result->defs.items[def]; + char code = rs_label_code(d->label); + const char *qn = d->qualified_name; + bool ok = code && qn && s->owner[def] && strlen(qn) > s->item_prefix_len && + memcmp(qn, s->item_prefix, s->item_prefix_len) == 0; + const char *suffix = ok ? qn + s->item_prefix_len : ""; + if (!ok || strpbrk(suffix, "\t\n\r")) { + rs_sb_putc(&s->sb, '!'); + return; + } + rs_sb_putc(&s->sb, code); + if (strcmp(suffix, name) == 0) { + rs_sb_putc(&s->sb, '='); + } else { + rs_sb_putc(&s->sb, ':'); + rs_sb_puts(&s->sb, suffix); + } +} + +/* The qualified name the extractor gives the type `owner` of this file: the + * parent_class of its members' definitions. */ +static const char *rs_owner_qn(rs_scan_t *s, const char *owner, size_t owner_len) { + char *out = (char *)rs_tmp_alloc(s, s->item_prefix_len + owner_len + SKIP_ONE); + if (!out) { + return NULL; + } + memcpy(out, s->item_prefix, s->item_prefix_len); + memcpy(out + s->item_prefix_len, owner, owner_len); + out[s->item_prefix_len + owner_len] = '\0'; + return out; +} + +/* ── Scope scan: records ─────────────────────────────────────────── */ + +static void rs_rec(rs_scan_t *s, char tag) { + rs_sb_putc(&s->sb, tag); +} + +static void rs_rec_u(rs_scan_t *s, uint32_t v) { + rs_sb_putc(&s->sb, '\t'); + rs_sb_putu(&s->sb, v); +} + +static void rs_rec_s(rs_scan_t *s, const char *v) { + rs_sb_putc(&s->sb, '\t'); + rs_sb_puts(&s->sb, v); +} + +static void rs_rec_end(rs_scan_t *s) { + rs_sb_putc(&s->sb, '\n'); +} + +/* A name some macro invocation may declare in module `mod` (-1: somewhere in + * the file). */ +static void rs_rec_maybe(rs_scan_t *s, int mod, const char *name) { + rs_rec(s, 'Q'); + if (mod < 0) { + rs_rec_s(s, "*"); + } else { + rs_rec_u(s, (uint32_t)mod); + } + rs_rec_s(s, name); + rs_rec_end(s); +} + +/* The visibility of an item and what its attributes add: the flags field. */ +static void rs_rec_flags(rs_scan_t *s, TSNode item, const rs_attrs_t *at) { + char vis = '-'; + TSTreeCursor cur = ts_tree_cursor_new(item); + if (ts_tree_cursor_goto_first_child(&cur)) { + do { + TSNode ch = ts_tree_cursor_current_node(&cur); + if (!rs_kind(ch, "visibility_modifier")) { + continue; + } + vis = 'p'; + uint32_t nc = ts_node_named_child_count(ch); + if (nc > 0) { + const char *k = ts_node_type(ts_node_named_child(ch, 0)); + vis = strcmp(k, "crate") == 0 + ? 'c' + : (strcmp(k, "super") == 0 ? 's' : (strcmp(k, "self") == 0 ? '-' : 'i')); + } + break; + } while (ts_tree_cursor_goto_next_sibling(&cur)); + } + ts_tree_cursor_delete(&cur); + rs_sb_putc(&s->sb, '\t'); + rs_sb_putc(&s->sb, vis); + if (at && at->test) { + rs_sb_putc(&s->sb, 't'); + } + if (at && at->macro_export) { + rs_sb_putc(&s->sb, 'e'); + } + if (at && at->cond_path) { + rs_sb_putc(&s->sb, 'c'); + } +} + +/* ── Scope scan: attributes ──────────────────────────────────────── */ + +static bool rs_word_is(const char *p, size_t n, const char *word) { + return strlen(word) == n && memcmp(p, word, n) == 0; +} + +/* True when the identifier `word` stands in text[0..n). */ +static bool rs_has_word(const char *text, size_t n, const char *word) { + size_t wl = strlen(word); + for (size_t i = 0; i + wl <= n; i++) { + bool starts = + i == 0 || !(isalnum((unsigned char)text[i - SKIP_ONE]) || text[i - SKIP_ONE] == '_'); + bool ends = i + wl == n || !(isalnum((unsigned char)text[i + wl]) || text[i + wl] == '_'); + if (starts && ends && memcmp(text + i, word, wl) == 0) { + return true; + } + } + return false; +} + +/* The string literal that follows `key =` in text[0..n), copied into `out`. + * false when there is none, it holds an escape, or it does not fit. */ +static bool rs_attr_string(const char *text, size_t n, char *out, size_t cap) { + const char *eq = (const char *)memchr(text, '=', n); + if (!eq) { + return false; + } + size_t i = (size_t)(eq - text) + SKIP_ONE; + while (i < n && isspace((unsigned char)text[i])) { + i++; + } + if (i >= n || text[i] != '"') { + return false; + } + size_t start = ++i; + while (i < n && text[i] != '"' && text[i] != '\\' && (unsigned char)text[i] >= ' ') { + i++; + } + if (i >= n || text[i] != '"' || i - start >= cap || i == start) { + return false; + } + memcpy(out, text + start, i - start); + out[i - start] = '\0'; + return true; +} + +/* The first identifier inside the parentheses of text[0..n). */ +static void rs_attr_first_arg(const char *text, size_t n, char *out, size_t cap) { + out[0] = '\0'; + const char *open = (const char *)memchr(text, '(', n); + if (!open) { + return; + } + size_t i = (size_t)(open - text) + SKIP_ONE; + while (i < n && isspace((unsigned char)text[i])) { + i++; + } + size_t start = i; + while (i < n && (isalnum((unsigned char)text[i]) || text[i] == '_')) { + i++; + } + if (i > start && i - start < cap) { + memcpy(out, text + start, i - start); + out[i - start] = '\0'; + } +} + +/* Append a derive name to the list of `at` (comma-joined). */ +static void rs_derive_put(rs_attrs_t *at, const char *name, size_t n) { + size_t have = strlen(at->derives); + if (n == 0) { + return; + } + if (have + n + PAIR_LEN >= sizeof(at->derives)) { + at->derives_more = true; + return; + } + if (have > 0) { + at->derives[have++] = ','; + } + memcpy(at->derives + have, name, n); + at->derives[have + n] = '\0'; +} + +/* The names inside every `derive(..)` of an attribute's text (also inside + * `cfg_attr(x, derive(..))`): the last segment of each path. */ +static void rs_attr_derives(const char *text, size_t n, rs_attrs_t *at) { + static const char word[] = "derive"; + const size_t wl = sizeof(word) - SKIP_ONE; + for (size_t i = 0; i + wl < n; i++) { + bool starts = + i == 0 || !(isalnum((unsigned char)text[i - SKIP_ONE]) || text[i - SKIP_ONE] == '_'); + size_t j = i + wl; + if (!starts || memcmp(text + i, word, wl) != 0) { + continue; + } + while (j < n && isspace((unsigned char)text[j])) { + j++; + } + if (j >= n || text[j] != '(') { + continue; + } + size_t id = 0; /* the identifier read last, and its length */ + size_t idlen = 0; + bool in_id = false; + for (j++; j < n && text[j] != ')'; j++) { + unsigned char c = (unsigned char)text[j]; + if (isalnum(c) || c == '_') { + id = in_id ? id : j; + idlen = in_id ? idlen + SKIP_ONE : SKIP_ONE; + in_id = true; + continue; + } + in_id = false; + if (c == ',') { + rs_derive_put(at, text + id, idlen); + } + idlen = (c == ',' || c == ':') ? 0 : idlen; + } + rs_derive_put(at, text + id, idlen); + at->derives_more = at->derives_more || j >= n; /* cut off before its `)` */ + i = j; + } +} + +/* Read one `#[...]` above an item into `at`. */ +static void rs_attr_read(const rs_scan_t *s, TSNode attr_item, rs_attrs_t *at) { + uint32_t a = ts_node_start_byte(attr_item); + uint32_t b = ts_node_end_byte(attr_item); + if (b <= a || b > s->src_len) { + return; + } + const char *text = s->src + a; + size_t n = b - a; + bool long_attr = n > RS_SCAN_ATTR_CAP; + n = long_attr ? RS_SCAN_ATTR_CAP : n; + /* the attribute's own name: the path after `#[` up to `(`, `=` or `]` */ + size_t i = 0; + while (i < n && (text[i] == '#' || text[i] == '[' || isspace((unsigned char)text[i]))) { + i++; + } + size_t name = i; + while (i < n && (isalnum((unsigned char)text[i]) || text[i] == '_' || text[i] == ':')) { + i++; + } + size_t nl = i - name; + const char *p = text + name; + static const char test_suffix[] = "::test"; + const size_t tsl = sizeof(test_suffix) - SKIP_ONE; + if (rs_word_is(p, nl, "test") || rs_word_is(p, nl, "bench") || + (nl > tsl && memcmp(p + nl - tsl, test_suffix, tsl) == 0)) { + at->test = true; + } else if (rs_word_is(p, nl, "cfg")) { + at->test = at->test || (rs_has_word(text, n, "test") && !rs_has_word(text, n, "not")); + } else if (rs_word_is(p, nl, "path")) { + if (long_attr || !rs_attr_string(text, n, at->path, sizeof(at->path))) { + at->cond_path = true; /* a path this scan cannot read */ + } + } else if (rs_word_is(p, nl, "cfg_attr")) { + at->cond_path = at->cond_path || long_attr || rs_has_word(text, n, "path"); + at->derives_more = at->derives_more || long_attr; + rs_attr_derives(text, n, at); + } else if (rs_word_is(p, nl, "derive")) { + at->derives_more = at->derives_more || long_attr; + rs_attr_derives(text, n, at); + } else if (rs_word_is(p, nl, "macro_export")) { + at->macro_export = true; + } else if (rs_word_is(p, nl, "proc_macro")) { + at->macro_kind = 'P'; + } else if (rs_word_is(p, nl, "proc_macro_attribute")) { + at->macro_kind = 'B'; + } else if (rs_word_is(p, nl, "proc_macro_derive")) { + at->macro_kind = 'V'; + rs_attr_first_arg(text, n, at->derive, sizeof(at->derive)); + } +} + +/* ── Scope scan: names local to a documented item ────────────────── */ + +/* True when a definition that starts at `line` carries references. */ +static bool rs_has_refs(const rs_scan_t *s, uint32_t line) { + int lo = 0; + int hi = s->nref_lines; + while (lo < hi) { + int mid = lo + ((hi - lo) / PAIR_LEN); + if (s->ref_lines[mid] < line) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + return lo < s->nref_lines && s->ref_lines[lo] == line; +} + +static void rs_put_local(rs_scan_t *s, TSNode name, bool *first) { + char buf[RS_SCAN_NAME_CAP]; + if (!rs_name_of(s, name, buf, sizeof(buf))) { + return; + } + if (!*first) { + rs_sb_putc(&s->sb, ','); + } + rs_sb_puts(&s->sb, buf); + *first = false; +} + +/* The type and const parameters of a `type_parameters` list. */ +static void rs_put_generics(rs_scan_t *s, TSNode tparams, bool *first) { + if (ts_node_is_null(tparams)) { + return; + } + uint32_t nc = ts_node_named_child_count(tparams); + for (uint32_t i = 0; i < nc; i++) { + TSNode p = ts_node_named_child(tparams, i); + if (rs_kind(p, "type_parameter") || rs_kind(p, "const_parameter")) { + rs_put_local(s, rs_field(p, "name"), first); + } else if (rs_kind(p, "type_identifier")) { + rs_put_local(s, p, first); + } + } +} + +/* Every name the patterns of a `parameters` list bind. The walk is a cursor + * walk of the list's subtree: no recursion. */ +static void rs_put_params(rs_scan_t *s, TSNode params, bool *first) { + if (ts_node_is_null(params)) { + return; + } + TSTreeCursor cur = ts_tree_cursor_new(params); + for (;;) { + TSNode n = ts_tree_cursor_current_node(&cur); + const char *field = ts_tree_cursor_current_field_name(&cur); + bool in_type = field && strcmp(field, "type") == 0; + if (rs_kind(n, "identifier") || rs_kind(n, "shorthand_field_identifier")) { + rs_put_local(s, n, first); + } + if (!in_type && ts_tree_cursor_goto_first_child(&cur)) { + continue; + } + bool moved = ts_tree_cursor_goto_next_sibling(&cur); + while (!moved && ts_tree_cursor_goto_parent(&cur)) { + moved = ts_tree_cursor_goto_next_sibling(&cur); + } + if (!moved) { + break; + } + } + ts_tree_cursor_delete(&cur); +} + +/* The P record of the definition at `line`: the type parameters of its owner + * (`outer`, may be null) and, for `item`, its own type parameters and + * parameter names. Written only for a definition with references that has + * such names. */ +static void rs_rec_locals(rs_scan_t *s, uint32_t line, TSNode outer, TSNode item) { + if (!rs_has_refs(s, line) || s->sb.failed) { + return; + } + size_t mark = s->sb.len; + rs_rec(s, 'P'); + rs_rec_u(s, line); + rs_sb_putc(&s->sb, '\t'); + size_t names_at = s->sb.len; + bool first = true; + rs_put_generics(s, outer, &first); + if (!ts_node_is_null(item)) { + rs_put_generics(s, rs_field(item, "type_parameters"), &first); + rs_put_params(s, rs_field(item, "parameters"), &first); + } + if (s->sb.failed) { + return; + } + if (s->sb.len == names_at) { + s->sb.len = mark; /* nothing local: no record */ + s->sb.buf[mark] = '\0'; + return; + } + rs_rec_end(s); +} + +/* ── Scope scan: members ─────────────────────────────────────────── */ + +static void rs_rec_member(rs_scan_t *s, uint32_t line, char kind, const char *name, int def) { + rs_rec(s, 'M'); + rs_rec_u(s, line); + rs_sb_putc(&s->sb, '\t'); + rs_sb_putc(&s->sb, kind); + rs_rec_s(s, name); + rs_sb_putc(&s->sb, '\t'); + rs_put_node(s, def, name); + rs_rec_end(s); +} + +/* The members a type's or an impl's body declares. `owner_qn` is the + * qualified name their definitions carry as parent (NULL: no definition is + * looked up). `callable_label` is the label of the functions' definitions. */ +static void rs_scan_members(rs_scan_t *s, int mod, TSNode body, const char *owner_qn, TSNode outer, + bool in_trait) { + if (ts_node_is_null(body)) { + return; + } + TSTreeCursor cur = ts_tree_cursor_new(body); + bool more = ts_tree_cursor_goto_first_child(&cur); + for (; more && !s->failed; more = ts_tree_cursor_goto_next_sibling(&cur)) { + TSNode m = ts_tree_cursor_current_node(&cur); + const char *k = ts_node_type(m); + char kind = 0; + const char *label = NULL; + if (strcmp(k, "function_item") == 0 || strcmp(k, "function_signature_item") == 0) { + kind = 'f'; + label = "Method"; + } else if (strcmp(k, "const_item") == 0) { + kind = 'c'; + label = in_trait ? "Variable" : NULL; /* an impl's constant has no definition */ + } else if (strcmp(k, "type_item") == 0 || strcmp(k, "associated_type") == 0) { + kind = 't'; + } else if (strcmp(k, "field_declaration") == 0) { + kind = 'd'; + label = "Field"; + } else if (strcmp(k, "enum_variant") == 0) { + kind = 'v'; + } + if (!kind) { + continue; + } + char name[RS_SCAN_NAME_CAP]; + if (ts_node_start_byte(m) >= s->err_byte || + !rs_name_of(s, rs_field(m, "name"), name, sizeof(name))) { + s->broken = true; /* a member this scan cannot show */ + continue; + } + uint32_t line = rs_line_of(m); + int def = + (label && owner_qn) ? rs_scan_def(s, mod, line, name, label, owner_qn) : CBM_NOT_FOUND; + rs_rec_member(s, line, kind, name, def); + TSNode none = {0}; + rs_rec_locals(s, line, outer, kind == 'f' ? m : none); + } + ts_tree_cursor_delete(&cur); +} + +/* The positional fields of a tuple struct: members `0`, `1` ... without a + * node. */ +static void rs_scan_tuple_fields(rs_scan_t *s, TSNode body) { + TSTreeCursor cur = ts_tree_cursor_new(body); + uint32_t index = 0; + bool more = ts_tree_cursor_goto_first_child(&cur); + for (; more && !s->failed; more = ts_tree_cursor_goto_next_sibling(&cur)) { + const char *field = ts_tree_cursor_current_field_name(&cur); + if (!field || strcmp(field, "type") != 0) { + continue; + } + char name[RS_UINT_DIGITS]; + snprintf(name, sizeof(name), "%u", index++); + rs_rec_member(s, 0, 'd', name, CBM_NOT_FOUND); + } + ts_tree_cursor_delete(&cur); +} + +/* ── Scope scan: `use` ───────────────────────────────────────────── */ + +typedef struct { + TSNode node; + const char *prefix; /* the path written before this part of the tree, "" at the top */ +} rs_use_frame_t; + +typedef struct { + rs_use_frame_t *v; + int n; + int cap; +} rs_use_stack_t; + +static void rs_use_push(rs_scan_t *s, rs_use_stack_t *st, TSNode node, const char *prefix) { + if (ts_node_is_null(node) || s->failed) { + return; + } + if (st->n >= st->cap) { + int ncap = st->cap ? st->cap * PAIR_LEN : RS_SCAN_INIT; + rs_use_frame_t *grown = + (rs_use_frame_t *)rs_tmp_alloc(s, (size_t)ncap * sizeof(rs_use_frame_t)); + if (!grown) { + return; + } + if (st->n > 0) { + memcpy(grown, st->v, (size_t)st->n * sizeof(rs_use_frame_t)); + } + st->v = grown; + st->cap = ncap; + } + st->v[st->n++] = (rs_use_frame_t){.node = node, .prefix = prefix}; +} + +/* `prefix::path` in the scratch arena. NULL (and the file is marked) when + * `node` is no plain path or the result is too long to carry. */ +static const char *rs_use_join(rs_scan_t *s, const char *prefix, TSNode node) { + char path[RS_SCAN_PATH_CAP]; + path[0] = '\0'; + if (!ts_node_is_null(node) && !rs_path_of(s, node, path, sizeof(path))) { + s->broken = true; + return NULL; + } + size_t pl = strlen(prefix); + size_t nl = strlen(path); + if (pl + nl + PAIR_LEN >= RS_SCAN_PATH_CAP) { + s->broken = true; + return NULL; + } + char *out = (char *)rs_tmp_alloc(s, pl + nl + PAIR_LEN + SKIP_ONE); + if (!out) { + return NULL; + } + snprintf(out, pl + nl + PAIR_LEN + SKIP_ONE, "%s%s%s", prefix, (pl && nl) ? "::" : "", path); + return out; +} + +static void rs_rec_use(rs_scan_t *s, int mod, TSNode decl, char kind, const char *local, + const char *path) { + rs_rec(s, 'U'); + rs_rec_u(s, (uint32_t)mod); + rs_rec_flags(s, decl, NULL); + rs_sb_putc(&s->sb, '\t'); + rs_sb_putc(&s->sb, kind); + rs_rec_s(s, local); + rs_rec_s(s, path); + rs_rec_end(s); +} + +/* A leaf of a use tree: `full` is the whole path, `alias` the name after + * `as` (NULL when there is none). `a::b::self` imports `b` itself, in the type + * namespace only. */ +static void rs_use_leaf(rs_scan_t *s, int mod, TSNode decl, const char *full, const char *alias) { + static const char self_seg[] = "::self"; + const size_t sl = sizeof(self_seg) - SKIP_ONE; + char path[RS_SCAN_PATH_CAP]; + snprintf(path, sizeof(path), "%s", full); + size_t n = strlen(path); + char kind = 'b'; + if (n > sl && strcmp(path + n - sl, self_seg) == 0) { + path[n - sl] = '\0'; + kind = 's'; + } + const char *last = strrchr(path, ':'); + const char *local = alias ? alias : (last ? last + SKIP_ONE : path); + bool keyword = + strcmp(local, "crate") == 0 || strcmp(local, "super") == 0 || strcmp(local, "self") == 0; + if (!local[0] || (keyword && !alias)) { + return; /* `use crate;` binds nothing */ + } + rs_rec_use(s, mod, decl, kind, local, path); +} + +/* The bindings of one `use` declaration: its tree is walked with a stack. */ +static void rs_item_use(rs_scan_t *s, TSNode decl, int mod) { + rs_use_stack_t st = {0}; + rs_use_push(s, &st, rs_field(decl, "argument"), ""); + while (st.n > 0 && !s->failed) { + rs_use_frame_t f = st.v[--st.n]; + const char *k = ts_node_type(f.node); + if (strcmp(k, "use_list") == 0) { + for (uint32_t i = ts_node_named_child_count(f.node); i > 0; i--) { + rs_use_push(s, &st, ts_node_named_child(f.node, i - SKIP_ONE), f.prefix); + } + } else if (strcmp(k, "scoped_use_list") == 0) { + const char *prefix = rs_use_join(s, f.prefix, rs_field(f.node, "path")); + if (prefix) { + rs_use_push(s, &st, rs_field(f.node, "list"), prefix); + } + } else if (strcmp(k, "use_wildcard") == 0) { + TSNode none = {0}; + TSNode inner = + ts_node_named_child_count(f.node) > 0 ? ts_node_named_child(f.node, 0) : none; + const char *path = rs_use_join(s, f.prefix, inner); + if (path && path[0]) { + rs_rec_use(s, mod, decl, 'g', "-", path); + } + } else if (strcmp(k, "use_as_clause") == 0) { + char alias[RS_SCAN_NAME_CAP]; + const char *full = rs_use_join(s, f.prefix, rs_field(f.node, "path")); + if (full && rs_name_of(s, rs_field(f.node, "alias"), alias, sizeof(alias))) { + rs_use_leaf(s, mod, decl, full, alias); + } else { + s->broken = true; + } + } else if (strcmp(k, "line_comment") != 0 && strcmp(k, "block_comment") != 0) { + const char *full = rs_use_join(s, f.prefix, f.node); + if (full) { + rs_use_leaf(s, mod, decl, full, NULL); + } + } + } +} + +static void rs_item_extern_crate(rs_scan_t *s, TSNode decl, int mod) { + char name[RS_SCAN_NAME_CAP]; + char alias[RS_SCAN_NAME_CAP]; + if (!rs_name_of(s, rs_field(decl, "name"), name, sizeof(name))) { + s->broken = true; + return; + } + bool renamed = rs_name_of(s, rs_field(decl, "alias"), alias, sizeof(alias)); + rs_rec_use(s, mod, decl, 'x', renamed ? alias : name, name); +} + +/* ── Scope scan: macro invocations ───────────────────────────────── */ + +typedef struct { + TSNode *v; + int n; + int cap; +} rs_node_stack_t; + +static void rs_node_push(rs_scan_t *s, rs_node_stack_t *st, TSNode node) { + if (s->failed) { + return; + } + if (st->n >= st->cap) { + int ncap = st->cap ? st->cap * PAIR_LEN : RS_SCAN_INIT; + TSNode *grown = (TSNode *)rs_tmp_alloc(s, (size_t)ncap * sizeof(TSNode)); + if (!grown) { + return; + } + if (st->n > 0) { + memcpy(grown, st->v, (size_t)st->n * sizeof(TSNode)); + } + st->v = grown; + st->cap = ncap; + } + st->v[st->n++] = node; +} + +/* A keyword after which a name is declared. */ +static bool rs_item_keyword(const char *t) { + static const char *const words[] = {"struct", "enum", "union", "trait", "fn", + "mod", "type", "const", "static"}; + for (size_t i = 0; i < sizeof(words) / sizeof(words[0]); i++) { + if (strcmp(words[i], t) == 0) { + return true; + } + } + return false; +} + +/* A word that may stand between such a keyword and the name (`static ref X`, + * `const unsafe fn f`). */ +static bool rs_item_modifier(const char *t) { + static const char *const words[] = {"ref", "mut", "unsafe", "async", "extern"}; + for (size_t i = 0; i < sizeof(words) / sizeof(words[0]); i++) { + if (strcmp(words[i], t) == 0) { + return true; + } + } + return false; +} + +static bool rs_delimiter(const char *t, bool open) { + return open ? (strcmp(t, "(") == 0 || strcmp(t, "[") == 0 || strcmp(t, "{") == 0) + : (strcmp(t, ")") == 0 || strcmp(t, "]") == 0 || strcmp(t, "}") == 0); +} + +/* A name a token tree may declare: a Q record for module `mod`, or (reading a + * macro definition) one more name of the list `names`. */ +static void rs_macro_name(rs_scan_t *s, int mod, const char *name, rs_sb_t *names) { + if (!names) { + rs_rec_maybe(s, mod, name); + return; + } + rs_sb_puts(names, name); + rs_sb_putc(names, ','); + s->failed = s->failed || names->failed; +} + +/* The names one token tree may declare: an identifier after an item keyword + * (at any depth), and at the top a whole argument that is one identifier + * (`define_label!(AppLabel, ..)`). Nested trees are pushed, not recursed. + * A metavariable (`struct $name`) is no identifier and is not recorded. */ +static void rs_macro_tree(rs_scan_t *s, rs_node_stack_t *st, TSNode tree, bool top, int mod, + rs_sb_t *names) { + bool after_keyword = false; + bool arg_start = false; + TSNode candidate = {0}; + TSTreeCursor cur = ts_tree_cursor_new(tree); + bool more = ts_tree_cursor_goto_first_child(&cur); + for (; more && !s->failed; more = ts_tree_cursor_goto_next_sibling(&cur)) { + TSNode c = ts_tree_cursor_current_node(&cur); + const char *t = ts_node_type(c); + bool ends_arg = strcmp(t, ",") == 0 || rs_delimiter(t, false); + if (!ts_node_is_null(candidate) && ends_arg) { + char name[RS_SCAN_NAME_CAP]; + if (rs_name_of(s, candidate, name, sizeof(name))) { + rs_macro_name(s, mod, name, names); + } + } + candidate = (TSNode){0}; + if (strcmp(t, "identifier") == 0) { + /* inside a token tree a contextual keyword (`union`) and a + * modifier (`ref` of `static ref X`) are plain identifiers */ + char name[RS_SCAN_NAME_CAP]; + bool named = rs_name_of(s, c, name, sizeof(name)); + if (named && after_keyword && rs_item_modifier(name)) { + arg_start = false; + continue; + } + if (named && after_keyword) { + rs_macro_name(s, mod, name, names); + } else if (top && arg_start) { + candidate = c; + } + after_keyword = named && !after_keyword && rs_item_keyword(name); + } else if (strcmp(t, "token_tree") == 0) { + rs_node_push(s, st, c); + after_keyword = false; + } else if (rs_item_keyword(t)) { + after_keyword = true; + } else if (!rs_item_modifier(t)) { + after_keyword = false; + } + arg_start = strcmp(t, ",") == 0 || rs_delimiter(t, true); + } + ts_tree_cursor_delete(&cur); +} + +/* A macro invocation among a module's items: what it expands to is not in + * the tree. The names written at the invocation that it may declare are + * recorded (Q), and so is the invocation itself (V): the macro's own body may + * declare names too (W records, of this file or another). */ +static void rs_item_macro(rs_scan_t *s, TSNode invocation, int mod) { + rs_node_stack_t st = {0}; + char name[RS_SCAN_NAME_CAP]; + TSNode macro = rs_field(invocation, "macro"); + if (!ts_node_is_null(macro) && rs_kind(macro, "scoped_identifier")) { + macro = rs_field(macro, "name"); + } + if (rs_name_of(s, macro, name, sizeof(name)) && + !(s->last_use_mod == mod && strcmp(s->last_use, name) == 0)) { + rs_rec(s, 'V'); + rs_rec_u(s, (uint32_t)mod); + rs_rec_s(s, name); + rs_rec_end(s); + s->last_use_mod = mod; + snprintf(s->last_use, sizeof(s->last_use), "%s", name); + } + uint32_t nc = ts_node_named_child_count(invocation); + for (uint32_t i = 0; i < nc; i++) { + TSNode c = ts_node_named_child(invocation, i); + if (rs_kind(c, "token_tree")) { + rs_macro_tree(s, &st, c, true, mod, NULL); + } + } + while (st.n > 0 && !s->failed) { + TSNode tree = st.v[--st.n]; + rs_macro_tree(s, &st, tree, false, mod, NULL); + } +} + +/* What the body of a `macro_rules!` definition declares by a written name + * (`pub struct Options { .. }` in a transcriber): a W record. A module that + * invokes the macro may then hold those names. */ +static void rs_macro_declares(rs_scan_t *s, TSNode def, const char *name) { + rs_node_stack_t st = {0}; + rs_sb_t names = {.a = s->tmp}; + uint32_t nc = ts_node_named_child_count(def); + for (uint32_t i = 0; i < nc; i++) { + TSNode rule = ts_node_named_child(def, i); + if (rs_kind(rule, "macro_rule")) { + rs_node_push(s, &st, rs_field(rule, "right")); + } + } + while (st.n > 0 && !s->failed) { + TSNode tree = st.v[--st.n]; + if (!ts_node_is_null(tree)) { + rs_macro_tree(s, &st, tree, false, 0, &names); + } + } + if (names.len > SKIP_ONE && !s->failed) { + rs_rec(s, 'W'); + rs_rec_s(s, name); + rs_sb_putc(&s->sb, '\t'); + rs_sb_putn(&s->sb, names.buf, names.len - SKIP_ONE); /* without the last comma */ + rs_rec_end(s); + } +} + +/* ── Scope scan: items ───────────────────────────────────────────── */ + +static void rs_block_push(rs_scan_t *s, TSNode node, int mod) { + if (ts_node_is_null(node) || s->failed) { + return; + } + if (s->nblocks >= s->blocks_cap) { + int ncap = s->blocks_cap ? s->blocks_cap * PAIR_LEN : RS_SCAN_INIT; + rs_block_t *grown = (rs_block_t *)rs_tmp_alloc(s, (size_t)ncap * sizeof(rs_block_t)); + if (!grown) { + return; + } + if (s->nblocks > 0) { + memcpy(grown, s->blocks, (size_t)s->nblocks * sizeof(rs_block_t)); + } + s->blocks = grown; + s->blocks_cap = ncap; + } + s->blocks[s->nblocks++] = (rs_block_t){.node = node, .mod = mod}; +} + +/* The first fields of a T record: T mod start end flags kind name, then the + * tab before the node field. */ +static void rs_rec_item(rs_scan_t *s, int mod, TSNode item, const rs_attrs_t *at, char kind, + const char *name) { + rs_rec(s, 'T'); + rs_rec_u(s, (uint32_t)mod); + rs_rec_u(s, rs_line_of(item)); + rs_rec_u(s, rs_end_line_of(item)); + rs_rec_flags(s, item, at); + rs_sb_putc(&s->sb, '\t'); + rs_sb_putc(&s->sb, kind); + rs_rec_s(s, name); + rs_sb_putc(&s->sb, '\t'); +} + +/* The path a type node names, generics dropped: `a::S` -> `a::S`. false + * for a type that is no path. */ +static bool rs_type_path(const rs_scan_t *s, TSNode type, char *out, size_t cap) { + if (!ts_node_is_null(type) && rs_kind(type, "generic_type")) { + type = rs_field(type, "type"); + } + if (ts_node_is_null(type) || + !(rs_kind(type, "type_identifier") || rs_kind(type, "scoped_type_identifier") || + rs_kind(type, "identifier") || rs_kind(type, "scoped_identifier"))) { + return false; + } + return rs_path_of(s, type, out, cap); +} + +/* True when `name` is one of the type parameters of `tparams`: `impl Tr + * for T` implements for every type, not for a type named T. */ +static bool rs_is_generic(const rs_scan_t *s, TSNode tparams, const char *name) { + if (ts_node_is_null(tparams)) { + return false; + } + uint32_t nc = ts_node_named_child_count(tparams); + for (uint32_t i = 0; i < nc; i++) { + TSNode p = ts_node_named_child(tparams, i); + TSNode pn = rs_kind(p, "type_parameter") ? rs_field(p, "name") : p; + char buf[RS_SCAN_NAME_CAP]; + if (rs_name_of(s, pn, buf, sizeof(buf)) && strcmp(buf, name) == 0) { + return true; + } + } + return false; +} + +static void rs_item_impl(rs_scan_t *s, TSNode n, int mod) { + TSNode type = rs_field(n, "type"); + TSNode trait = rs_field(n, "trait"); + TSNode tparams = rs_field(n, "type_parameters"); + char self_path[RS_SCAN_PATH_CAP]; + char trait_path[RS_SCAN_PATH_CAP]; + bool has_self = rs_type_path(s, type, self_path, sizeof(self_path)) && + !rs_is_generic(s, tparams, self_path); + bool has_trait = !ts_node_is_null(trait); + bool trait_ok = has_trait && rs_type_path(s, trait, trait_path, sizeof(trait_path)); + rs_rec(s, 'I'); + rs_rec_u(s, (uint32_t)mod); + rs_rec_u(s, rs_line_of(n)); + rs_rec_u(s, rs_end_line_of(n)); + rs_rec_s(s, has_self ? self_path : "?"); + rs_rec_s(s, has_trait ? (trait_ok ? trait_path : "?") : "-"); + rs_rec_end(s); + /* the extractor names a method . */ + const char *owner_qn = NULL; + if (has_self && !ts_node_is_null(type)) { + uint32_t a = ts_node_start_byte(type); + uint32_t b = ts_node_end_byte(type); + const char *lt = b > a ? (const char *)memchr(s->src + a, '<', b - a) : NULL; + size_t len = lt ? (size_t)(lt - (s->src + a)) : (size_t)(b - a); + owner_qn = rs_owner_qn(s, s->src + a, len); + } + rs_scan_members(s, mod, rs_field(n, "body"), owner_qn, tparams, false); +} + +/* What the scan does with a module-level item of one node kind. */ +typedef struct { + const char *node_kind; + char kind; + const char *label; /* of its definition; NULL: the extractor makes none */ +} rs_item_kind_t; + +static const rs_item_kind_t *rs_item_kind(const char *k) { + static const rs_item_kind_t kinds[] = { + {"struct_item", 'S', "Struct"}, + {"enum_item", 'E', "Enum"}, + {"union_item", 'U', "Class"}, + {"trait_item", 'R', "Interface"}, + {"type_item", 'A', "Type"}, + {"function_item", 'F', "Function"}, + {"function_signature_item", 'F', "Function"}, + {"const_item", 'C', "Variable"}, + {"static_item", 'G', "Variable"}, + {"macro_definition", 'M', NULL}, + }; + for (size_t i = 0; i < sizeof(kinds) / sizeof(kinds[0]); i++) { + if (strcmp(kinds[i].node_kind, k) == 0) { + return &kinds[i]; + } + } + return NULL; +} + +/* The members of a struct, union, enum or trait. */ +static void rs_item_members(rs_scan_t *s, int mod, TSNode n, char kind, const char *name) { + TSNode body = rs_field(n, "body"); + TSNode tparams = rs_field(n, "type_parameters"); + if (ts_node_is_null(body)) { + return; + } + if (rs_kind(body, "ordered_field_declaration_list")) { + rs_scan_tuple_fields(s, body); + return; + } + rs_scan_members(s, mod, body, rs_owner_qn(s, name, strlen(name)), tparams, kind == 'R'); +} + +/* A named item: its T record, its members, its local names. */ +static void rs_item_named(rs_scan_t *s, TSNode n, const rs_item_kind_t *ik, int mod, + const rs_attrs_t *at) { + char name[RS_SCAN_NAME_CAP]; + if (!rs_name_of(s, rs_field(n, "name"), name, sizeof(name))) { + s->broken = true; + return; + } + if (strcmp(name, "_") == 0) { + return; /* `const _: () = ..` declares nothing */ + } + char kind = ik->kind; + const char *shown = name; + if (kind == 'F' && at->macro_kind) { + kind = at->macro_kind; /* the function IS a procedural macro */ + shown = (kind == 'V' && at->derive[0]) ? at->derive : name; + } + uint32_t line = rs_line_of(n); + int def = ik->label ? rs_scan_def(s, mod, line, name, ik->label, NULL) : CBM_NOT_FOUND; + rs_rec_item(s, mod, n, at, kind, shown); + rs_put_node(s, def, shown); + char alias[RS_SCAN_PATH_CAP]; + bool aliased = kind == 'A' && rs_type_path(s, rs_field(n, "type"), alias, sizeof(alias)); + bool derived = + (kind == 'S' || kind == 'E' || kind == 'U') && (at->derives[0] || at->derives_more); + if (derived) { + rs_rec_s(s, at->derives); + rs_sb_puts(&s->sb, at->derives_more ? (at->derives[0] ? ",*" : "*") : ""); + } else { + rs_rec_s(s, aliased ? alias : "-"); + } + rs_rec_end(s); + if (rs_kind(n, "macro_definition")) { + rs_macro_declares(s, n, name); + } + if (kind == 'S' || kind == 'U' || kind == 'E' || kind == 'R') { + rs_item_members(s, mod, n, kind, name); + } + TSNode none = {0}; + rs_rec_locals(s, line, none, n); +} + +static void rs_item_mod(rs_scan_t *s, TSNode n, int mod, const rs_attrs_t *at) { + char name[RS_SCAN_NAME_CAP]; + if (!rs_name_of(s, rs_field(n, "name"), name, sizeof(name))) { + s->broken = true; + return; + } + TSNode body = rs_field(n, "body"); + if (ts_node_is_null(body)) { + rs_rec(s, 'D'); + rs_rec_u(s, (uint32_t)mod); + rs_rec_flags(s, n, at); + rs_rec_s(s, name); + rs_rec_s(s, at->path[0] ? at->path : "-"); + rs_rec_end(s); + return; + } + int id = s->next_mod++; + rs_rec(s, 'N'); + rs_rec_u(s, (uint32_t)id); + rs_rec_u(s, (uint32_t)mod); + rs_rec_u(s, rs_line_of(n)); + rs_rec_u(s, rs_end_line_of(n)); + rs_rec_flags(s, n, at); + if (at->path[0] && !at->cond_path) { + rs_sb_putc(&s->sb, 'c'); /* `#[path]` on an inline module moves its `mod x;` files */ + } + rs_rec_s(s, name); + rs_rec_end(s); + rs_block_push(s, body, id); +} + +/* An item that starts where the tree is no longer trusted: only its name is + * kept, as one that may be declared somewhere in the file. */ +static void rs_item_untrusted(rs_scan_t *s, TSNode n) { + char name[RS_SCAN_NAME_CAP]; + s->broken = true; + if (rs_name_of(s, rs_field(n, "name"), name, sizeof(name))) { + rs_rec_maybe(s, CBM_NOT_FOUND, name); + } +} + +static void rs_scan_item(rs_scan_t *s, TSNode n, int mod, const rs_attrs_t *at) { + const char *k = ts_node_type(n); + if (ts_node_start_byte(n) >= s->err_byte || strcmp(k, "ERROR") == 0) { + rs_item_untrusted(s, n); + return; + } + const rs_item_kind_t *ik = rs_item_kind(k); + if (ik) { + rs_item_named(s, n, ik, mod, at); + } else if (strcmp(k, "mod_item") == 0) { + rs_item_mod(s, n, mod, at); + } else if (strcmp(k, "use_declaration") == 0) { + rs_item_use(s, n, mod); + } else if (strcmp(k, "extern_crate_declaration") == 0) { + rs_item_extern_crate(s, n, mod); + } else if (strcmp(k, "impl_item") == 0) { + rs_item_impl(s, n, mod); + } else if (strcmp(k, "foreign_mod_item") == 0) { + rs_block_push(s, rs_field(n, "body"), mod); /* its items are the module's */ + } else if (strcmp(k, "macro_invocation") == 0) { + rs_item_macro(s, n, mod); + } else if (strcmp(k, "expression_statement") == 0 && ts_node_named_child_count(n) > 0 && + rs_kind(ts_node_named_child(n, 0), "macro_invocation")) { + rs_item_macro(s, ts_node_named_child(n, 0), mod); + } +} + +/* The items of one block, in order; attributes belong to the item after + * them. */ +static void rs_scan_block(rs_scan_t *s, rs_block_t block) { + rs_attrs_t *at = (rs_attrs_t *)rs_tmp_alloc(s, sizeof(*at)); + if (!at) { + return; + } + memset(at, 0, sizeof(*at)); + TSTreeCursor cur = ts_tree_cursor_new(block.node); + bool more = ts_tree_cursor_goto_first_child(&cur); + for (; more && !s->failed; more = ts_tree_cursor_goto_next_sibling(&cur)) { + TSNode n = ts_tree_cursor_current_node(&cur); + const char *k = ts_node_type(n); + if (strcmp(k, "attribute_item") == 0) { + rs_attr_read(s, n, at); + continue; + } + if (!ts_node_is_named(n) || strcmp(k, "line_comment") == 0 || + strcmp(k, "block_comment") == 0 || strcmp(k, "inner_attribute_item") == 0) { + continue; + } + rs_scan_item(s, n, block.mod, at); + memset(at, 0, sizeof(*at)); + } + ts_tree_cursor_delete(&cur); +} + +/* ── Scope scan: the driver ──────────────────────────────────────── */ + +/* True when the delimiters in text[0..n) pair up. */ +static bool rs_delims_balanced(const char *text, size_t n) { + static const char open[] = "([{"; + static const char close[] = ")]}"; + enum { KINDS = 3 }; + long depth[KINDS] = {0, 0, 0}; + for (size_t i = 0; i < n; i++) { + for (int k = 0; k < KINDS; k++) { + depth[k] += text[i] == open[k]; + depth[k] -= text[i] == close[k]; + if (depth[k] < 0) { + return false; + } + } + } + return depth[0] == 0 && depth[1] == 0 && depth[PAIR_LEN] == 0; +} + +/* An error node that cannot have moved an item boundary: it lies inside a + * block (a function body: nothing in there is scanned), and the parser + * neither skipped unpaired delimiters nor made one up. Syntax the grammar + * does not know yet parses this way; the items around the function are then + * exactly where the tree puts them. */ +static bool rs_error_contained(const rs_scan_t *s, TSNode n, bool in_block) { + uint32_t a = ts_node_start_byte(n); + uint32_t b = ts_node_end_byte(n); + if (!in_block || b > s->src_len) { + return false; + } + if (ts_node_is_missing(n)) { + const char *t = ts_node_type(n); + return !rs_delimiter(t, true) && !rs_delimiter(t, false); + } + return b <= a || rs_delims_balanced(s->src + a, b - a); +} + +/* The first ERROR or MISSING node of the tree, in document order, that is not + * contained (rs_error_contained); a null node when there is none. Walks only + * the nodes that hold an error, with a cursor. */ +static TSNode rs_first_error(const rs_scan_t *s, TSNode root) { + TSNode found = {0}; + if (!ts_node_has_error(root)) { + return found; + } + TSTreeCursor cur = ts_tree_cursor_new(root); + int depth = 0; + int block_depth = CBM_NOT_FOUND; /* depth of the outermost block around the cursor */ + for (bool more = true; more;) { + TSNode n = ts_tree_cursor_current_node(&cur); + bool error = ts_node_is_missing(n) || rs_kind(n, "ERROR"); + if (error && !rs_error_contained(s, n, block_depth >= 0)) { + found = n; + break; + } + if (block_depth < 0 && rs_kind(n, "block")) { + block_depth = depth; + } + if (!error && ts_node_has_error(n) && ts_tree_cursor_goto_first_child(&cur)) { + depth++; + continue; + } + for (;;) { /* on to the next node: a sibling, or an ancestor's sibling */ + if (block_depth == depth) { + block_depth = CBM_NOT_FOUND; /* leaving the block itself */ + } + if (ts_tree_cursor_goto_next_sibling(&cur)) { + break; + } + if (!ts_tree_cursor_goto_parent(&cur)) { + more = false; + break; + } + depth--; + } + } + ts_tree_cursor_delete(&cur); + return found; +} + +static int rs_u32_cmp(const void *a, const void *b) { + uint32_t x = *(const uint32_t *)a; + uint32_t y = *(const uint32_t *)b; + return (x > y) - (x < y); +} + +/* The start lines of the definitions whose docs hold references (the file's + * own doc is no definition), sorted. */ +static void rs_scan_ref_lines(rs_scan_t *s) { + const CBMDocLinkArray *links = &s->ctx->result->doc_links; + s->ref_lines = (uint32_t *)rs_tmp_alloc(s, (size_t)links->count * sizeof(uint32_t)); + if (!s->ref_lines) { + return; + } + int n = 0; + for (int i = 0; i < links->count; i++) { + if (!(links->items[i].flags & CBM_DOCLINK_FLAG_FILE)) { + s->ref_lines[n++] = links->items[i].def_line; + } + } + if (n > SKIP_ONE) { + qsort(s->ref_lines, (size_t)n, sizeof(uint32_t), rs_u32_cmp); + } + s->nref_lines = n; +} + +/* Two definitions of one qualified name that are one member name of one type + * (a field and the method named like it), or one item name of one module + * (declared twice under different cfg): the node stands for both, and a + * reference in the doc of either may leave from it. Definitions of different + * modules or parents only share a name the graph cannot tell apart. */ +static bool rs_same_declared_name(const rs_scan_t *s, int a, int b) { + const CBMDefArray *defs = &s->ctx->result->defs; + const char *pa = defs->items[a].parent_class; + const char *pb = defs->items[b].parent_class; + bool same_parent = (!pa && !pb) || (pa && pb && strcmp(pa, pb) == 0); + return same_parent && s->def_mod[a] != 0 && s->def_mod[a] == s->def_mod[b]; +} + +/* Z records: a documented definition with references that does not own its + * node and shares it with an unrelated definition (rs_same_declared_name). + * Its references would otherwise leave from that other definition's node. */ +static void rs_scan_unowned(rs_scan_t *s) { + const CBMDefArray *defs = &s->ctx->result->defs; + for (int i = 0; i < defs->count && !s->failed; i++) { + const CBMDefinition *d = &defs->items[i]; + if (s->owner[i] || !d->docstring || !d->qualified_name || !d->name || + !rs_has_refs(s, d->start_line) || + strncmp(d->qualified_name, s->item_prefix, s->item_prefix_len) != 0 || + (s->owner_of[i] >= 0 && rs_same_declared_name(s, i, s->owner_of[i]))) { + continue; + } + const char *suffix = d->qualified_name + s->item_prefix_len; + if (strpbrk(suffix, "\t\n\r")) { + continue; + } + rs_rec(s, 'Z'); + rs_rec_u(s, d->start_line); + rs_rec_s(s, suffix); + rs_rec_end(s); + } +} + +const char *cbm_doclink_rs_scan_scope(CBMExtractCtx *ctx) { + if (!ctx || !ctx->result || ts_node_is_null(ctx->root) || !ctx->source) { + return NULL; + } + rs_scan_t s = {.ctx = ctx, + .tmp = rs_scratch(ctx), + .src = ctx->source, + .src_len = ctx->source_len > 0 ? (uint32_t)ctx->source_len : 0, + .sb = {.a = ctx->arena}, + .err_byte = UINT32_MAX, + .next_mod = SKIP_ONE, + .last_use_mod = CBM_NOT_FOUND}; + /* the qualified name of an item `x` of this file, minus the `x` */ + const char *probe = cbm_fqn_compute(s.tmp, ctx->project, ctx->rel_path, "x"); + if (!probe || strlen(probe) < PAIR_LEN) { + return NULL; + } + s.item_prefix = probe; + s.item_prefix_len = strlen(probe) - SKIP_ONE; + rs_scan_defs(&s); + rs_scan_ref_lines(&s); + TSNode err = rs_first_error(&s, ctx->root); + rs_sb_puts(&s.sb, CBM_DOCLINK_RS_SCOPE_TAG "\n"); + if (!ts_node_is_null(err)) { + s.err_byte = ts_node_start_byte(err); + s.broken = true; + rs_rec(&s, 'E'); + rs_rec_u(&s, rs_line_of(err)); + rs_rec_end(&s); + } + rs_block_push(&s, ctx->root, 0); + for (int b = 0; b < s.nblocks && !s.failed; b++) { + rs_scan_block(&s, s.blocks[b]); + } + if (!s.failed) { + rs_scan_unowned(&s); + } + if (s.broken) { + rs_sb_puts(&s.sb, "B\n"); + } + if (s.failed || s.sb.failed || !s.sb.buf) { + return NULL; + } + return s.sb.buf; +} + +/* ── Cargo manifest ────────────────────────────────────────────────── + * + * The scope blob of a `Cargo.toml` (tag line "rs1", then "C"): what the Rust + * resolver knows of packages, targets and dependencies. The indexer extracts + * the file anyway, so the resolver reads no file of its own. + * K table key value a key of [package], [lib] or + * [workspace.package] the resolver uses + * A kind one [[bin]] / [[example]] / [[test]] / + * [[bench]] target; its keys follow as + * a key value `name` or `path` of that target + * W the manifest has a [workspace] table + * X kind name attr value a dependency. kind: d dependencies, v dev, + * b build, w workspace.dependencies; attr '-' + * when the entry is only named, else `path`, + * `package` or `workspace` + * B the TOML tree has a parse error + * A value this scan cannot carry (an escape, a multi-line string, a tab) is + * written as "?": the resolver then knows the key is set, not to what. */ + +enum { RS_TOML_SEGS = 8 }; + +typedef struct { + char seg[RS_TOML_SEGS][RS_SCAN_NAME_CAP]; + int n; + bool ok; /* false: more segments, a longer one, or an escape */ +} rs_toml_key_t; + +enum { + RS_TABLE_OTHER = 0, + RS_TABLE_PACKAGE, + RS_TABLE_LIB, + RS_TABLE_WORKSPACE_PACKAGE, + RS_TABLE_TARGET, /* one element of a target array */ + RS_TABLE_DEPS, /* a dependency table: `dep` is its kind */ + RS_TABLE_DEP, /* the table of one dependency: `dep` and `name` */ +}; + +typedef struct { + int kind; + char dep; + const char *name; /* RS_TABLE_DEP: the dependency; else the table's dotted name */ + bool is_workspace; /* [workspace] or a table below it */ +} rs_table_t; + +void cbm_doclink_rs_manifest_parse_doc(CBMExtractCtx *ctx, const CBMDefinition *def, + const char *doc, uint32_t doc_line) { + (void)ctx; + (void)def; + (void)doc; + (void)doc_line; /* a manifest holds no doc-comment references */ +} + +static bool rs_is_manifest(const char *rel_path) { + static const char name[] = "Cargo.toml"; + const char *slash = strrchr(rel_path, '/'); + return strcmp(slash ? slash + SKIP_ONE : rel_path, name) == 0; +} + +/* The dotted key written at text[0..n): `a.'b c'."d"` -> a, b c, d. */ +static void rs_toml_key(const char *text, size_t n, rs_toml_key_t *out) { + out->n = 0; + out->ok = true; + size_t i = 0; + while (i < n && out->ok) { + while (i < n && (text[i] == ' ' || text[i] == '\t')) { + i++; + } + char quote = (i < n && (text[i] == '"' || text[i] == '\'')) ? text[i] : 0; + size_t start = quote ? i + SKIP_ONE : i; + size_t e = start; + while (e < n && + (quote ? text[e] != quote : (text[e] != '.' && text[e] != ' ' && text[e] != '\t'))) { + out->ok = out->ok && text[e] != '\\'; + e++; + } + if (out->n >= RS_TOML_SEGS || e - start >= RS_SCAN_NAME_CAP || e == start || + (quote && e >= n)) { + out->ok = false; + return; + } + memcpy(out->seg[out->n], text + start, e - start); + out->seg[out->n][e - start] = '\0'; + out->n++; + i = quote ? e + SKIP_ONE : e; + while (i < n && (text[i] == ' ' || text[i] == '\t')) { + i++; + } + if (i < n && text[i] != '.') { + out->ok = false; + return; + } + i++; + } +} + +static void rs_toml_key_of(const rs_scan_t *s, TSNode key, rs_toml_key_t *out) { + uint32_t a = ts_node_start_byte(key); + uint32_t b = ts_node_end_byte(key); + if (ts_node_is_null(key) || b <= a || b > s->src_len) { + out->n = 0; + out->ok = false; + return; + } + rs_toml_key(s->src + a, b - a, out); +} + +static bool rs_toml_is_key(TSNode n) { + return rs_kind(n, "bare_key") || rs_kind(n, "quoted_key") || rs_kind(n, "dotted_key"); +} + +/* A string or boolean value as text; "?" for one the blob cannot carry. NULL + * for a value of another kind. */ +static const char *rs_toml_scalar(rs_scan_t *s, TSNode v) { + if (ts_node_is_null(v)) { + return NULL; + } + uint32_t a = ts_node_start_byte(v); + uint32_t b = ts_node_end_byte(v); + if (b <= a || b > s->src_len) { + return NULL; + } + const char *text = s->src + a; + size_t n = b - a; + if (rs_kind(v, "boolean")) { + return rs_word_is(text, n, "true") ? "true" : "false"; + } + if (!rs_kind(v, "string")) { + return NULL; + } + char q = text[0]; + bool plain = n >= PAIR_LEN && (q == '"' || q == '\'') && text[n - SKIP_ONE] == q && + n - PAIR_LEN < RS_SCAN_PATH_CAP; + for (size_t i = SKIP_ONE; plain && i + SKIP_ONE < n; i++) { + unsigned char c = (unsigned char)text[i]; + plain = c != (unsigned char)q && c >= ' ' && !(q == '"' && c == '\\'); + } + if (!plain || n == PAIR_LEN) { + return "?"; + } + char *out = (char *)rs_tmp_alloc(s, n); + if (!out) { + return NULL; + } + memcpy(out, text + SKIP_ONE, n - PAIR_LEN); + out[n - PAIR_LEN] = '\0'; + return out; +} + +static char rs_dep_kind(const char *seg) { + static const struct { + const char *name; + char kind; + } table[] = { + {"dependencies", 'd'}, {"dev-dependencies", 'v'}, {"dev_dependencies", 'v'}, + {"build-dependencies", 'b'}, {"build_dependencies", 'b'}, + }; + for (size_t i = 0; i < sizeof(table) / sizeof(table[0]); i++) { + if (strcmp(table[i].name, seg) == 0) { + return table[i].kind; + } + } + return 0; +} + +/* What the table named `key` is. `array`: it is written `[[key]]`. */ +static rs_table_t rs_table_class(const rs_toml_key_t *key, bool array) { + rs_table_t t = {.kind = RS_TABLE_OTHER}; + if (!key->ok || key->n == 0) { + return t; + } + const char *first = key->seg[0]; + const char *last = key->seg[key->n - SKIP_ONE]; + t.is_workspace = strcmp(first, "workspace") == 0; + static const char *const targets[] = {"bin", "example", "test", "bench"}; + for (size_t i = 0; array && key->n == SKIP_ONE && i < sizeof(targets) / sizeof(targets[0]); + i++) { + if (strcmp(first, targets[i]) == 0) { + t.kind = RS_TABLE_TARGET; + t.name = targets[i]; + return t; + } + } + if (array) { + return t; + } + if (key->n == SKIP_ONE && strcmp(first, "package") == 0) { + t.kind = RS_TABLE_PACKAGE; + t.name = "package"; + } else if (key->n == SKIP_ONE && strcmp(first, "lib") == 0) { + t.kind = RS_TABLE_LIB; + t.name = "lib"; + } else if (key->n == PAIR_LEN && t.is_workspace && strcmp(last, "package") == 0) { + t.kind = RS_TABLE_WORKSPACE_PACKAGE; + t.name = "workspace.package"; + } else if (rs_dep_kind(last)) { + t.kind = RS_TABLE_DEPS; + t.dep = t.is_workspace ? 'w' : rs_dep_kind(last); + } else if (key->n >= PAIR_LEN && rs_dep_kind(key->seg[key->n - PAIR_LEN])) { + t.kind = RS_TABLE_DEP; + t.dep = t.is_workspace ? 'w' : rs_dep_kind(key->seg[key->n - PAIR_LEN]); + t.name = last; + } + return t; +} + +static void rs_rec_dep(rs_scan_t *s, char kind, const char *name, const char *attr, + const char *value) { + rs_rec(s, 'X'); + rs_sb_putc(&s->sb, '\t'); + rs_sb_putc(&s->sb, kind); + rs_rec_s(s, name); + rs_rec_s(s, attr); + rs_rec_s(s, value); + rs_rec_end(s); +} + +static bool rs_dep_attr(const char *key) { + return strcmp(key, "path") == 0 || strcmp(key, "package") == 0 || strcmp(key, "workspace") == 0; +} + +/* The key and value node of a `pair`. */ +static void rs_toml_pair(TSNode pair, TSNode *key, TSNode *value) { + *key = (TSNode){0}; + *value = (TSNode){0}; + uint32_t nc = ts_node_named_child_count(pair); + for (uint32_t i = 0; i < nc; i++) { + TSNode c = ts_node_named_child(pair, i); + if (rs_kind(c, "comment")) { + continue; + } + if (ts_node_is_null(*key) && rs_toml_is_key(c)) { + *key = c; + } else if (!ts_node_is_null(*key)) { + *value = c; + } + } +} + +/* One entry of a dependency table: `name = "1"`, `name = { path = .. }`, + * `name.workspace = true`. */ +static void rs_manifest_dep(rs_scan_t *s, char kind, const rs_toml_key_t *key, TSNode value) { + const char *name = key->seg[0]; + if (strpbrk(name, "\t\n\r")) { + return; + } + rs_rec_dep(s, kind, name, "-", "-"); + if (key->n == PAIR_LEN && rs_dep_attr(key->seg[1])) { + const char *v = rs_toml_scalar(s, value); + rs_rec_dep(s, kind, name, key->seg[1], v ? v : "?"); + return; + } + if (key->n != SKIP_ONE || ts_node_is_null(value) || !rs_kind(value, "inline_table")) { + return; + } + uint32_t nc = ts_node_named_child_count(value); + for (uint32_t i = 0; i < nc && !s->failed; i++) { + TSNode inner = ts_node_named_child(value, i); + TSNode ik; + TSNode iv; + if (!rs_kind(inner, "pair")) { + continue; + } + rs_toml_pair(inner, &ik, &iv); + rs_toml_key_t in_key; + rs_toml_key_of(s, ik, &in_key); + if (in_key.ok && in_key.n == SKIP_ONE && rs_dep_attr(in_key.seg[0])) { + const char *v = rs_toml_scalar(s, iv); + rs_rec_dep(s, kind, name, in_key.seg[0], v ? v : "?"); + } + } +} + +/* True for the keys of [package], [lib], [workspace.package] and a target + * that the resolver reads. */ +static bool rs_manifest_key_used(const char *key) { + static const char *const keys[] = {"name", "edition", "build", "path", + "autobins", "autoexamples", "autotests", "autobenches", + "proc-macro", "proc_macro"}; + for (size_t i = 0; i < sizeof(keys) / sizeof(keys[0]); i++) { + if (strcmp(keys[i], key) == 0) { + return true; + } + } + return false; +} + +/* `key = value` of [package], [lib], [workspace.package] or a target. + * `edition.workspace = true` and `edition = { workspace = true }` are both + * written as the key `edition.workspace`. */ +static void rs_manifest_setting(rs_scan_t *s, const rs_table_t *t, const rs_toml_key_t *key, + TSNode value) { + if (!rs_manifest_key_used(key->seg[0])) { + return; + } + const char *v = rs_toml_scalar(s, value); + bool inherited = key->n == PAIR_LEN && strcmp(key->seg[1], "workspace") == 0; + if (key->n == SKIP_ONE && !v && !ts_node_is_null(value) && rs_kind(value, "inline_table")) { + uint32_t a = ts_node_start_byte(value); + uint32_t b = ts_node_end_byte(value); + inherited = b > a && b <= s->src_len && rs_has_word(s->src + a, b - a, "workspace"); + v = "true"; + } + if (key->n > PAIR_LEN || (key->n == PAIR_LEN && !inherited)) { + return; + } + if (t->kind == RS_TABLE_TARGET) { + rs_rec(s, 'a'); + } else { + rs_rec(s, 'K'); + rs_rec_s(s, t->name); + } + rs_sb_putc(&s->sb, '\t'); + rs_sb_puts(&s->sb, key->seg[0]); + if (inherited) { + rs_sb_puts(&s->sb, ".workspace"); + } + rs_rec_s(s, v ? v : "?"); + rs_rec_end(s); +} + +/* The pairs of one table (a `table` or `table_array_element` node). */ +static void rs_manifest_table(rs_scan_t *s, TSNode table, bool array) { + rs_toml_key_t name = {.n = 0, .ok = false}; + rs_table_t t = {.kind = RS_TABLE_OTHER}; + TSTreeCursor cur = ts_tree_cursor_new(table); + bool more = ts_tree_cursor_goto_first_child(&cur); + bool named = false; + for (; more && !s->failed; more = ts_tree_cursor_goto_next_sibling(&cur)) { + TSNode c = ts_tree_cursor_current_node(&cur); + if (!named && rs_toml_is_key(c)) { + named = true; + rs_toml_key_of(s, c, &name); + t = rs_table_class(&name, array); + if (t.is_workspace) { + rs_sb_puts(&s->sb, "W\n"); + } + if (t.kind == RS_TABLE_TARGET) { + rs_rec(s, 'A'); + rs_rec_s(s, t.name); + rs_rec_end(s); + } else if (t.kind == RS_TABLE_DEP && !strpbrk(t.name, "\t\n\r")) { + rs_rec_dep(s, t.dep, t.name, "-", "-"); + } + continue; + } + if (!rs_kind(c, "pair") || t.kind == RS_TABLE_OTHER) { + continue; + } + TSNode k; + TSNode v; + rs_toml_pair(c, &k, &v); + rs_toml_key_t key; + rs_toml_key_of(s, k, &key); + if (!key.ok || key.n == 0) { + continue; + } + if (t.kind == RS_TABLE_DEPS) { + rs_manifest_dep(s, t.dep, &key, v); + } else if (t.kind == RS_TABLE_DEP) { + if (key.n == SKIP_ONE && rs_dep_attr(key.seg[0])) { + const char *val = rs_toml_scalar(s, v); + rs_rec_dep(s, t.dep, t.name, key.seg[0], val ? val : "?"); + } + } else { + rs_manifest_setting(s, &t, &key, v); + } + } + ts_tree_cursor_delete(&cur); +} + +const char *cbm_doclink_rs_manifest_scan_scope(CBMExtractCtx *ctx) { + if (!ctx || !ctx->result || !ctx->rel_path || !ctx->source || ts_node_is_null(ctx->root) || + !rs_is_manifest(ctx->rel_path)) { + return NULL; + } + rs_scan_t s = {.ctx = ctx, + .tmp = rs_scratch(ctx), + .src = ctx->source, + .src_len = ctx->source_len > 0 ? (uint32_t)ctx->source_len : 0, + .sb = {.a = ctx->arena}}; + rs_sb_puts(&s.sb, CBM_DOCLINK_RS_SCOPE_TAG "\nC\n"); + TSTreeCursor cur = ts_tree_cursor_new(ctx->root); + bool more = ts_tree_cursor_goto_first_child(&cur); + for (; more && !s.failed; more = ts_tree_cursor_goto_next_sibling(&cur)) { + TSNode c = ts_tree_cursor_current_node(&cur); + if (rs_kind(c, "table")) { + rs_manifest_table(&s, c, false); + } else if (rs_kind(c, "table_array_element")) { + rs_manifest_table(&s, c, true); + } + } + ts_tree_cursor_delete(&cur); + if (ts_node_has_error(ctx->root)) { + rs_sb_puts(&s.sb, "B\n"); + } + if (s.failed || s.sb.failed || !s.sb.buf) { + return NULL; + } + return s.sb.buf; +} + +/* ── Persisted form ──────────────────────────────────────────────── */ + +/* Field positions (0-based, the tag is field 0) that hold line numbers. */ +static bool rs_line_field(char tag, int field) { + enum { F1 = 1, F2 = 2, F3 = 3, F4 = 4 }; + switch (tag) { + case 'N': + return field == F3 || field == F4; + case 'T': + case 'I': + return field == F2 || field == F3; + case 'M': + return field == F1; + default: + return false; + } +} + +/* Records only the file's own references need: never persisted. */ +static bool rs_fresh_only(char tag) { + return tag == 'P' || tag == 'Z' || tag == 'E'; +} + +char *cbm_doclink_rs_portable_scope(const char *scope) { + size_t n = strlen(scope); + char *out = (char *)cbm_alloc(CBM_MEM_CLASS_OTHER, n + SKIP_ONE); + if (!out) { + return NULL; + } + size_t w = 0; + const char *p = scope; + bool first = true; + while (*p) { + const char *nl = strchr(p, '\n'); + size_t len = nl ? (size_t)(nl - p) : strlen(p); + char tag = first ? 0 : p[0]; + const char *next = nl ? nl + SKIP_ONE : p + len; + if (rs_fresh_only(tag)) { + p = next; + first = false; + continue; + } + int field = 0; + size_t i = 0; + while (i < len) { + size_t fend = i; + while (fend < len && p[fend] != '\t') { + fend++; + } + if (rs_line_field(tag, field)) { + out[w++] = '0'; + } else { + memcpy(out + w, p + i, fend - i); + w += fend - i; + } + i = fend; + if (i < len) { + out[w++] = '\t'; + i++; + field++; + } + } + if (nl) { + out[w++] = '\n'; + } + p = next; + first = false; + } + out[w] = '\0'; + return out; +} diff --git a/src/pipeline/doc_links.c b/src/pipeline/doc_links.c index b486d6267..a365ddf41 100644 --- a/src/pipeline/doc_links.c +++ b/src/pipeline/doc_links.c @@ -47,7 +47,7 @@ const char *cbm_doclink_reason_name(int reason) { static const cbm_doclink_resolver_t *const DOCLINK_RESOLVERS[] = { &cbm_doclink_cs_resolver, &cbm_doclink_md_resolver, &cbm_doclink_pdf_resolver, &cbm_doclink_rst_resolver, &cbm_doclink_adoc_resolver, &cbm_doclink_go_resolver, - &cbm_doclink_jvm_resolver, + &cbm_doclink_jvm_resolver, &cbm_doclink_rs_resolver, }; enum { DOCLINK_RESOLVER_COUNT = sizeof(DOCLINK_RESOLVERS) / sizeof(DOCLINK_RESOLVERS[0]) }; diff --git a/src/pipeline/doc_links.h b/src/pipeline/doc_links.h index 5ea4e46af..a85e3d512 100644 --- a/src/pipeline/doc_links.h +++ b/src/pipeline/doc_links.h @@ -297,6 +297,7 @@ void cbm_doclink_cs_test_unit_memo(bool on); bool cbm_doclinks_binds_lines(CBMLanguage lang); extern const cbm_doclink_resolver_t cbm_doclink_go_resolver; extern const cbm_doclink_resolver_t cbm_doclink_jvm_resolver; +extern const cbm_doclink_resolver_t cbm_doclink_rs_resolver; /* True when `rel_path` is a scope input of some language (scope_input). */ bool cbm_doclinks_is_scope_input(const char *rel_path); diff --git a/src/pipeline/doc_links_rs.c b/src/pipeline/doc_links_rs.c new file mode 100644 index 000000000..0f6c02460 --- /dev/null +++ b/src/pipeline/doc_links_rs.c @@ -0,0 +1,4447 @@ +/* + * doc_links_rs.c — Rust intra-doc link resolution for doc_links.h. + * + * The index is built from the scope blobs of the repository's Rust files and + * Cargo manifests (internal/cbm/doclink_rs.c); nothing is read from disk. + * Graph nodes are looked up by qualified name only, and a node is taken only + * when its label and file are the declaration's, so the closure-repair route + * (line-less proxy nodes) resolves exactly as a full build does. + * + * Model. + * package a Cargo manifest with [package]: its directory, edition and + * dependencies. A dependency names a crate of the repository + * only through `path` (its own, or the workspace table's) + * crate root a target root file of a package: the lib (src/lib.rs or + * [lib] path), src/main.rs, src/bin/, tests/, benches/, + * examples/, build.rs, and explicit target paths + * module a file's root module or an inline `mod x { }`. A `mod x;` + * attaches the file rustc would read: x.rs or x/mod.rs beside a + * crate root, a mod.rs or a `#[path]` file, under / beside + * any other file, or the `#[path]` file. A file can be attached + * more than once and belong to several crate roots; a file no + * root reaches is an orphan + * binding what a module declares under a name: an item, a child + * module, a named import, a name a macro invocation may declare + * + * Lookup follows rustc's name resolution as rustdoc applies it to intra-doc + * links, one namespace (type, value, macro) at a time: + * first segment `crate` (every crate root of the file), `self`, `super` + * (repeatable), `Self` (the impl's self type, the trait, or + * the documented type), `::name` (extern prelude), else + * 1. the module's own bindings + * 2. its glob imports, through re-exporting globs, only + * names visible to the importing module + * 3. the extern prelude: std, core, alloc, proc_macro, + * test, the package's dependencies, and the package's + * own lib from its other targets + * 4. the std prelude, primitive types, std macros + * The first level with a binding in the namespace decides. A + * level whose content is not known (a glob of an external + * module) is not looked past. + * later segments a module's levels 1 and 2; for a type: enum variant, items + * of its inherent impls, then items of trait impls whose + * trait is in scope of the documenting module, then (value + * namespace) fields; for a trait its own items; type aliases + * are followed + * The candidates of the namespaces are united. One entity binds. A trait and + * a derive macro of one name are the trait (rustdoc's rule). Anything else + * with two or more is ambiguous. A disambiguator or suffix restricts the + * namespace and the kind; an entity of another kind is no match. + * + * Nothing is resolved around what the scan could not place: in a file with a + * parse error (or constructs the blob cannot carry) a name its modules do not + * show is a graph gap, never looked up further out; so are a name a macro + * invocation may declare, an import that cannot be followed, a `mod` whose + * file is not indexed, and `crate::` or `super::` from an orphan file. + */ +#include "pipeline/doc_links.h" + +#include "doclink.h" +#include "helpers.h" /* cbm_fqn_compute, cbm_fqn_module_source_lang, cbm_is_test_file */ +#include "foundation/arena.h" +#include "foundation/constants.h" +#include "foundation/hash_table.h" +#include "foundation/log.h" +#include "foundation/mem_core.h" + +#include +#include +#include +#include +#include + +enum { + RS_NS_TYPE = 1, + RS_NS_VALUE = 2, + RS_NS_MACRO = 4, + RS_NS_ALL = 7, + RS_NS_COUNT = 3, +}; + +static const int RS_NS_LIST[RS_NS_COUNT] = {RS_NS_TYPE, RS_NS_VALUE, RS_NS_MACRO}; + +/* What a path names. */ +enum { + RS_E_NONE = 0, + RS_E_ITEM, /* idx: item */ + RS_E_MOD, /* idx: module */ + RS_E_MEMBER, /* idx: member */ + RS_E_IMPL, /* idx: impl whose self type is no item of the repository (`Self` only) */ + RS_E_EXTERNAL, /* outside the repository */ + RS_E_GAP, /* declared in the repository, but not placed or not followed */ +}; + +enum { + RS_SET_CAP = 8, /* distinct candidates kept; more is ambiguous either way */ + RS_ROOTS_KEPT = 4, /* crate roots kept per file; a file of more has no single crate */ + RS_IMPORT_PASSES = 32, /* an import behind a longer chain of imports is not followed */ + RS_ALIAS_HOPS = 8, /* a chain of type aliases; longer is not followed */ + RS_UP_STEPS = 256, /* module ancestors walked for one question; more is unknown */ + RS_GLOB_VISIT = 64, /* modules reached through globs for one name; more is unknown */ + RS_IMPL_SCAN = 256, /* trait impls of one type searched for a default item */ + RS_QN_BUF = 2048, + RS_PATH_BUF = 1024, + RS_PATH_DEPTH = RS_PATH_BUF / 2, /* inline modules around a `mod x;` */ + RS_FIELDS_MAX = 9, + RS_TAIL = 4, /* path components read from the end */ + RS_EDITION_2015 = 2015, + RS_EDITION_2018 = 2018, + RS_EDITION_UNKNOWN = 0, + RS_EDITION_INHERITED = -1, + RS_EDITION_UNREADABLE = -2, + RS_PKG_NONE = -1, + RS_PKG_CONFLICT = -2, + RS_VIEW_A = 1, /* out-of-line children are read beside the file */ + RS_VIEW_B = 2, /* ... from the directory named like the file */ + RS_RS_EXT = 3, /* strlen(".rs") */ +}; + +enum { RS_FILE_OTHER = 0, RS_FILE_SOURCE, RS_FILE_MANIFEST }; +enum { RS_BIND_ITEM = 0, RS_BIND_CHILD, RS_BIND_IMPORT, RS_BIND_MAYBE }; +enum { RS_STATE_NEW = 0, RS_STATE_DONE }; +enum { + RS_TARGET_LIB = 0, + RS_TARGET_BIN, + RS_TARGET_EXAMPLE, + RS_TARGET_TEST, + RS_TARGET_BENCH, + RS_TARGET_BUILD +}; + +typedef struct { + uint8_t kind; /* RS_E_* */ + bool via_import; /* bound by a named import: an exact naming */ + bool weak; /* the name was not found, but is not known to be absent either (a gap + * or an external glob could hold it): a binding found in another + * namespace overrides it */ + int idx; +} rs_ent_t; + +typedef struct { + rs_ent_t v[RS_SET_CAP]; + int n; + bool overflow; +} rs_set_t; + +/* ── Index data ──────────────────────────────────────────────────── */ + +typedef struct { + int file; + int parent; /* the inline parent module; CBM_NOT_FOUND for a file's root module */ + const char *name; + uint32_t start; /* lines: a re-extracted file's only */ + uint32_t end; + char vis; + bool cfg_test; + bool path_attr; /* an inline module with `#[path]`: where its `mod x;` files are is not known */ + bool product; /* reachable as product code: not test-only */ + int bind_first; + int bind_count; + int glob_first; + int glob_count; + int named_first; + int named_count; + int attach_first; /* a file's root module: the `mod x;` declarations naming the file */ + int attach_count; +} rs_mod_t; + +typedef struct { + int mod; /* the declaring module */ + const char *name; + const char *path; /* the #[path] literal, NULL when there is none */ + char vis; + bool cfg_test; + bool cond_path; /* its file depends on cfg: not followed */ + int child; /* the declared file's root module, CBM_NOT_FOUND when not known */ +} rs_decl_t; + +typedef struct { + int child; + int decl; +} rs_attach_t; + +typedef struct { + int file; + int mod; + const char *name; + char kind; /* S E U R A F C G M V B P */ + char vis; + bool test; + bool macro_export; + const char *derives; /* a type's derive names, comma-joined; NULL when it has none */ + uint32_t start; + uint32_t end; + char label; /* node label code, 0 when the item has no node */ + const char *suffix; /* the node's qualified name after the file's item prefix */ + const char *alias; /* A: the aliased path, NULL when it is no path */ + rs_ent_t alias_ent; + int member_first; + int member_count; + int impl_first; /* into rs_index_t.item_impls */ + int impl_count; +} rs_item_t; + +typedef struct { + int file; + int mod; + int owner; /* the item it belongs to, or -(impl) - 1 */ + bool owner_test; + const char *name; + char kind; /* f c t d v */ + char label; + const char *suffix; + int seq; +} rs_member_t; + +typedef struct { + int file; + int mod; + uint32_t start; + uint32_t end; + const char *self_path; /* NULL when the self type is no path */ + const char *trait_path; /* NULL for an inherent impl, "?" when it is no path */ + rs_ent_t self_ent; + rs_ent_t trait_ent; + int member_first; + int member_count; +} rs_impl_t; + +typedef struct { + int item; + int impl; +} rs_item_impl_t; + +/* An impl's member, by the type the impl is for. */ +typedef struct { + int item; + const char *name; + int impl; + int member; +} rs_assoc_t; + +typedef struct { + int file; + int mod; + char vis; + char kind; /* b named, s `self` (type namespace only), g glob, x extern crate */ + const char *local; + const char **segs; + int nsegs; + bool global; + bool spread; /* what it names has no known namespace (external, or a gap): res[] holds + * it in all three, and a binding the module really has in one of them + * takes precedence there */ + int state; + int done_pass; + int seq; + rs_ent_t res[RS_NS_COUNT]; /* named: per namespace; the others: res[0] */ +} rs_import_t; + +typedef struct { + int mod; + const char *name; + int kind; /* RS_BIND_* */ + int ref; + int seq; + char vis; +} rs_bind_t; + +/* An index entry sorted by (mod, seq). */ +typedef struct { + int mod; + int seq; + int ref; +} rs_ref_t; + +/* A name the body of the `macro_rules!` macro `macro` declares (W records). */ +typedef struct { + const char *name; + const char *macro; +} rs_macdecl_t; + +/* Module `mod` invokes a macro of this name among its items (V records). */ +typedef struct { + int mod; + const char *macro; +} rs_macuse_t; + +/* A range of a file in which `Self` means one type: a type item's own + * extent, or an impl block. */ +typedef struct { + uint32_t start; + uint32_t end; + int item; /* CBM_NOT_FOUND for an impl */ + int impl; +} rs_scope_t; + +typedef struct { + uint32_t line; + const char *text; /* P: comma-joined names; Z: the unowned definition's QN suffix */ +} rs_note_t; + +/* From line `from` on, the innermost module is `mod`. */ +typedef struct { + uint32_t from; + int mod; +} rs_span_t; + +typedef struct { + const char *rel_path; + const char *item_prefix; /* qualified name of an item of the file, minus its name */ + const char *module_qn; + int kind; + bool test_path; + bool broken; /* its modules may declare names the blob does not show */ + bool fresh; /* re-extracted in this run: its blob has line numbers */ + bool crate_root; + bool queued; + bool roots_top; /* more crate roots than are kept */ + bool in_lib; /* some crate root of it is a package's lib target */ + uint8_t views; /* RS_VIEW_* its `mod x;` declarations are read under */ + uint8_t views_done; + uint32_t err_line; /* first line whose definitions have no scope; 0: none */ + int mod_base; /* its root module */ + int nmods; + int decl_first; + int decl_count; + int item_first; + int item_count; + int root_pkg; /* a crate root: its package (RS_PKG_*) */ + int pkg; /* the package of its crate roots (RS_PKG_*) */ + int roots[RS_ROOTS_KEPT]; + int nroots; + int scope_first; + int scope_count; + int local_first; /* P records */ + int local_count; + int unowned_first; /* Z records */ + int unowned_count; + int span_first; + int span_count; + int test_first; /* start lines of the items marked as tests */ + int test_count; +} rs_file_t; + +typedef struct { + char kind; /* d dependencies, v dev, b build, w workspace.dependencies */ + const char *name; + const char *attr; /* "-", "path", "package", "workspace" */ + const char *value; + int seq; +} rs_rawdep_t; + +typedef struct { + const char *key; /* the crate's name in scope */ + rs_ent_t target; +} rs_dep_t; + +typedef struct { + int file; + const char *dir; /* "" for the repository root */ + const char *name; + const char *lib_name; /* the lib crate's name; NULL when it has none that is known */ + const char *lib_path; + const char *build; + bool has_package; + bool is_workspace; + bool broken; + bool no_autobins; + bool no_autoexamples; + bool no_autotests; + bool no_autobenches; + int edition; + int ws_edition; + rs_rawdep_t *raw; + int nraw; + const char **targets; /* explicit target paths, as written */ + int ntargets; + rs_dep_t *deps; /* sorted by key */ + int ndeps; + int lib_root; /* the lib target's root module, CBM_NOT_FOUND when it has none */ + int ws; /* its workspace root's package, CBM_NOT_FOUND */ +} rs_pkg_t; + +typedef struct { + CBMArena arena; + bool oom; + const char *project; + rs_file_t *files; + int nfiles; + int nsources; + int *run_to_file; + int run_count; + rs_mod_t *mods; + int nmods; + rs_decl_t *decls; + int ndecls; + rs_attach_t *attach; + int nattach; + rs_item_t *items; + int nitems; + rs_member_t *members; + int nmembers; + rs_impl_t *impls; + int nimpls; + rs_item_impl_t *item_impls; + int nitem_impls; + rs_assoc_t *assoc; + int nassoc; + rs_import_t *imports; + int nimports; + rs_ref_t *globs; /* the glob imports, by module */ + int nglobs; + rs_ref_t *named; /* the named imports, by module */ + int nnamed; + rs_bind_t *binds; + int nbinds; + rs_ref_t *maybes; /* Q records: ref indexes maybe_names */ + const char **maybe_names; + int nmaybes; + rs_macdecl_t *macdecls; /* sorted by (name, macro) */ + int nmacdecls; + rs_macuse_t *macuses; /* sorted by (mod, macro) */ + int nmacuses; + rs_scope_t *scopes; + int nscopes; + rs_note_t *locals; + int nlocals; + rs_note_t *unowned; + int nunowned; + rs_span_t *spans; + int nspans; + uint32_t *test_lines; + int ntest_lines; + rs_pkg_t *pkgs; + int npkgs; + CBMHashTable *file_by_path; /* rel_path -> file + 1 */ + CBMHashTable *pkg_by_dir; /* directory -> package + 1 */ + CBMHashTable *blanket; /* names of members of impls for no named type */ + int crate_roots; + int orphans; + int gap_imports; +} rs_index_t; + +/* ── Small helpers ───────────────────────────────────────────────── */ + +static void *ix_alloc(rs_index_t *ix, size_t n) { + void *p = cbm_arena_alloc(&ix->arena, n ? n : SKIP_ONE); + ix->oom = ix->oom || !p; + return p; +} + +static void *ix_calloc(rs_index_t *ix, size_t count, size_t size) { + size_t n = count * size; + void *p = ix_alloc(ix, n); + if (p) { + memset(p, 0, n ? n : SKIP_ONE); + } + return p; +} + +static char *ix_strdup(rs_index_t *ix, const char *s) { + char *p = cbm_arena_strdup(&ix->arena, s ? s : ""); + ix->oom = ix->oom || !p; + return p; +} + +static char *ix_strndup(rs_index_t *ix, const char *s, size_t n) { + char *p = cbm_arena_strndup(&ix->arena, s, n); + ix->oom = ix->oom || !p; + return p; +} + +/* A name without the raw-identifier marker: `r#type` and `type` are one name. */ +static const char *rs_plain(const char *name) { + return (name[0] == 'r' && name[1] == '#' && name[PAIR_LEN]) ? name + PAIR_LEN : name; +} + +/* Split `line` in place at tabs into at most `max` fields; the count. */ +static int rs_fields(char *line, char **out, int max) { + int n = 0; + out[n++] = line; + for (char *p = line; *p && n < max; p++) { + if (*p == '\t') { + *p = '\0'; + out[n++] = p + SKIP_ONE; + } + } + return n; +} + +static bool rs_ends_with(const char *s, const char *suffix) { + size_t n = strlen(s); + size_t sl = strlen(suffix); + return n >= sl && strcmp(s + n - sl, suffix) == 0; +} + +static const char *rs_basename(const char *rel) { + const char *slash = strrchr(rel, '/'); + return slash ? slash + SKIP_ONE : rel; +} + +/* Append "/" and s[0..n) to the path in out[0..*w); nothing when n is 0. false + * when it does not fit. */ +static bool rs_path_put(char *out, size_t cap, size_t *w, const char *s, size_t n) { + if (n == 0) { + return true; + } + if (*w + n + PAIR_LEN >= cap) { + return false; + } + if (*w > 0) { + out[(*w)++] = '/'; + } + memcpy(out + *w, s, n); + *w += n; + out[*w] = '\0'; + return true; +} + +/* Normalize `dir`/`path` (repository-relative): "." and ".." segments, doubled + * and trailing slashes. false when it leaves the repository or does not fit. */ +static bool rs_join_path(const char *dir, const char *path, char *out, size_t cap) { + char tmp[RS_PATH_BUF]; + int k = (path[0] == '/') ? CBM_NOT_FOUND + : snprintf(tmp, sizeof(tmp), "%s%s%s", dir, dir[0] ? "/" : "", path); + if (k < 0 || (size_t)k >= sizeof(tmp)) { + return false; + } + size_t w = 0; + out[0] = '\0'; + for (const char *p = tmp; *p;) { + const char *slash = strchr(p, '/'); + size_t len = slash ? (size_t)(slash - p) : strlen(p); + const char *next = slash ? slash + SKIP_ONE : p + len; + if (len == PAIR_LEN && p[0] == '.' && p[1] == '.') { + if (w == 0) { + return false; + } + while (w > 0 && out[w - SKIP_ONE] != '/') { + w--; + } + w = w > 0 ? w - SKIP_ONE : 0; + out[w] = '\0'; + } else if (len > 0 && !(len == SKIP_ONE && p[0] == '.')) { + if (!rs_path_put(out, cap, &w, p, len)) { + return false; + } + } + p = next; + } + return true; +} + +static bool rs_is_test_dir(const char *seg, size_t n) { + static const char *const dirs[] = {"tests", "test", "benches", "examples", "fixtures"}; + for (size_t i = 0; i < sizeof(dirs) / sizeof(dirs[0]); i++) { + if (strlen(dirs[i]) == n && memcmp(dirs[i], seg, n) == 0) { + return true; + } + } + return false; +} + +/* Test, bench, example and fixture code by its place in the tree: Cargo's + * target directories, plus the indexer's own test-file rule. */ +static bool rs_is_test_path(const char *rel) { + if (cbm_is_test_file(rel, CBM_LANG_RUST)) { + return true; + } + for (const char *p = rel;;) { + const char *slash = strchr(p, '/'); + if (!slash) { + return false; + } + if (rs_is_test_dir(p, (size_t)(slash - p))) { + return true; + } + p = slash + SKIP_ONE; + } +} + +static int rs_ns_index(int ns) { + return ns == RS_NS_TYPE ? 0 : (ns == RS_NS_VALUE ? SKIP_ONE : PAIR_LEN); +} + +static int rs_item_ns(char kind) { + switch (kind) { + case 'S': + case 'E': + case 'U': + case 'R': + case 'A': + return RS_NS_TYPE; + case 'F': + case 'C': + case 'G': + return RS_NS_VALUE; + default: + return RS_NS_MACRO; + } +} + +/* A variant is a type-namespace name here: its constructor in the value + * namespace is not what a link names (rustdoc drops it). */ +static int rs_member_ns(char kind) { + return (kind == 't' || kind == 'v') ? RS_NS_TYPE : RS_NS_VALUE; +} + +static bool rs_item_is_type(char kind) { + return kind == 'S' || kind == 'E' || kind == 'U' || kind == 'R' || kind == 'A'; +} + +static bool rs_in_list(const char *const *list, size_t n, const char *name) { + for (size_t i = 0; i < n; i++) { + if (strcmp(list[i], name) == 0) { + return true; + } + } + return false; +} + +#define RS_IN(list, name) rs_in_list((list), sizeof(list) / sizeof((list)[0]), (name)) + +/* ── Candidate sets ──────────────────────────────────────────────── */ + +static rs_ent_t rs_ent(uint8_t kind, int idx) { + return (rs_ent_t){.kind = kind, .idx = idx}; +} + +/* RS_E_GAP or RS_E_EXTERNAL for a name that was not found (rs_ent_t.weak). */ +static rs_ent_t rs_weak(uint8_t kind) { + return (rs_ent_t){.kind = kind, .weak = true}; +} + +static bool rs_ent_same(rs_ent_t a, rs_ent_t b) { + return a.kind == b.kind && (a.kind == RS_E_EXTERNAL || a.kind == RS_E_GAP || a.idx == b.idx); +} + +/* Add `e`. Every entity outside the repository counts as one candidate (the + * outcome is `external` whichever it is), and so does every gap. */ +static void rs_set_add(rs_set_t *s, rs_ent_t e) { + if (e.kind == RS_E_NONE) { + return; + } + for (int i = 0; i < s->n; i++) { + if (rs_ent_same(s->v[i], e)) { + s->v[i].via_import = s->v[i].via_import || e.via_import; + s->v[i].weak = s->v[i].weak && e.weak; + return; + } + } + if (s->n >= RS_SET_CAP) { + s->overflow = true; + return; + } + s->v[s->n++] = e; +} + +static void rs_set_merge(rs_set_t *dst, const rs_set_t *src) { + for (int i = 0; i < src->n; i++) { + rs_set_add(dst, src->v[i]); + } + dst->overflow = dst->overflow || src->overflow; +} + +static bool rs_set_empty(const rs_set_t *s) { + return s->n == 0 && !s->overflow; +} + +static bool rs_set_has(const rs_set_t *s, rs_ent_t e) { + for (int i = 0; i < s->n; i++) { + if (rs_ent_same(s->v[i], e)) { + return true; + } + } + return false; +} + +static bool rs_set_equal(const rs_set_t *a, const rs_set_t *b) { + if (a->n != b->n || a->overflow || b->overflow) { + return false; + } + for (int i = 0; i < a->n; i++) { + if (!rs_set_has(b, a->v[i])) { + return false; + } + } + return true; +} + +/* ── Scope blobs: counting ───────────────────────────────────────── */ + +typedef struct { + int mods; + int decls; + int items; + int members; + int impls; + int imports; + int maybes; + int locals; + int unowned; + int deps; + int targets; + int macuses; + int macdecls; +} rs_counts_t; + +/* The number of comma-separated names a W record carries. */ +static int rs_count_names(const char *line, const char *end) { + int n = SKIP_ONE; + for (const char *p = line; p < end; p++) { + n += *p == ','; + } + return n; +} + +static void rs_count_blob(const char *blob, rs_counts_t *c) { + for (const char *p = blob; p && *p;) { + const char *eol = strchr(p, '\n'); + switch (p[0]) { + case 'V': + c->macuses++; + break; + case 'W': + c->macdecls += rs_count_names(p, eol ? eol : p + strlen(p)); + break; + case 'N': + c->mods++; + break; + case 'D': + c->decls++; + break; + case 'T': + c->items++; + break; + case 'M': + c->members++; + break; + case 'I': + c->impls++; + break; + case 'U': + c->imports++; + break; + case 'Q': + c->maybes++; + break; + case 'P': + c->locals++; + break; + case 'Z': + c->unowned++; + break; + case 'X': + c->deps++; + break; + case 'a': + c->targets++; + break; + default: + break; + } + p = eol ? eol + SKIP_ONE : NULL; + } +} + +/* True when `scope` is a manifest's blob: the line after the tag is "C". */ +static bool rs_blob_is_manifest(const char *scope) { + const char *nl = scope ? strchr(scope, '\n') : NULL; + return nl && nl[SKIP_ONE] == 'C' && (nl[PAIR_LEN] == '\n' || nl[PAIR_LEN] == '\0'); +} + +/* ── Scope blobs: a source file ──────────────────────────────────── */ + +/* The node field of a T or M record: '!' for none, else a label code and '=' + * (the node's name is the record's name as written) or ':' and the node's + * name after the item prefix. */ +static void rs_parse_node(const char *field, const char *name, char *label, const char **suffix) { + *label = 0; + *suffix = NULL; + if (!field[0] || field[0] == '!') { + return; + } + if (field[SKIP_ONE] == '=') { + *label = field[0]; + *suffix = name; + } else if (field[SKIP_ONE] == ':' && field[PAIR_LEN]) { + *label = field[0]; + *suffix = field + PAIR_LEN; + } +} + +/* A module field of a record of file `f`: the module, or CBM_NOT_FOUND when + * the number is not one of the file's. */ +static int rs_mod_field(const rs_file_t *f, const char *field) { + char *end = NULL; + long id = strtol(field, &end, CBM_DECIMAL_BASE); + if (end == field || *end || id < 0 || id >= f->nmods) { + return CBM_NOT_FOUND; + } + return f->mod_base + (int)id; +} + +static uint32_t rs_u32_field(const char *field) { + return (uint32_t)strtoul(field, NULL, CBM_DECIMAL_BASE); +} + +/* What is being built while one source blob is read. */ +typedef struct { + rs_index_t *ix; + rs_file_t *f; + int file; + int next_mod; /* the id the next N record must carry */ + int owner_item; /* the T or I record members belong to */ + int owner_impl; + int owner_mod; + bool owner_test; +} rs_reader_t; + +static void rs_read_module(rs_reader_t *r, char **fld, int n) { + enum { F_ID = 1, F_PARENT, F_START, F_END, F_FLAGS, F_NAME, F_COUNT }; + rs_index_t *ix = r->ix; + int parent = n == F_COUNT ? rs_mod_field(r->f, fld[F_PARENT]) : CBM_NOT_FOUND; + if (parent < 0 || (int)rs_u32_field(fld[F_ID]) != r->next_mod || r->next_mod >= r->f->nmods || + parent >= r->f->mod_base + r->next_mod || !fld[F_FLAGS][0]) { + r->f->broken = true; + return; + } + rs_mod_t *m = &ix->mods[r->f->mod_base + r->next_mod]; + r->next_mod++; + m->parent = parent; + m->start = rs_u32_field(fld[F_START]); + m->end = rs_u32_field(fld[F_END]); + m->vis = fld[F_FLAGS][0]; + m->cfg_test = strchr(fld[F_FLAGS] + SKIP_ONE, 't') != NULL; + m->path_attr = strchr(fld[F_FLAGS] + SKIP_ONE, 'c') != NULL; + m->name = rs_plain(fld[F_NAME]); +} + +static void rs_read_decl(rs_reader_t *r, char **fld, int n) { + enum { F_MOD = 1, F_FLAGS, F_NAME, F_PATH, F_COUNT }; + int mod = n == F_COUNT ? rs_mod_field(r->f, fld[F_MOD]) : CBM_NOT_FOUND; + if (mod < 0 || !fld[F_FLAGS][0]) { + r->f->broken = true; + return; + } + rs_decl_t *d = &r->ix->decls[r->ix->ndecls++]; + r->f->decl_count++; + d->mod = mod; + d->vis = fld[F_FLAGS][0]; + d->cfg_test = strchr(fld[F_FLAGS] + SKIP_ONE, 't') != NULL; + d->cond_path = strchr(fld[F_FLAGS] + SKIP_ONE, 'c') != NULL; + d->name = rs_plain(fld[F_NAME]); + d->path = strcmp(fld[F_PATH], "-") == 0 ? NULL : fld[F_PATH]; + d->child = CBM_NOT_FOUND; +} + +static void rs_read_import(rs_reader_t *r, char **fld, int n) { + enum { F_MOD = 1, F_VIS, F_KIND, F_LOCAL, F_PATH, F_COUNT }; + rs_index_t *ix = r->ix; + int mod = n == F_COUNT ? rs_mod_field(r->f, fld[F_MOD]) : CBM_NOT_FOUND; + if (mod < 0) { + r->f->broken = true; + return; + } + rs_import_t *imp = &ix->imports[ix->nimports]; + imp->file = r->file; + imp->mod = mod; + imp->vis = fld[F_VIS][0]; + imp->kind = fld[F_KIND][0]; + imp->local = rs_plain(fld[F_LOCAL]); + imp->seq = ix->nimports++; + CBMDocLinkRsPath path; + if (cbm_doclink_rs_parse_path(fld[F_PATH], strlen(fld[F_PATH]), false, &path) != + CBM_DOCLINK_RS_PATH_OK) { + return; /* no segments: an import that is not followed */ + } + imp->segs = (const char **)ix_alloc(ix, (size_t)path.nsegs * sizeof(char *)); + imp->global = path.global; + for (int i = 0; imp->segs && i < path.nsegs; i++) { + imp->segs[i] = ix_strdup(ix, rs_plain(path.segs[i])); + } + imp->nsegs = imp->segs ? path.nsegs : 0; +} + +static void rs_read_item(rs_reader_t *r, char **fld, int n) { + enum { F_MOD = 1, F_START, F_END, F_FLAGS, F_KIND, F_NAME, F_NODE, F_EXTRA, F_COUNT }; + rs_index_t *ix = r->ix; + int mod = n == F_COUNT ? rs_mod_field(r->f, fld[F_MOD]) : CBM_NOT_FOUND; + r->owner_item = CBM_NOT_FOUND; + r->owner_impl = CBM_NOT_FOUND; + if (mod < 0 || !fld[F_FLAGS][0]) { + r->f->broken = true; + return; + } + int idx = ix->nitems++; + rs_item_t *it = &ix->items[idx]; + r->f->item_count++; + it->file = r->file; + it->mod = mod; + it->start = rs_u32_field(fld[F_START]); + it->end = rs_u32_field(fld[F_END]); + it->vis = fld[F_FLAGS][0]; + it->test = strchr(fld[F_FLAGS] + SKIP_ONE, 't') != NULL; + it->macro_export = strchr(fld[F_FLAGS] + SKIP_ONE, 'e') != NULL; + it->kind = fld[F_KIND][0]; + it->name = rs_plain(fld[F_NAME]); + rs_parse_node(fld[F_NODE], fld[F_NAME], &it->label, &it->suffix); + const char *extra = strcmp(fld[F_EXTRA], "-") != 0 ? fld[F_EXTRA] : NULL; + it->alias = it->kind == 'A' ? extra : NULL; + it->derives = (it->kind == 'S' || it->kind == 'E' || it->kind == 'U') ? extra : NULL; + it->member_first = ix->nmembers; + r->owner_item = idx; + r->owner_mod = mod; + r->owner_test = it->test; + if (r->f->fresh && rs_item_is_type(it->kind)) { + ix->scopes[ix->nscopes++] = + (rs_scope_t){.start = it->start, .end = it->end, .item = idx, .impl = CBM_NOT_FOUND}; + r->f->scope_count++; + } +} + +static void rs_read_impl(rs_reader_t *r, char **fld, int n) { + enum { F_MOD = 1, F_START, F_END, F_SELF, F_TRAIT, F_COUNT }; + rs_index_t *ix = r->ix; + int mod = n == F_COUNT ? rs_mod_field(r->f, fld[F_MOD]) : CBM_NOT_FOUND; + r->owner_item = CBM_NOT_FOUND; + r->owner_impl = CBM_NOT_FOUND; + if (mod < 0) { + r->f->broken = true; + return; + } + int idx = ix->nimpls++; + rs_impl_t *im = &ix->impls[idx]; + im->file = r->file; + im->mod = mod; + im->start = rs_u32_field(fld[F_START]); + im->end = rs_u32_field(fld[F_END]); + im->self_path = strcmp(fld[F_SELF], "?") == 0 ? NULL : fld[F_SELF]; + im->trait_path = strcmp(fld[F_TRAIT], "-") == 0 ? NULL : fld[F_TRAIT]; + im->member_first = ix->nmembers; + r->owner_impl = idx; + r->owner_mod = mod; + r->owner_test = false; + if (r->f->fresh) { + ix->scopes[ix->nscopes++] = + (rs_scope_t){.start = im->start, .end = im->end, .item = CBM_NOT_FOUND, .impl = idx}; + r->f->scope_count++; + } +} + +static void rs_read_member(rs_reader_t *r, char **fld, int n) { + enum { F_START = 1, F_KIND, F_NAME, F_NODE, F_COUNT }; + rs_index_t *ix = r->ix; + if (n != F_COUNT || (r->owner_item < 0 && r->owner_impl < 0)) { + r->f->broken = true; + return; + } + int idx = ix->nmembers++; + rs_member_t *m = &ix->members[idx]; + m->file = r->file; + m->mod = r->owner_mod; + m->owner_test = r->owner_test; + m->kind = fld[F_KIND][0]; + m->name = rs_plain(fld[F_NAME]); + m->seq = idx; + rs_parse_node(fld[F_NODE], fld[F_NAME], &m->label, &m->suffix); + if (r->owner_item >= 0) { + m->owner = r->owner_item; + ix->items[r->owner_item].member_count++; + } else { + m->owner = -r->owner_impl - SKIP_ONE; + ix->impls[r->owner_impl].member_count++; + } +} + +/* A name a macro invocation may declare. One that may be declared anywhere in + * the file ('*') follows a parse error, and such a file is `broken` anyway. */ +static void rs_read_maybe(rs_reader_t *r, char **fld, int n) { + enum { F_MOD = 1, F_NAME, F_COUNT }; + rs_index_t *ix = r->ix; + int mod = n == F_COUNT ? rs_mod_field(r->f, fld[F_MOD]) : CBM_NOT_FOUND; + if (mod < 0) { + r->f->broken = true; /* '*', or a record this code did not write */ + return; + } + ix->maybe_names[ix->nmaybes] = rs_plain(fld[F_NAME]); + ix->maybes[ix->nmaybes] = (rs_ref_t){.mod = mod, .seq = ix->nmaybes, .ref = ix->nmaybes}; + ix->nmaybes++; +} + +/* `V mod name`: the module invokes a macro of that name. */ +static void rs_read_macuse(rs_reader_t *r, char **fld, int n) { + enum { F_MOD = 1, F_NAME, F_COUNT }; + rs_index_t *ix = r->ix; + int mod = n == F_COUNT ? rs_mod_field(r->f, fld[F_MOD]) : CBM_NOT_FOUND; + if (mod < 0) { + r->f->broken = true; + return; + } + ix->macuses[ix->nmacuses++] = (rs_macuse_t){.mod = mod, .macro = rs_plain(fld[F_NAME])}; +} + +/* `W name names`: what the body of the macro `name` declares. */ +static void rs_read_macdecl(rs_reader_t *r, char **fld, int n) { + enum { F_MACRO = 1, F_NAMES, F_COUNT }; + rs_index_t *ix = r->ix; + if (n != F_COUNT) { + r->f->broken = true; + return; + } + const char *macro = rs_plain(fld[F_MACRO]); + for (char *p = fld[F_NAMES]; p && *p;) { + char *comma = strchr(p, ','); + if (comma) { + *comma = '\0'; + } + ix->macdecls[ix->nmacdecls++] = (rs_macdecl_t){.name = rs_plain(p), .macro = macro}; + p = comma ? comma + SKIP_ONE : NULL; + } +} + +static void rs_read_note(rs_reader_t *r, char **fld, int n, bool local) { + enum { F_LINE = 1, F_TEXT, F_COUNT }; + rs_index_t *ix = r->ix; + if (n != F_COUNT || !r->f->fresh) { + return; + } + rs_note_t note = {.line = rs_u32_field(fld[F_LINE]), .text = fld[F_TEXT]}; + if (local) { + ix->locals[ix->nlocals++] = note; + r->f->local_count++; + } else { + ix->unowned[ix->nunowned++] = note; + r->f->unowned_count++; + } +} + +static void rs_read_record(rs_reader_t *r, char *line) { + char *fld[RS_FIELDS_MAX]; + char tag = line[0]; + int n = rs_fields(line, fld, RS_FIELDS_MAX); + switch (tag) { + case 'B': + r->f->broken = true; + break; + case 'E': + if (n == PAIR_LEN && r->f->fresh) { + r->f->err_line = rs_u32_field(fld[SKIP_ONE]); + } + break; + case 'N': + rs_read_module(r, fld, n); + break; + case 'D': + rs_read_decl(r, fld, n); + break; + case 'U': + rs_read_import(r, fld, n); + break; + case 'T': + rs_read_item(r, fld, n); + break; + case 'I': + rs_read_impl(r, fld, n); + break; + case 'M': + rs_read_member(r, fld, n); + break; + case 'Q': + rs_read_maybe(r, fld, n); + break; + case 'V': + rs_read_macuse(r, fld, n); + break; + case 'W': + rs_read_macdecl(r, fld, n); + break; + case 'P': + rs_read_note(r, fld, n, true); + break; + case 'Z': + rs_read_note(r, fld, n, false); + break; + default: + break; + } +} + +/* Read the scope blob of source file `fi` (its modules are allocated). */ +static void rs_read_source(rs_index_t *ix, int fi, const char *blob) { + rs_file_t *f = &ix->files[fi]; + f->decl_first = ix->ndecls; + f->item_first = ix->nitems; + f->scope_first = ix->nscopes; + f->local_first = ix->nlocals; + f->unowned_first = ix->nunowned; + char *buf = blob ? ix_strdup(ix, blob) : NULL; + if (!buf) { + f->broken = true; /* no scope: what the file declares is not known */ + return; + } + rs_reader_t r = {.ix = ix, + .f = f, + .file = fi, + .next_mod = SKIP_ONE, + .owner_item = CBM_NOT_FOUND, + .owner_impl = CBM_NOT_FOUND}; + bool first = true; + for (char *line = buf; line && *line;) { + char *nl = strchr(line, '\n'); + if (nl) { + *nl = '\0'; + } + if (!first) { + rs_read_record(&r, line); + } + first = false; + line = nl ? nl + SKIP_ONE : NULL; + } + if (r.next_mod != f->nmods) { + f->broken = true; /* a module record was not read: its number names nothing */ + } +} + +/* ── Scope blobs: a Cargo manifest ───────────────────────────────── */ + +static int rs_parse_edition(const char *value) { + char *end = NULL; + long year = strtol(value, &end, CBM_DECIMAL_BASE); + if (end == value || *end || year < RS_EDITION_2015) { + return RS_EDITION_UNREADABLE; + } + return (int)year; +} + +static bool rs_is_false(const char *value) { + return strcmp(value, "false") == 0; +} + +static void rs_package_key(rs_pkg_t *p, const char *key, const char *value) { + if (strcmp(key, "name") == 0) { + p->name = strcmp(value, "?") != 0 ? value : NULL; + } else if (strcmp(key, "edition") == 0) { + p->edition = rs_parse_edition(value); + } else if (strcmp(key, "edition.workspace") == 0) { + p->edition = RS_EDITION_INHERITED; + } else if (strcmp(key, "build") == 0) { + p->build = value; + } else if (strcmp(key, "autobins") == 0) { + p->no_autobins = rs_is_false(value); + } else if (strcmp(key, "autoexamples") == 0) { + p->no_autoexamples = rs_is_false(value); + } else if (strcmp(key, "autotests") == 0) { + p->no_autotests = rs_is_false(value); + } else if (strcmp(key, "autobenches") == 0) { + p->no_autobenches = rs_is_false(value); + } +} + +static void rs_manifest_key(rs_pkg_t *p, const char *table, const char *key, const char *value) { + if (strcmp(table, "package") == 0) { + p->has_package = true; + rs_package_key(p, key, value); + } else if (strcmp(table, "lib") == 0) { + if (strcmp(key, "name") == 0) { + p->lib_name = value; + } else if (strcmp(key, "path") == 0) { + p->lib_path = value; + } + } else if (strcmp(table, "workspace.package") == 0 && strcmp(key, "edition") == 0) { + p->ws_edition = rs_parse_edition(value); + } +} + +static void rs_manifest_record(rs_pkg_t *p, char *line) { + enum { F_1 = 1, F_2, F_3, F_4, K_FIELDS = 4, A_FIELDS = 3, X_FIELDS = 5 }; + char *fld[X_FIELDS]; + char tag = line[0]; + int n = rs_fields(line, fld, X_FIELDS); + if (tag == 'K' && n == K_FIELDS) { + rs_manifest_key(p, fld[F_1], fld[F_2], fld[F_3]); + } else if (tag == 'a' && n == A_FIELDS && strcmp(fld[F_1], "path") == 0) { + p->targets[p->ntargets++] = fld[F_2]; + } else if (tag == 'W') { + p->is_workspace = true; + } else if (tag == 'X' && n == X_FIELDS && fld[F_1][0]) { + p->raw[p->nraw] = (rs_rawdep_t){.kind = fld[F_1][0], + .name = fld[F_2], + .attr = fld[F_3], + .value = fld[F_4], + .seq = p->nraw}; + p->nraw++; + } else if (tag == 'B') { + p->broken = true; + } +} + +static void rs_read_manifest(rs_index_t *ix, int fi, const char *blob) { + rs_pkg_t *p = &ix->pkgs[ix->npkgs]; + const char *rel = ix->files[fi].rel_path; + const char *slash = strrchr(rel, '/'); + rs_counts_t c = {0}; + rs_count_blob(blob, &c); + memset(p, 0, sizeof(*p)); + p->file = fi; + p->dir = ix_strndup(ix, rel, slash ? (size_t)(slash - rel) : 0); + p->lib_root = CBM_NOT_FOUND; + p->ws = CBM_NOT_FOUND; + p->raw = (rs_rawdep_t *)ix_alloc(ix, (size_t)c.deps * sizeof(rs_rawdep_t)); + p->targets = (const char **)ix_alloc(ix, (size_t)c.targets * sizeof(char *)); + char *buf = ix_strdup(ix, blob); + if (!buf || !p->dir || !p->raw || !p->targets) { + return; + } + int skipped = 0; + for (char *line = buf; line && *line;) { + char *nl = strchr(line, '\n'); + if (nl) { + *nl = '\0'; + } + if (skipped >= PAIR_LEN) { /* after the tag line and the "C" line */ + rs_manifest_record(p, line); + } + skipped++; + line = nl ? nl + SKIP_ONE : NULL; + } + ix->npkgs++; +} + +/* ── Paths -> files and packages ─────────────────────────────────── */ + +static int rs_file_at(const rs_index_t *ix, const char *rel) { + return (int)(intptr_t)cbm_ht_get(ix->file_by_path, rel) - SKIP_ONE; +} + +/* The source file at `rel`, CBM_NOT_FOUND when none is indexed. */ +static int rs_source_at(const rs_index_t *ix, const char *rel) { + int fi = rs_file_at(ix, rel); + return (fi >= 0 && ix->files[fi].kind == RS_FILE_SOURCE) ? fi : CBM_NOT_FOUND; +} + +/* The package whose manifest is in directory `dir` ("" at the root). */ +static int rs_pkg_at(const rs_index_t *ix, const char *dir) { + return (int)(intptr_t)cbm_ht_get(ix->pkg_by_dir, dir[0] ? dir : ".") - SKIP_ONE; +} + +/* `-` in a package name is `_` in the crate's name. */ +static const char *rs_crate_name(rs_index_t *ix, const char *name) { + char *out = ix_strdup(ix, name); + for (char *p = out; p && *p; p++) { + if (*p == '-') { + *p = '_'; + } + } + return out; +} + +/* The workspace root of package `pi`: the nearest manifest with a [workspace] + * table at or above its directory. */ +static int rs_workspace_of(const rs_index_t *ix, int pi) { + char dir[RS_PATH_BUF]; + if (snprintf(dir, sizeof(dir), "%s", ix->pkgs[pi].dir) >= (int)sizeof(dir)) { + return CBM_NOT_FOUND; + } + for (;;) { + int q = rs_pkg_at(ix, dir); + if (q >= 0 && ix->pkgs[q].is_workspace) { + return q; + } + if (!dir[0]) { + return CBM_NOT_FOUND; + } + char *slash = strrchr(dir, '/'); + if (slash) { + *slash = '\0'; + } else { + dir[0] = '\0'; + } + } +} + +/* The edition of package `pkg`: a year, or RS_EDITION_UNKNOWN. A manifest + * without the key is edition 2015 (Cargo's default). */ +static int rs_edition(const rs_index_t *ix, int pkg) { + if (pkg < 0) { + return RS_EDITION_UNKNOWN; + } + const rs_pkg_t *p = &ix->pkgs[pkg]; + int edition = p->edition; + if (edition == RS_EDITION_INHERITED) { + edition = p->ws >= 0 ? ix->pkgs[p->ws].ws_edition : RS_EDITION_UNREADABLE; + } + if (edition == 0) { + return p->broken ? RS_EDITION_UNKNOWN : RS_EDITION_2015; + } + return edition > 0 ? edition : RS_EDITION_UNKNOWN; +} + +/* ── Crate roots ─────────────────────────────────────────────────── */ + +static void rs_mark_root(rs_index_t *ix, int fi, int pi, bool lib) { + rs_file_t *f = &ix->files[fi]; + if (!f->crate_root) { + f->crate_root = true; + f->root_pkg = pi; + ix->crate_roots++; + } else if (f->root_pkg != pi) { + f->root_pkg = RS_PKG_CONFLICT; /* a target of two packages */ + } + if (lib && ix->pkgs[pi].lib_root < 0) { + ix->pkgs[pi].lib_root = f->mod_base; + } +} + +/* A target path written in a manifest. */ +static void rs_claim_path(rs_index_t *ix, int pi, const char *path, bool lib) { + char rel[RS_PATH_BUF]; + if (strcmp(path, "?") == 0 || !rs_join_path(ix->pkgs[pi].dir, path, rel, sizeof(rel))) { + return; + } + int fi = rs_source_at(ix, rel); + if (fi >= 0) { + rs_mark_root(ix, fi, pi, lib); + } +} + +static void rs_explicit_roots(rs_index_t *ix, int pi) { + const rs_pkg_t *p = &ix->pkgs[pi]; + if (!p->has_package) { + return; + } + if (p->lib_path) { + rs_claim_path(ix, pi, p->lib_path, true); + } + for (int i = 0; i < p->ntargets; i++) { + rs_claim_path(ix, pi, p->targets[i], false); + } + if (p->build && !rs_is_false(p->build) && strcmp(p->build, "true") != 0) { + rs_claim_path(ix, pi, p->build, false); + } +} + +/* The last components of a path, c[0] the file name. */ +typedef struct { + const char *c[RS_TAIL]; + size_t len[RS_TAIL]; + int n; +} rs_tail_t; + +static void rs_tail(const char *rel, rs_tail_t *t) { + memset(t, 0, sizeof(*t)); + const char *end = rel + strlen(rel); + while (t->n < RS_TAIL && end > rel) { + const char *s = end; + while (s > rel && s[-SKIP_ONE] != '/') { + s--; + } + t->c[t->n] = s; + t->len[t->n] = (size_t)(end - s); + t->n++; + end = s > rel ? s - SKIP_ONE : rel; + } +} + +static bool rs_tail_is(const rs_tail_t *t, int k, const char *name) { + return k < t->n && strlen(name) == t->len[k] && memcmp(t->c[k], name, t->len[k]) == 0; +} + +/* The package whose directory holds component `k` of `rel`. */ +static int rs_pkg_before(const rs_index_t *ix, const char *rel, const rs_tail_t *t, int k) { + char dir[RS_PATH_BUF]; + if (k >= t->n) { + return CBM_NOT_FOUND; + } + size_t dl = (size_t)(t->c[k] - rel); + size_t n = dl > 0 ? dl - SKIP_ONE : 0; + if (n >= sizeof(dir)) { + return CBM_NOT_FOUND; + } + memcpy(dir, rel, n); + dir[n] = '\0'; + return rs_pkg_at(ix, dir); +} + +/* A file where Cargo discovers a target of package `pi` on its own. */ +static void rs_claim_auto(rs_index_t *ix, int fi, int pi, int target) { + if (pi < 0 || !ix->pkgs[pi].has_package) { + return; + } + const rs_pkg_t *p = &ix->pkgs[pi]; + bool off = + (target == RS_TARGET_LIB && p->lib_path) || (target == RS_TARGET_BIN && p->no_autobins) || + (target == RS_TARGET_EXAMPLE && p->no_autoexamples) || + (target == RS_TARGET_TEST && p->no_autotests) || + (target == RS_TARGET_BENCH && p->no_autobenches) || (target == RS_TARGET_BUILD && p->build); + if (!off) { + rs_mark_root(ix, fi, pi, target == RS_TARGET_LIB); + } +} + +/* Cargo's target auto-discovery, read off the file's own path. */ +static void rs_auto_roots(rs_index_t *ix, int fi) { + static const struct { + const char *dir; + int target; + } dirs[] = { + {"examples", RS_TARGET_EXAMPLE}, {"tests", RS_TARGET_TEST}, {"benches", RS_TARGET_BENCH}}; + enum { C0 = 0, C1, C2, C3 }; + const char *rel = ix->files[fi].rel_path; + rs_tail_t t; + rs_tail(rel, &t); + bool main_rs = rs_tail_is(&t, C0, "main.rs"); + if (rs_tail_is(&t, C1, "src")) { + if (rs_tail_is(&t, C0, "lib.rs")) { + rs_claim_auto(ix, fi, rs_pkg_before(ix, rel, &t, C1), RS_TARGET_LIB); + } + if (main_rs) { + rs_claim_auto(ix, fi, rs_pkg_before(ix, rel, &t, C1), RS_TARGET_BIN); + } + } + if (rs_tail_is(&t, C1, "bin") && rs_tail_is(&t, C2, "src")) { + rs_claim_auto(ix, fi, rs_pkg_before(ix, rel, &t, C2), RS_TARGET_BIN); + } + if (main_rs && rs_tail_is(&t, C2, "bin") && rs_tail_is(&t, C3, "src")) { + rs_claim_auto(ix, fi, rs_pkg_before(ix, rel, &t, C3), RS_TARGET_BIN); + } + for (size_t i = 0; i < sizeof(dirs) / sizeof(dirs[0]); i++) { + if (rs_tail_is(&t, C1, dirs[i].dir)) { + rs_claim_auto(ix, fi, rs_pkg_before(ix, rel, &t, C1), dirs[i].target); + } + if (main_rs && rs_tail_is(&t, C2, dirs[i].dir)) { + rs_claim_auto(ix, fi, rs_pkg_before(ix, rel, &t, C2), dirs[i].target); + } + } + if (rs_tail_is(&t, C0, "build.rs")) { + rs_claim_auto(ix, fi, rs_pkg_before(ix, rel, &t, C0), RS_TARGET_BUILD); + } +} + +static void rs_crate_roots(rs_index_t *ix) { + for (int pi = 0; pi < ix->npkgs; pi++) { + ix->pkgs[pi].ws = rs_workspace_of(ix, pi); + rs_explicit_roots(ix, pi); + } + for (int fi = 0; fi < ix->nfiles; fi++) { + if (ix->files[fi].kind == RS_FILE_SOURCE) { + rs_auto_roots(ix, fi); + } + } + for (int pi = 0; pi < ix->npkgs; pi++) { + rs_pkg_t *p = &ix->pkgs[pi]; + const char *name = p->lib_name ? p->lib_name : p->name; + p->lib_name = (name && strcmp(name, "?") != 0) ? rs_crate_name(ix, name) : NULL; + } +} + +/* ── `mod x;` -> file ────────────────────────────────────────────── */ + +/* The directory the file of declaration `d` is looked for in, under view + * `view` of the declaring file: the file's directory, the file's stem (view + * B), then the inline modules around the declaration. A `#[path]` directly in + * a file is relative to the file's directory under either view. false when + * it is not known (a `#[path]` on an inline module) or too long to be a path. */ +static bool rs_decl_dir(const rs_index_t *ix, const rs_decl_t *d, int view, char *out, size_t cap) { + const char *names[RS_PATH_DEPTH]; + int n = 0; + int m = d->mod; + while (ix->mods[m].parent >= 0) { + if (ix->mods[m].path_attr || n >= RS_PATH_DEPTH) { + return false; + } + names[n++] = ix->mods[m].name; + m = ix->mods[m].parent; + } + const char *rel = ix->files[ix->mods[m].file].rel_path; + const char *base = rs_basename(rel); + size_t w = 0; + out[0] = '\0'; + if (base > rel && !rs_path_put(out, cap, &w, rel, (size_t)(base - rel) - SKIP_ONE)) { + return false; + } + bool stem = view == RS_VIEW_B && !(d->path && n == 0); + size_t bl = strlen(base); + if (stem && (bl <= RS_RS_EXT || !rs_path_put(out, cap, &w, base, bl - RS_RS_EXT))) { + return false; + } + for (int i = n - SKIP_ONE; i >= 0; i--) { + if (!rs_path_put(out, cap, &w, names[i], strlen(names[i]))) { + return false; + } + } + return true; +} + +/* A file a declaration can name, and the view the file is then read under. */ +typedef struct { + int file; + uint8_t implies; +} rs_cand_t; + +static int rs_cand_add(const rs_index_t *ix, const char *rel, uint8_t implies, rs_cand_t *out, + int n) { + int fi = rs_source_at(ix, rel); + if (fi >= 0) { + out[n++] = (rs_cand_t){.file = fi, .implies = implies}; + } + return n; +} + +/* The indexed files declaration `d` can name under `view`: rustc reads + * `/.rs` (its own children then live under `/`) or + * `//mod.rs`, or the `#[path]` file (children beside it). */ +static int rs_decl_cands(const rs_index_t *ix, const rs_decl_t *d, int view, + rs_cand_t out[PAIR_LEN]) { + char dir[RS_PATH_BUF]; + char path[RS_PATH_BUF]; + int n = 0; + if (d->cond_path || !rs_decl_dir(ix, d, view, dir, sizeof(dir))) { + return 0; + } + if (d->path) { + if (rs_join_path(dir, d->path, path, sizeof(path))) { + n = rs_cand_add(ix, path, RS_VIEW_A, out, n); + } + return n; + } + const char *sep = dir[0] ? "/" : ""; + if (snprintf(path, sizeof(path), "%s%s%s.rs", dir, sep, d->name) < (int)sizeof(path)) { + n = rs_cand_add(ix, path, RS_VIEW_B, out, n); + } + if (snprintf(path, sizeof(path), "%s%s%s/mod.rs", dir, sep, d->name) < (int)sizeof(path)) { + n = rs_cand_add(ix, path, RS_VIEW_A, out, n); + } + return n; +} + +typedef struct { + int *v; + int n; +} rs_stack_t; + +static void rs_views_push(rs_index_t *ix, rs_stack_t *st, int fi, uint8_t bits) { + rs_file_t *f = &ix->files[fi]; + if ((f->views | bits) == f->views) { + return; + } + f->views |= bits; + if (!f->queued) { + f->queued = true; + st->v[st->n++] = fi; + } +} + +/* Propagate views until nothing changes. A file is processed once per view it + * gains, so the work is linear in the declarations. */ +static void rs_views_run(rs_index_t *ix, rs_stack_t *st) { + while (st->n > 0) { + int fi = st->v[--st->n]; + rs_file_t *f = &ix->files[fi]; + f->queued = false; + uint8_t todo = f->views & (uint8_t)~f->views_done; + f->views_done |= todo; + for (int view = RS_VIEW_A; view <= RS_VIEW_B; view++) { + for (int d = f->decl_first; (todo & view) && d < f->decl_first + f->decl_count; d++) { + rs_cand_t c[PAIR_LEN]; + int n = rs_decl_cands(ix, &ix->decls[d], view, c); + for (int k = 0; k < n; k++) { + rs_views_push(ix, st, c[k].file, c[k].implies); + } + } + } + } +} + +/* The one file declaration `d` names under the views of its file, or + * CBM_NOT_FOUND when there is none or more than one. */ +static int rs_decl_file(const rs_index_t *ix, const rs_decl_t *d) { + const rs_file_t *f = &ix->files[ix->mods[d->mod].file]; + int found = CBM_NOT_FOUND; + for (int view = RS_VIEW_A; view <= RS_VIEW_B; view++) { + rs_cand_t c[PAIR_LEN]; + int n = (f->views & view) ? rs_decl_cands(ix, d, view, c) : 0; + for (int k = 0; k < n; k++) { + if (found >= 0 && found != c[k].file) { + return CBM_NOT_FOUND; + } + found = c[k].file; + } + } + return found; +} + +static int rs_attach_cmp(const void *a, const void *b) { + const rs_attach_t *x = (const rs_attach_t *)a; + const rs_attach_t *y = (const rs_attach_t *)b; + if (x->child != y->child) { + return x->child < y->child ? -1 : 1; + } + return (x->decl > y->decl) - (x->decl < y->decl); +} + +/* Decide which file every `mod x;` names. + * + * Where rustc looks depends on how the DECLARING file was reached: beside it + * (view A: a crate root, a mod.rs, a `#[path]` file) or under a directory + * named like it (view B: `name.rs` found by its parent). Views are propagated + * from the crate roots to a fixed point; a file nothing reaches may be either, + * and so may one reached both ways. Under both views a declaration names a + * file only when exactly one indexed file fits. */ +static void rs_module_tree(rs_index_t *ix) { + rs_stack_t st = {.v = (int *)ix_alloc(ix, (size_t)ix->nfiles * sizeof(int))}; + ix->attach = (rs_attach_t *)ix_alloc(ix, (size_t)ix->ndecls * sizeof(rs_attach_t)); + if (!st.v || !ix->attach) { + return; + } + for (int fi = 0; fi < ix->nfiles; fi++) { + const rs_file_t *f = &ix->files[fi]; + bool beside = f->crate_root || strcmp(rs_basename(f->rel_path), "mod.rs") == 0; + if (f->kind == RS_FILE_SOURCE && beside) { + rs_views_push(ix, &st, fi, RS_VIEW_A); + } + } + rs_views_run(ix, &st); + for (int fi = 0; fi < ix->nfiles; fi++) { + if (ix->files[fi].kind == RS_FILE_SOURCE && ix->files[fi].views == 0) { + rs_views_push(ix, &st, fi, RS_VIEW_A | RS_VIEW_B); + } + } + rs_views_run(ix, &st); + for (int d = 0; d < ix->ndecls; d++) { + int fi = rs_decl_file(ix, &ix->decls[d]); + if (fi >= 0) { + ix->decls[d].child = ix->files[fi].mod_base; + ix->attach[ix->nattach++] = (rs_attach_t){.child = ix->files[fi].mod_base, .decl = d}; + } + } + if (ix->nattach > SKIP_ONE) { + qsort(ix->attach, (size_t)ix->nattach, sizeof(rs_attach_t), rs_attach_cmp); + } + for (int i = 0; i < ix->nattach; i++) { + rs_mod_t *child = &ix->mods[ix->attach[i].child]; + if (child->attach_count == 0) { + child->attach_first = i; + } + child->attach_count++; + } +} + +/* ── Crate roots of every file, product code ─────────────────────── */ + +static int rs_pkg_join(int a, int b) { + if (a == RS_PKG_NONE || a == b) { + return b; + } + return b == RS_PKG_NONE ? a : RS_PKG_CONFLICT; +} + +static bool rs_roots_add(rs_file_t *f, int root) { + for (int i = 0; i < f->nroots; i++) { + if (f->roots[i] == root) { + return false; + } + } + if (f->nroots >= RS_ROOTS_KEPT) { + bool changed = !f->roots_top; + f->roots_top = true; + return changed; + } + f->roots[f->nroots++] = root; + return true; +} + +/* What file `c` takes over from a file that declares it. true on a change. */ +static bool rs_file_inherit(rs_file_t *c, const rs_file_t *p) { + bool changed = p->roots_top && !c->roots_top; + c->roots_top = c->roots_top || p->roots_top; + for (int i = 0; i < p->nroots; i++) { + changed = rs_roots_add(c, p->roots[i]) || changed; + } + int pkg = rs_pkg_join(c->pkg, p->pkg); + changed = changed || pkg != c->pkg || (p->in_lib && !c->in_lib); + c->pkg = pkg; + c->in_lib = c->in_lib || p->in_lib; + return changed; +} + +/* The crate roots, the package and the lib membership of every file: a + * fixed point over the `mod x;` edges. Each value only grows and is bounded, + * so a file is revisited a bounded number of times. */ +static void rs_file_roots(rs_index_t *ix) { + rs_stack_t st = {.v = (int *)ix_alloc(ix, (size_t)ix->nfiles * sizeof(int))}; + if (!st.v) { + return; + } + for (int fi = 0; fi < ix->nfiles; fi++) { + rs_file_t *f = &ix->files[fi]; + if (f->kind != RS_FILE_SOURCE || !f->crate_root) { + continue; + } + f->roots[f->nroots++] = f->mod_base; + f->pkg = f->root_pkg; + f->in_lib = f->root_pkg >= 0 && ix->pkgs[f->root_pkg].lib_root == f->mod_base; + f->queued = true; + st.v[st.n++] = fi; + } + while (st.n > 0) { + int fi = st.v[--st.n]; + rs_file_t *f = &ix->files[fi]; + f->queued = false; + for (int d = f->decl_first; d < f->decl_first + f->decl_count; d++) { + int child = ix->decls[d].child; + if (child < 0) { + continue; + } + rs_file_t *c = &ix->files[ix->mods[child].file]; + if (rs_file_inherit(c, f) && !c->queued) { + c->queued = true; + st.v[st.n++] = ix->mods[child].file; + } + } + } +} + +/* A file no crate root reaches still lies in some package's directory: its + * dependencies are that package's. Which target it would belong to (and so + * what `crate` is) stays unknown. */ +static void rs_orphan_packages(rs_index_t *ix) { + for (int fi = 0; fi < ix->nfiles; fi++) { + rs_file_t *f = &ix->files[fi]; + char dir[RS_PATH_BUF]; + const char *base = rs_basename(f->rel_path); + size_t dl = base > f->rel_path ? (size_t)(base - f->rel_path) - SKIP_ONE : 0; + if (f->kind != RS_FILE_SOURCE || f->nroots > 0 || f->pkg != RS_PKG_NONE || + dl >= sizeof(dir)) { + continue; + } + memcpy(dir, f->rel_path, dl); + dir[dl] = '\0'; + for (;;) { + int q = rs_pkg_at(ix, dir); + if (q >= 0 && ix->pkgs[q].has_package) { + f->pkg = q; + break; + } + char *slash = strrchr(dir, '/'); + if (!dir[0]) { + break; + } + dir[slash ? (size_t)(slash - dir) : 0] = '\0'; + } + } +} + +static void rs_product_seed(rs_index_t *ix, rs_stack_t *st, int fi) { + const rs_file_t *f = &ix->files[fi]; + rs_mod_t *root = &ix->mods[f->mod_base]; + if (root->product || f->test_path) { + return; + } + root->product = true; + st->v[st->n++] = fi; +} + +/* Product code: what is reached from a crate root (or an orphan file) outside + * the test directories without passing a `#[cfg(test)]` module or + * declaration. Everything else is test-only. */ +static void rs_product(rs_index_t *ix) { + rs_stack_t st = {.v = (int *)ix_alloc(ix, (size_t)ix->nfiles * sizeof(int))}; + if (!st.v) { + return; + } + for (int fi = 0; fi < ix->nfiles; fi++) { + const rs_file_t *f = &ix->files[fi]; + if (f->kind != RS_FILE_SOURCE) { + continue; + } + bool orphan = !f->crate_root && ix->mods[f->mod_base].attach_count == 0; + ix->orphans += orphan; + if (f->crate_root || orphan) { + rs_product_seed(ix, &st, fi); + } + } + while (st.n > 0) { + const rs_file_t *f = &ix->files[st.v[--st.n]]; + for (int m = SKIP_ONE; m < f->nmods; m++) { + rs_mod_t *mod = &ix->mods[f->mod_base + m]; + mod->product = mod->parent >= 0 && ix->mods[mod->parent].product && !mod->cfg_test; + } + for (int d = f->decl_first; d < f->decl_first + f->decl_count; d++) { + const rs_decl_t *decl = &ix->decls[d]; + if (decl->child >= 0 && !decl->cfg_test && ix->mods[decl->mod].product) { + rs_product_seed(ix, &st, ix->mods[decl->child].file); + } + } + } +} + +/* ── Lookup: state ───────────────────────────────────────────────── */ + +typedef struct { + const rs_index_t *ix; + int pass; /* while imports are resolved: one done in an earlier pass may be read; + * 0: all are done */ + bool assoc; /* associated items are looked up (not while the index is built) */ + bool blocked; /* an import the answer depends on is not resolved yet */ +} rs_look_t; + +/* Where a path is written. */ +typedef struct { + rs_look_t *lk; + int mod; /* the module */ + int root; /* the crate root module it is resolved under; CBM_NOT_FOUND: not known */ + int pkg; /* that crate's package; CBM_NOT_FOUND: not known */ + bool own_lib; /* the package's lib crate is in the extern prelude */ + int edition; + int skip; /* the import being resolved; CBM_NOT_FOUND */ +} rs_at_t; + +/* A path as plain segments (no raw-identifier markers). */ +typedef struct { + const char *const *segs; + int n; + bool global; /* written with a leading `::` */ +} rs_path_t; + +/* What a path names. */ +typedef struct { + rs_set_t ns[RS_NS_COUNT]; /* the last segment, per namespace */ + rs_set_t owners; /* what the last segment was looked up in */ + bool ambiguous; /* a segment before the last names several things */ +} rs_found_t; + +static bool rs_is(const char *s, const char *word) { + return strcmp(s, word) == 0; +} + +static bool rs_import_ready(const rs_look_t *lk, const rs_import_t *imp) { + return imp->state == RS_STATE_DONE && (lk->pass == 0 || imp->done_pass < lk->pass); +} + +/* First binding in binds[lo, hi) that is not below `name` (`upper`: that is + * above it). */ +static int rs_bind_bound(const rs_index_t *ix, int lo, int hi, const char *name, bool upper) { + while (lo < hi) { + int mid = lo + ((hi - lo) / PAIR_LEN); + int c = strcmp(ix->binds[mid].name, name); + if (c < 0 || (upper && c == 0)) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + return lo; +} + +/* The bindings of module `m` named `name`: their count, the first in *first. */ +static int rs_bind_range(const rs_index_t *ix, int m, const char *name, int *first) { + const rs_mod_t *mod = &ix->mods[m]; + int end = mod->bind_first + mod->bind_count; + *first = rs_bind_bound(ix, mod->bind_first, end, name, false); + return rs_bind_bound(ix, *first, end, name, true) - *first; +} + +static int rs_member_bound(const rs_index_t *ix, int lo, int hi, const char *name, bool upper) { + while (lo < hi) { + int mid = lo + ((hi - lo) / PAIR_LEN); + int c = strcmp(ix->members[mid].name, name); + if (c < 0 || (upper && c == 0)) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + return lo; +} + +/* The members named `name` among members[first, first + count) (sorted by + * name) whose kind is one of `kinds` and whose namespace is in `ns`. */ +static void rs_owner_members(const rs_index_t *ix, int first, int count, const char *name, int ns, + const char *kinds, rs_set_t *out) { + int lo = rs_member_bound(ix, first, first + count, name, false); + int hi = rs_member_bound(ix, lo, first + count, name, true); + for (int i = lo; i < hi && !out->overflow; i++) { + const rs_member_t *m = &ix->members[i]; + if ((rs_member_ns(m->kind) & ns) && strchr(kinds, m->kind)) { + rs_set_add(out, rs_ent(RS_E_MEMBER, i)); + } + } +} + +/* ── Lookup: module tree questions ───────────────────────────────── */ + +/* The modules `super` names in module `m`: the inline parent, or every module + * that declares the file. An orphan file's parent is not known. */ +static void rs_parents(const rs_index_t *ix, int m, rs_set_t *out) { + const rs_mod_t *mod = &ix->mods[m]; + if (mod->parent >= 0) { + rs_set_add(out, rs_ent(RS_E_MOD, mod->parent)); + return; + } + if (mod->attach_count > RS_SET_CAP || + (mod->attach_count == 0 && !ix->files[mod->file].crate_root)) { + rs_set_add(out, rs_weak(RS_E_GAP)); + return; + } + for (int i = 0; i < mod->attach_count; i++) { + rs_set_add(out, rs_ent(RS_E_MOD, ix->decls[ix->attach[mod->attach_first + i].decl].mod)); + } +} + +/* Is module `m` the module `ancestor` or inside it? 1 yes, 0 no, -1 not + * known (an orphan on the way up, or too far to walk). */ +static int rs_mod_within(const rs_index_t *ix, int m, int ancestor) { + int stack[RS_SET_CAP]; + int sp = 0; + bool unknown = false; + stack[sp++] = m; + for (int steps = 0; sp > 0; steps++) { + int cur = stack[--sp]; + if (cur == ancestor) { + return SKIP_ONE; + } + const rs_mod_t *mod = &ix->mods[cur]; + if (steps >= RS_UP_STEPS || (mod->parent < 0 && mod->attach_count > RS_SET_CAP - sp)) { + return CBM_NOT_FOUND; + } + if (mod->parent >= 0) { + stack[sp++] = mod->parent; + continue; + } + unknown = unknown || (mod->attach_count == 0 && !ix->files[mod->file].crate_root); + for (int i = 0; i < mod->attach_count; i++) { + stack[sp++] = ix->decls[ix->attach[mod->attach_first + i].decl].mod; + } + } + return unknown ? CBM_NOT_FOUND : 0; +} + +/* Are two files part of one crate? 1 yes, 0 no, -1 not known. */ +static int rs_same_crate(const rs_index_t *ix, int fa, int fb) { + const rs_file_t *a = &ix->files[fa]; + const rs_file_t *b = &ix->files[fb]; + if (fa == fb) { + return SKIP_ONE; + } + if (a->roots_top || b->roots_top || a->nroots == 0 || b->nroots == 0) { + return CBM_NOT_FOUND; + } + for (int i = 0; i < a->nroots; i++) { + for (int j = 0; j < b->nroots; j++) { + if (a->roots[i] == b->roots[j]) { + return SKIP_ONE; + } + } + } + return 0; +} + +/* Can module `from` see a binding of visibility `vis` declared in module + * `owner`? 1 yes, 0 no, -1 not known. */ +static int rs_visible(const rs_index_t *ix, char vis, int owner, int from) { + if (vis == 'p' || owner == from) { + return SKIP_ONE; + } + if (vis == 'c') { + return rs_same_crate(ix, ix->mods[owner].file, ix->mods[from].file); + } + if (vis == '-') { + return rs_mod_within(ix, from, owner); + } + if (vis != 's') { + return CBM_NOT_FOUND; /* pub(in path): the path is not carried */ + } + rs_set_t up = {0}; + rs_parents(ix, owner, &up); + int best = up.n == 0 ? rs_mod_within(ix, from, owner) : 0; + for (int i = 0; i < up.n && best <= 0; i++) { + int v = up.v[i].kind == RS_E_MOD ? rs_mod_within(ix, from, up.v[i].idx) : CBM_NOT_FOUND; + best = (v > 0 || best == 0) ? v : best; + } + return best; +} + +/* ── Lookup: names a macro's body declares ───────────────────────── */ + +static int rs_macuse_cmp(const void *a, const void *b) { + const rs_macuse_t *x = (const rs_macuse_t *)a; + const rs_macuse_t *y = (const rs_macuse_t *)b; + if (x->mod != y->mod) { + return x->mod < y->mod ? -1 : 1; + } + return strcmp(x->macro, y->macro); +} + +static int rs_macdecl_cmp(const void *a, const void *b) { + const rs_macdecl_t *x = (const rs_macdecl_t *)a; + const rs_macdecl_t *y = (const rs_macdecl_t *)b; + int c = strcmp(x->name, y->name); + return c ? c : strcmp(x->macro, y->macro); +} + +/* True when module `m` invokes a macro named `macro` among its items. */ +static bool rs_mod_invokes(const rs_index_t *ix, int m, const char *macro) { + rs_macuse_t key = {.mod = m, .macro = macro}; + int lo = 0; + int hi = ix->nmacuses; + while (lo < hi) { + int mid = lo + ((hi - lo) / PAIR_LEN); + if (rs_macuse_cmp(&ix->macuses[mid], &key) < 0) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + return lo < ix->nmacuses && rs_macuse_cmp(&ix->macuses[lo], &key) == 0; +} + +/* May a macro invoked in module `m` declare `name` there? True when the body + * of some `macro_rules!` macro of the repository declares that name and the + * module invokes a macro of that macro's name. Macros are matched by name + * only (which macro an invocation means is not resolved), so this can say + * "maybe" too often, never too rarely. */ +static bool rs_macro_may_declare(const rs_index_t *ix, int m, const char *name) { + int lo = 0; + int hi = ix->nmacdecls; + while (lo < hi) { + int mid = lo + ((hi - lo) / PAIR_LEN); + if (strcmp(ix->macdecls[mid].name, name) < 0) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + for (int i = lo; i < ix->nmacdecls && rs_is(ix->macdecls[i].name, name); i++) { + if (i - lo >= RS_GLOB_VISIT || rs_mod_invokes(ix, m, ix->macdecls[i].macro)) { + return true; /* declared by more macros than are checked: maybe */ + } + } + return false; +} + +/* ── Lookup: level 1, a module's own bindings ────────────────────── */ + +typedef struct { + int skip; /* an import that is not consulted; CBM_NOT_FOUND */ + int from; /* a glob import of this module asks: only what it can see; + * CBM_NOT_FOUND: everything */ +} rs_own_opt_t; + +/* What binding `b` is in namespace `ns`; RS_E_NONE when it is nothing there. + * `glob`: asked through a glob import, which a textual macro does not pass. */ +static rs_ent_t rs_bind_ent(rs_look_t *lk, const rs_bind_t *b, int ns, bool glob) { + const rs_index_t *ix = lk->ix; + rs_ent_t none = {0}; + if (b->kind == RS_BIND_ITEM) { + const rs_item_t *it = &ix->items[b->ref]; + bool textual = it->kind == 'M' && !it->macro_export; + return ((rs_item_ns(it->kind) & ns) && !(glob && textual)) ? rs_ent(RS_E_ITEM, b->ref) + : none; + } + if (b->kind == RS_BIND_CHILD) { + if (ns != RS_NS_TYPE) { + return none; + } + return b->ref >= 0 ? rs_ent(RS_E_MOD, b->ref) : rs_ent(RS_E_GAP, 0); + } + const rs_import_t *imp = &ix->imports[b->ref]; + if (!rs_import_ready(lk, imp)) { + lk->blocked = true; + return none; + } + bool type_only = imp->kind != 'b'; + if (type_only && ns != RS_NS_TYPE) { + return none; + } + rs_ent_t e = imp->res[type_only ? 0 : rs_ns_index(ns)]; + e.via_import = e.kind != RS_E_NONE; + return e; +} + +/* Level 1: what module `m` itself binds as `name` in namespace `ns`. A name + * it does not show may still be declared there when a macro invocation could + * declare it or the file did not parse: that is a gap, not an absence. */ +static void rs_own(rs_look_t *lk, int m, const char *name, int ns, const rs_own_opt_t *opt, + rs_set_t *out) { + const rs_index_t *ix = lk->ix; + int first = 0; + int n = rs_bind_range(ix, m, name, &first); + bool maybe = false; + rs_set_t vague = {0}; /* imports of something whose namespace is not known */ + for (int i = first; i < first + n && !out->overflow; i++) { + const rs_bind_t *b = &ix->binds[i]; + if (b->kind == RS_BIND_MAYBE) { + maybe = true; + continue; + } + if (b->kind == RS_BIND_IMPORT && b->ref == opt->skip) { + continue; + } + /* visibility first: a binding the asking module cannot see is not + * looked at, so an import of its own that is not resolved yet does + * not hold the answer up */ + int v = opt->from >= 0 ? rs_visible(ix, b->vis, m, opt->from) : SKIP_ONE; + if (v == 0) { + continue; + } + rs_ent_t e = rs_bind_ent(lk, b, ns, opt->from >= 0); + if (e.kind == RS_E_NONE) { + continue; + } + bool spread = b->kind == RS_BIND_IMPORT && ix->imports[b->ref].spread; + rs_set_add(spread ? &vague : out, v > 0 ? e : rs_weak(RS_E_GAP)); + } + if (rs_set_empty(out)) { + /* nothing the module really has in this namespace: the import may be it */ + rs_set_merge(out, &vague); + } + if (rs_set_empty(out) && + (maybe || ix->files[ix->mods[m].file].broken || rs_macro_may_declare(ix, m, name))) { + rs_set_add(out, rs_weak(RS_E_GAP)); + } +} + +/* ── Lookup: level 2, glob imports ───────────────────────────────── */ + +static bool rs_int_in(const int *v, int n, int x) { + for (int i = 0; i < n; i++) { + if (v[i] == x) { + return true; + } + } + return false; +} + +/* What a walk through glob imports met besides bindings. */ +typedef struct { + bool open; /* a glob of something outside the repository: it may hold the name */ + bool gap; /* a glob that cannot be followed, or a binding of unknown visibility */ + bool pending; /* a glob that is not resolved yet (only while imports are resolved) */ +} rs_glob_flags_t; + +/* One glob import `imp` of module `cur`, asked for `name` on behalf of module + * `m`. Returns the module whose own globs are to be asked next (the target + * does not bind the name itself), or CBM_NOT_FOUND. */ +static int rs_glob_step(rs_look_t *lk, const rs_import_t *imp, int cur, const rs_own_opt_t *opt, + const char *name, int ns, rs_set_t *out, rs_glob_flags_t *fl) { + const rs_index_t *ix = lk->ix; + int m = opt->from; + int v = cur == m ? SKIP_ONE : rs_visible(ix, imp->vis, cur, m); + if (v <= 0) { + fl->gap = fl->gap || v < 0; + return CBM_NOT_FOUND; + } + if (!rs_import_ready(lk, imp)) { + fl->pending = true; + return CBM_NOT_FOUND; + } + rs_ent_t t = imp->res[0]; + if (t.kind == RS_E_MOD) { + rs_set_t found = {0}; + rs_own(lk, t.idx, name, ns, opt, &found); + for (int i = 0; i < found.n; i++) { + if (found.v[i].weak) { + fl->gap = true; + } else { + rs_set_add(out, found.v[i]); + } + } + out->overflow = out->overflow || found.overflow; + return rs_set_empty(&found) ? t.idx : CBM_NOT_FOUND; + } + if (t.kind == RS_E_ITEM && ix->items[t.idx].kind == 'E') { + const rs_item_t *en = &ix->items[t.idx]; + rs_owner_members(ix, en->member_first, en->member_count, name, ns, "v", out); + return CBM_NOT_FOUND; + } + fl->open = fl->open || t.kind == RS_E_EXTERNAL; + fl->gap = fl->gap || t.kind != RS_E_EXTERNAL; + return CBM_NOT_FOUND; +} + +/* Level 2: what the glob imports of module `m` bring in as `name`, through + * re-exporting globs. `skip`: the import being resolved, which is neither + * asked as a glob nor read as a binding (it cannot supply its own name). + * Anything that cannot be followed is a gap. */ +static void rs_globs(rs_look_t *lk, int m, const char *name, int ns, int skip, rs_set_t *out, + rs_glob_flags_t *fl) { + const rs_index_t *ix = lk->ix; + rs_own_opt_t opt = {.skip = skip, .from = m}; + int seen[RS_GLOB_VISIT]; /* also the queue: seen[next, nseen) is still to be asked */ + int nseen = 0; + int next = 0; + int steps = 0; + seen[nseen++] = m; + while (next < nseen && !out->overflow) { + int cur = seen[next++]; + const rs_mod_t *cm = &ix->mods[cur]; + for (int g = 0; g < cm->glob_count; g++) { + int gi = ix->globs[cm->glob_first + g].ref; + if (gi == skip) { + continue; + } + if (++steps > RS_UP_STEPS) { + fl->gap = true; + break; + } + int into = rs_glob_step(lk, &ix->imports[gi], cur, &opt, name, ns, out, fl); + if (into < 0 || rs_int_in(seen, nseen, into)) { + continue; + } + if (nseen >= RS_GLOB_VISIT) { + fl->gap = true; + continue; + } + seen[nseen++] = into; + } + } + if (rs_set_empty(out) && fl->gap) { + rs_set_add(out, rs_weak(RS_E_GAP)); + } +} + +/* Levels 1 and 2 of module `m`: what `m::name` names. A glob that is not + * resolved yet may still bring the name in: the answer has to wait. */ +static void rs_mod_member(rs_look_t *lk, int m, const char *name, int ns, int skip, rs_set_t *out) { + rs_own_opt_t opt = {.skip = skip, .from = CBM_NOT_FOUND}; + rs_glob_flags_t fl = {0}; + rs_own(lk, m, name, ns, &opt, out); + if (!rs_set_empty(out)) { + return; + } + rs_globs(lk, m, name, ns, skip, out, &fl); + bool real = out->n > 0 && !out->v[0].weak; + if (fl.pending && !real) { + memset(out, 0, sizeof(*out)); + lk->blocked = true; + return; + } + if (rs_set_empty(out) && fl.open) { + rs_set_add(out, rs_weak(RS_E_EXTERNAL)); + } +} + +/* ── Lookup: levels 3 and 4, the preludes ────────────────────────── */ + +/* Defined by the language and its standard library, not by any corpus. */ +static const char *const RS_BUILTIN_CRATES[] = {"std", "core", "alloc", "proc_macro", "test"}; + +/* The std prelude, type namespace (all editions; a name of a later edition's + * prelude is still the standard library's). */ +static const char *const RS_PRELUDE_TYPES[] = {"Option", + "Result", + "Vec", + "String", + "Box", + "Some", + "None", + "Ok", + "Err", + "Send", + "Sized", + "Sync", + "Unpin", + "Copy", + "Drop", + "Fn", + "FnMut", + "FnOnce", + "AsyncFn", + "AsyncFnMut", + "AsyncFnOnce", + "AsMut", + "AsRef", + "From", + "Into", + "Clone", + "Default", + "Eq", + "Ord", + "PartialEq", + "PartialOrd", + "Extend", + "IntoIterator", + "Iterator", + "DoubleEndedIterator", + "ExactSizeIterator", + "ToOwned", + "ToString", + "TryFrom", + "TryInto", + "FromIterator", + "Future", + "IntoFuture"}; + +static const char *const RS_PRELUDE_VALUES[] = { + "Some", "None", "Ok", "Err", "drop", "size_of", "size_of_val", "align_of", "align_of_val"}; + +/* std's macros, and the derive and attribute macros of the prelude. */ +static const char *const RS_STD_MACROS[] = {"assert", + "assert_eq", + "assert_ne", + "cfg", + "column", + "compile_error", + "concat", + "dbg", + "debug_assert", + "debug_assert_eq", + "debug_assert_ne", + "env", + "eprint", + "eprintln", + "file", + "format", + "format_args", + "include", + "include_bytes", + "include_str", + "is_x86_feature_detected", + "line", + "matches", + "module_path", + "option_env", + "panic", + "print", + "println", + "stringify", + "thread_local", + "todo", + "try", + "unimplemented", + "unreachable", + "vec", + "write", + "writeln", + "Clone", + "Copy", + "Debug", + "Default", + "Eq", + "Hash", + "Ord", + "PartialEq", + "PartialOrd", + "derive", + "test", + "bench", + "global_allocator"}; + +/* rustdoc's names for primitive types (resolve_primitive). */ +static const char *const RS_PRIMITIVES[] = { + "isize", "i8", "i16", "i32", "i64", "i128", "usize", "u8", "u16", + "u32", "u64", "u128", "f16", "f32", "f64", "f128", "char", "bool", + "true", "false", "str", "&str", "slice", "array", "tuple", "unit", "pointer", + "*const", "*mut", "reference", "&", "&mut", "fn", "never", "!"}; + +static bool rs_std_name(const char *name, int ns) { + if (ns == RS_NS_TYPE) { + return RS_IN(RS_PRELUDE_TYPES, name); + } + return ns == RS_NS_VALUE ? RS_IN(RS_PRELUDE_VALUES, name) : RS_IN(RS_STD_MACROS, name); +} + +/* Level 3: the extern prelude of the crate the path is written in. `*unknown`: + * that crate's dependencies are not known, so a miss says nothing. */ +static void rs_extern(const rs_at_t *at, const char *name, rs_set_t *out, bool *unknown) { + const rs_index_t *ix = at->lk->ix; + if (RS_IN(RS_BUILTIN_CRATES, name)) { + rs_set_add(out, rs_ent(RS_E_EXTERNAL, 0)); + return; + } + if (at->pkg < 0) { + *unknown = true; + return; + } + const rs_pkg_t *p = &ix->pkgs[at->pkg]; + if (p->lib_root >= 0 && p->lib_name && rs_is(p->lib_name, name)) { + /* the package's own lib: in scope in its other targets only, and which + * target a file without a crate root belongs to is not known */ + if (at->own_lib) { + rs_set_add(out, rs_ent(RS_E_MOD, p->lib_root)); + } + *unknown = !at->own_lib && at->root < 0; + if (at->own_lib || *unknown) { + return; + } + } + int lo = 0; + int hi = p->ndeps; + while (lo < hi) { + int mid = lo + ((hi - lo) / PAIR_LEN); + if (strcmp(p->deps[mid].key, name) < 0) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + if (lo < p->ndeps && rs_is(p->deps[lo].key, name)) { + rs_set_add(out, p->deps[lo].target); + return; + } + *unknown = p->broken; +} + +/* A `macro_rules!` macro is in textual scope in the modules below the one + * that defines it. */ +static void rs_textual_macro(const rs_index_t *ix, int m, const char *name, rs_set_t *out) { + int cur = m; + for (int step = 0; step < RS_UP_STEPS; step++) { + const rs_mod_t *mod = &ix->mods[cur]; + int up = mod->parent; + if (up < 0 && mod->attach_count == SKIP_ONE) { + up = ix->decls[ix->attach[mod->attach_first].decl].mod; + } + if (up < 0) { + return; + } + int first = 0; + int n = rs_bind_range(ix, up, name, &first); + for (int i = first; i < first + n; i++) { + const rs_bind_t *b = &ix->binds[i]; + if (b->kind == RS_BIND_ITEM && ix->items[b->ref].kind == 'M') { + rs_set_add(out, rs_ent(RS_E_ITEM, b->ref)); + return; + } + } + cur = up; + } + rs_set_add(out, rs_weak(RS_E_GAP)); /* deeper than is walked: not known */ +} + +/* The lexical lookup of a name: levels 1 to 4, the first that binds it in the + * namespace. + * + * Level 2 can hold a name without showing it: a glob of something outside the + * repository, a glob that cannot be followed, and (while imports are being + * resolved) a glob that is not resolved yet. What then follows depends on + * whose path it is: + * - an import path may look on. rustc rejects an import path whose first + * name is brought in by a glob AND bound further out (E0659), so in + * source that compiles a name found at level 3 or 4 is not in the glob; + * - any other path (a doc link, a type) may not: there a glob's name + * silently wins over the preludes. The lookup ends with a weak answer. + * A name found nowhere waits for an unresolved glob, which may still bring + * it in. */ +static void rs_lexical(const rs_at_t *at, const char *name, int ns, rs_set_t *out) { + rs_look_t *lk = at->lk; + rs_own_opt_t opt = {.skip = at->skip, .from = CBM_NOT_FOUND}; + rs_glob_flags_t fl = {0}; + rs_set_t outer = {0}; + bool unknown = false; + rs_own(lk, at->mod, name, ns, &opt, out); + if (rs_set_empty(out) && ns == RS_NS_MACRO) { + rs_textual_macro(lk->ix, at->mod, name, out); + } + if (!rs_set_empty(out)) { + if (out->v[0].weak && at->skip >= 0 && ns == RS_NS_TYPE) { + /* an import path in a module that may declare more than it shows: + * a crate of the extern prelude stands all the same, because an + * import path whose first name is both a local name and a crate + * is an error (E0659) */ + rs_extern(at, name, &outer, &unknown); + if (!rs_set_empty(&outer)) { + *out = outer; + } + } + return; + } + rs_globs(lk, at->mod, name, ns, at->skip, out, &fl); + if (out->n > 0 && !out->v[0].weak) { + return; /* brought in by a glob that is resolved */ + } + bool hidden = fl.open || fl.pending || !rs_set_empty(out); + if (!hidden || at->skip >= 0) { + if (ns == RS_NS_TYPE) { + rs_extern(at, name, &outer, &unknown); + } + if (rs_set_empty(&outer) && rs_std_name(name, ns)) { + rs_set_add(&outer, rs_ent(RS_E_EXTERNAL, 0)); + } + if (!rs_set_empty(&outer)) { + *out = outer; + return; + } + } + if (fl.pending) { + memset(out, 0, sizeof(*out)); + lk->blocked = true; + return; + } + if (rs_set_empty(out) && (fl.open || unknown)) { + rs_set_add(out, rs_weak(fl.open ? RS_E_EXTERNAL : RS_E_GAP)); + } + if (rs_set_empty(out) && at->skip >= 0 && ns == RS_NS_TYPE && at->pkg >= 0) { + /* the first name of an import path, bound nowhere: the module shows + * everything it declares (else the answer above was a gap), so what + * is left is a crate of the extern prelude under a name the manifest + * does not give (a lib target named unlike its package) */ + rs_set_add(out, rs_weak(RS_E_EXTERNAL)); + } +} + +/* ── Lookup: associated items ────────────────────────────────────── */ + +/* Declarations of one entity under different cfg: two items of one module, + * name and kind, or two members of one owner. */ +static bool rs_cfg_twins(const rs_index_t *ix, rs_ent_t a, rs_ent_t b) { + if (a.kind != b.kind) { + return false; + } + if (a.kind == RS_E_ITEM) { + const rs_item_t *x = &ix->items[a.idx]; + const rs_item_t *y = &ix->items[b.idx]; + return x->mod == y->mod && x->kind == y->kind && rs_is(x->name, y->name); + } + if (a.kind == RS_E_MEMBER) { + const rs_member_t *x = &ix->members[a.idx]; + const rs_member_t *y = &ix->members[b.idx]; + return x->owner == y->owner && x->kind == y->kind && rs_is(x->name, y->name); + } + return false; +} + +static bool rs_has_node(const rs_index_t *ix, rs_ent_t e) { + if (e.kind == RS_E_ITEM) { + return ix->items[e.idx].label != 0; + } + return e.kind == RS_E_MEMBER && ix->members[e.idx].label != 0; +} + +/* The graph keeps one node per qualified name, so of two cfg twins the one + * that owns the node stands for both. Two with a node each stay two. */ +static void rs_collapse(const rs_index_t *ix, rs_set_t *s) { + for (int i = 0; i < s->n; i++) { + for (int j = i + SKIP_ONE; j < s->n;) { + bool ni = rs_has_node(ix, s->v[i]); + bool nj = rs_has_node(ix, s->v[j]); + if (!rs_cfg_twins(ix, s->v[i], s->v[j]) || (ni && nj)) { + j++; + continue; + } + if (nj) { + bool via = s->v[i].via_import || s->v[j].via_import; + s->v[i] = s->v[j]; + s->v[i].via_import = via; + } + s->v[j] = s->v[--s->n]; + } + } +} + +/* The item a chain of type aliases ends at; an external or gap entity when it + * leaves what is known. */ +static rs_ent_t rs_alias_end(const rs_index_t *ix, int item) { + rs_ent_t e = rs_ent(RS_E_ITEM, item); + for (int hop = 0; hop < RS_ALIAS_HOPS; hop++) { + if (ix->items[e.idx].kind != 'A') { + return e; + } + e = ix->items[e.idx].alias_ent; + if (e.kind != RS_E_ITEM) { + return rs_weak(e.kind == RS_E_EXTERNAL ? RS_E_EXTERNAL : RS_E_GAP); + } + } + return rs_weak(RS_E_GAP); +} + +static int rs_assoc_bound(const rs_index_t *ix, int item, const char *name, bool upper) { + int lo = 0; + int hi = ix->nassoc; + while (lo < hi) { + int mid = lo + ((hi - lo) / PAIR_LEN); + const rs_assoc_t *a = &ix->assoc[mid]; + int c = a->item != item ? (a->item < item ? -1 : 1) : strcmp(a->name, name); + if (c < 0 || (upper && c == 0)) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + return lo; +} + +static const char *rs_last_segment(const char *path) { + const char *last = path; + for (const char *p = path; *p; p++) { + if (p[0] == ':' && p[SKIP_ONE] == ':') { + last = p + PAIR_LEN; + } + } + return rs_plain(last); +} + +/* Is the trait of impl `im` in scope in the module the path is written in? + * rustdoc offers a trait impl's items only then. 1 yes, 0 no, -1 not known. + * A trait outside the repository is known by its name only. */ +static int rs_trait_in_scope(const rs_at_t *at, const rs_impl_t *im) { + const rs_index_t *ix = at->lk->ix; + rs_ent_t t = im->trait_ent; + if (t.kind != RS_E_ITEM && t.kind != RS_E_EXTERNAL) { + return CBM_NOT_FOUND; + } + const char *name = + t.kind == RS_E_ITEM ? ix->items[t.idx].name : rs_last_segment(im->trait_path); + rs_set_t found = {0}; + rs_lexical(at, name, RS_NS_TYPE, &found); + if (rs_set_has(&found, t)) { + return SKIP_ONE; + } + const rs_mod_t *mod = &ix->mods[at->mod]; + for (int i = 0; i < mod->named_count; i++) { /* imported under another name */ + const rs_import_t *imp = &ix->imports[ix->named[mod->named_first + i].ref]; + rs_ent_t e = imp->res[0]; + if (imp->kind != 'b' || e.kind != t.kind || imp->nsegs == 0) { + continue; + } + if (t.kind == RS_E_ITEM ? e.idx == t.idx : rs_is(imp->segs[imp->nsegs - SKIP_ONE], name)) { + return SKIP_ONE; + } + } + return rs_set_has(&found, rs_ent(RS_E_GAP, 0)) ? CBM_NOT_FOUND : 0; +} + +/* Items a trait declares and an impl of it for type `item` does not define + * itself. Returns true when some could not be decided. */ +static bool rs_trait_defaults(const rs_at_t *at, int item, const char *name, int ns, + rs_set_t *out) { + const rs_index_t *ix = at->lk->ix; + const rs_item_t *it = &ix->items[item]; + bool unsure = it->impl_count > RS_IMPL_SCAN; + for (int i = 0; i < it->impl_count && i < RS_IMPL_SCAN && !out->overflow; i++) { + const rs_impl_t *im = &ix->impls[ix->item_impls[it->impl_first + i].impl]; + if (!im->trait_path || im->trait_ent.kind != RS_E_ITEM) { + continue; + } + const rs_item_t *tr = &ix->items[im->trait_ent.idx]; + rs_set_t own = {0}; + rs_set_t dflt = {0}; + rs_owner_members(ix, im->member_first, im->member_count, name, ns, "fct", &own); + if (!rs_set_empty(&own) || tr->kind != 'R') { + continue; + } + rs_owner_members(ix, tr->member_first, tr->member_count, name, ns, "fct", &dflt); + if (rs_set_empty(&dflt)) { + continue; + } + int v = rs_trait_in_scope(at, im); + if (v > 0) { + rs_set_merge(out, &dflt); + } + unsure = unsure || v < 0; + } + return unsure; +} + +/* Two impls of one trait: the scope question has one answer for both. */ +static bool rs_same_trait(const rs_impl_t *a, const rs_impl_t *b) { + if (a->trait_ent.kind != b->trait_ent.kind) { + return false; + } + if (a->trait_ent.kind == RS_E_ITEM) { + return a->trait_ent.idx == b->trait_ent.idx; + } + return a->trait_ent.kind == RS_E_EXTERNAL && + rs_is(rs_last_segment(a->trait_path), rs_last_segment(b->trait_path)); +} + +/* The items of the impls of type `item` named `name`: inherent impls first, + * then the trait impls whose trait is in scope where the path is written. */ +static void rs_impl_items(const rs_at_t *at, int item, const char *name, int ns, rs_set_t *out) { + const rs_index_t *ix = at->lk->ix; + int first = rs_assoc_bound(ix, item, name, false); + int end = rs_assoc_bound(ix, item, name, true); + bool unsure = false; + const rs_impl_t *asked = NULL; /* the impl whose trait was asked about last */ + int answer = 0; + for (int i = first; i < end && !out->overflow; i++) { + const rs_assoc_t *a = &ix->assoc[i]; + if (!ix->impls[a->impl].trait_path && (rs_member_ns(ix->members[a->member].kind) & ns)) { + rs_set_add(out, rs_ent(RS_E_MEMBER, a->member)); + } + } + bool inherent = !rs_set_empty(out); + for (int i = first; i < end && !inherent && !out->overflow; i++) { + const rs_assoc_t *a = &ix->assoc[i]; + const rs_impl_t *im = &ix->impls[a->impl]; + if (!im->trait_path || !(rs_member_ns(ix->members[a->member].kind) & ns)) { + continue; + } + if (!asked || !rs_same_trait(asked, im)) { + asked = im; + answer = rs_trait_in_scope(at, im); + } + if (answer > 0) { + rs_set_add(out, rs_ent(RS_E_MEMBER, a->member)); + } + unsure = unsure || answer < 0; + } + if (rs_set_empty(out)) { + unsure = rs_trait_defaults(at, item, name, ns, out) || unsure; + } + if (rs_set_empty(out) && unsure) { + rs_set_add(out, rs_weak(RS_E_GAP)); + } +} + +/* `Type::name`: a variant, an item of an impl, a field (rustdoc's order). */ +static void rs_type_assoc(const rs_at_t *at, int item, const char *name, int ns, rs_set_t *out) { + const rs_index_t *ix = at->lk->ix; + rs_ent_t end = rs_alias_end(ix, item); + if (end.kind != RS_E_ITEM) { + rs_set_add(out, end); + return; + } + const rs_item_t *it = &ix->items[end.idx]; + if (it->kind == 'R') { + rs_owner_members(ix, it->member_first, it->member_count, name, ns, "fct", out); + return; + } + if (it->kind != 'S' && it->kind != 'E' && it->kind != 'U') { + return; + } + if (it->kind == 'E') { + rs_owner_members(ix, it->member_first, it->member_count, name, ns, "v", out); + } + if (rs_set_empty(out)) { + rs_impl_items(at, end.idx, name, ns, out); + } + if (rs_set_empty(out) && ns == RS_NS_VALUE) { + rs_owner_members(ix, it->member_first, it->member_count, name, ns, "d", out); + } +} + +/* What `name` is inside item `item`. While the index is being built only an + * enum's variants are known; anything else through a type is not followed. */ +static void rs_item_member(const rs_at_t *at, int item, const char *name, int ns, rs_set_t *out) { + const rs_index_t *ix = at->lk->ix; + const rs_item_t *it = &ix->items[item]; + if (!rs_item_is_type(it->kind)) { + return; /* a function, a constant, a macro: nothing is inside */ + } + if (at->lk->assoc) { + rs_type_assoc(at, item, name, ns, out); + return; + } + if (it->kind == 'E') { + rs_owner_members(ix, it->member_first, it->member_count, name, ns, "v", out); + } + if (rs_set_empty(out)) { + rs_set_add(out, rs_weak(RS_E_GAP)); + } +} + +/* What `name` is inside `owner`, in namespace `ns`. */ +static void rs_member(const rs_at_t *at, rs_ent_t owner, const char *name, int ns, rs_set_t *out) { + const rs_index_t *ix = at->lk->ix; + switch (owner.kind) { + case RS_E_MOD: + rs_mod_member(at->lk, owner.idx, name, ns, at->skip, out); + break; + case RS_E_ITEM: + rs_item_member(at, owner.idx, name, ns, out); + break; + case RS_E_IMPL: { + const rs_impl_t *im = &ix->impls[owner.idx]; + rs_owner_members(ix, im->member_first, im->member_count, name, ns, "fct", out); + if (rs_set_empty(out)) { + rs_set_add(out, rs_weak(im->self_ent.kind == RS_E_EXTERNAL ? RS_E_EXTERNAL : RS_E_GAP)); + } + break; + } + case RS_E_MEMBER: + if (ix->members[owner.idx].kind == 'v') { + rs_set_add(out, rs_weak(RS_E_GAP)); /* a variant's fields are not carried */ + } + break; + case RS_E_EXTERNAL: + case RS_E_GAP: + rs_set_add(out, rs_weak(owner.kind)); + break; + default: + break; + } +} + +/* ── Lookup: a whole path ────────────────────────────────────────── */ + +/* The modules or the type that the keywords a path starts with name: `crate`, + * `self`, `super` (repeatable), `Self`, or a leading `::`. Returns how many + * segments they are; 0 when the path starts with a plain name. */ +static int rs_path_head(const rs_at_t *at, const rs_path_t *p, rs_ent_t self, rs_set_t *cur) { + const rs_index_t *ix = at->lk->ix; + const char *first = p->segs[0]; + int i = SKIP_ONE; + if (p->global) { + bool unknown = false; + rs_extern(at, first, cur, &unknown); + if (rs_set_empty(cur) && (unknown || at->edition < RS_EDITION_2018)) { + rs_set_add(cur, rs_weak(RS_E_GAP)); /* before 2018 `::name` is crate-relative */ + } + } else if (rs_is(first, "crate")) { + rs_set_add(cur, at->root >= 0 ? rs_ent(RS_E_MOD, at->root) : rs_weak(RS_E_GAP)); + } else if (rs_is(first, "self")) { + rs_set_add(cur, rs_ent(RS_E_MOD, at->mod)); + } else if (rs_is(first, "super")) { + rs_parents(ix, at->mod, cur); + } else if (rs_is(first, "Self")) { + rs_set_add(cur, self); + } else { + return 0; + } + for (; i < p->n && rs_is(p->segs[i], "super"); i++) { + rs_set_t up = {0}; + for (int k = 0; k < cur->n; k++) { + if (cur->v[k].kind == RS_E_MOD) { + rs_parents(ix, cur->v[k].idx, &up); + } else { + rs_set_add(&up, rs_weak(RS_E_GAP)); + } + } + up.overflow = up.overflow || cur->overflow; + *cur = up; + } + return i; +} + +/* A name looked up from the crate root: an edition-2015 `use` path. */ +static void rs_crate_relative(const rs_at_t *at, const char *name, int ns, rs_set_t *out) { + bool unknown = false; + if (ns == RS_NS_TYPE && RS_IN(RS_BUILTIN_CRATES, name)) { + rs_set_add(out, rs_ent(RS_E_EXTERNAL, 0)); + return; + } + if (at->root < 0) { + rs_set_add(out, rs_weak(RS_E_GAP)); + return; + } + rs_mod_member(at->lk, at->root, name, ns, at->skip, out); + if (rs_set_empty(out) && ns == RS_NS_TYPE) { + rs_extern(at, name, out, &unknown); + } + if (rs_set_empty(out) && unknown) { + rs_set_add(out, rs_weak(RS_E_GAP)); + } +} + +/* The first segment of a path that starts with a plain name. A doc link, a + * type path and an edition-2018 import look it up lexically; an edition-2015 + * import starts at the crate root. With the edition unknown both must agree. */ +static void rs_first(const rs_at_t *at, const char *name, int ns, bool import, rs_set_t *out) { + if (!import || at->edition >= RS_EDITION_2018) { + rs_lexical(at, name, ns, out); + return; + } + rs_set_t rooted = {0}; + rs_crate_relative(at, name, ns, &rooted); + if (at->edition == RS_EDITION_2015) { + *out = rooted; + return; + } + rs_lexical(at, name, ns, out); + if (!rs_set_equal(out, &rooted)) { + memset(out, 0, sizeof(*out)); + rs_set_add(out, rs_weak(RS_E_GAP)); + } +} + +/* One segment before the last: what `name` is, as a type or module, inside + * each thing of `cur`. false when one of them holds several. */ +static bool rs_walk_step(const rs_at_t *at, const rs_set_t *cur, const char *name, rs_set_t *next) { + for (int k = 0; k < cur->n; k++) { + rs_set_t one = {0}; + rs_member(at, cur->v[k], name, RS_NS_TYPE, &one); + rs_collapse(at->lk->ix, &one); + if (one.n > SKIP_ONE || one.overflow) { + return false; + } + rs_set_merge(next, &one); + } + return true; +} + +/* Resolve path `p` written at `at`: the candidates of its last segment in + * every namespace of `mask`. */ +static void rs_walk(const rs_at_t *at, const rs_path_t *p, int mask, rs_ent_t self, bool import, + rs_found_t *out) { + rs_set_t cur = {0}; + int i = rs_path_head(at, p, self, &cur); + if (i == 0 && p->n == SKIP_ONE) { + for (int k = 0; k < RS_NS_COUNT; k++) { + if (mask & RS_NS_LIST[k]) { + rs_first(at, p->segs[0], RS_NS_LIST[k], import, &out->ns[k]); + } + } + return; + } + if (i == 0) { + rs_first(at, p->segs[0], RS_NS_TYPE, import, &cur); + rs_collapse(at->lk->ix, &cur); + if (cur.n > SKIP_ONE) { + out->ambiguous = true; + return; + } + if (rs_set_empty(&cur) && RS_IN(RS_PRIMITIVES, p->segs[0])) { + rs_set_add(&cur, rs_ent(RS_E_EXTERNAL, 0)); /* `str::len`: of a primitive type */ + } + i = SKIP_ONE; + } + if (cur.overflow) { + out->ambiguous = true; + return; + } + if (i == p->n) { /* the keywords are the whole path */ + if (mask & RS_NS_TYPE) { + rs_set_merge(&out->ns[0], &cur); + } + return; + } + for (; i + SKIP_ONE < p->n && !rs_set_empty(&cur); i++) { + rs_set_t next = {0}; + if (!rs_walk_step(at, &cur, p->segs[i], &next)) { + out->ambiguous = true; + return; + } + cur = next; + } + for (int k = 0; k < RS_NS_COUNT; k++) { + for (int e = 0; (mask & RS_NS_LIST[k]) && e < cur.n; e++) { + rs_member(at, cur.v[e], p->segs[p->n - SKIP_ONE], RS_NS_LIST[k], &out->ns[k]); + } + } + out->owners = cur; +} + +/* The contexts a path written in module `mod` is resolved under: one per + * crate root of its file, or a single one without a root when the file has + * none it can name (an orphan, or a file of more crates than are kept). */ +static int rs_contexts(rs_look_t *lk, int mod, int skip, rs_at_t out[RS_ROOTS_KEPT]) { + const rs_index_t *ix = lk->ix; + const rs_file_t *f = &ix->files[ix->mods[mod].file]; + rs_at_t base = { + .lk = lk, .mod = mod, .root = CBM_NOT_FOUND, .pkg = CBM_NOT_FOUND, .skip = skip}; + if (f->nroots == 0 || f->roots_top) { + base.pkg = f->pkg >= 0 ? f->pkg : CBM_NOT_FOUND; + base.own_lib = base.pkg >= 0 && f->roots_top && !f->in_lib; + base.edition = rs_edition(ix, base.pkg); + out[0] = base; + return SKIP_ONE; + } + for (int i = 0; i < f->nroots; i++) { + int pkg = ix->files[ix->mods[f->roots[i]].file].root_pkg; + out[i] = base; + out[i].root = f->roots[i]; + out[i].pkg = pkg >= 0 ? pkg : CBM_NOT_FOUND; + out[i].own_lib = pkg >= 0 && ix->pkgs[pkg].lib_root != f->roots[i]; + out[i].edition = rs_edition(ix, out[i].pkg); + } + return f->nroots; +} + +/* Resolve `p` written in module `mod`, under every crate root of its file: + * what the roots find is united, so roots that disagree make it ambiguous. + * `import`: the import whose path it is, CBM_NOT_FOUND for any other path. */ +static void rs_resolve_path(rs_look_t *lk, int mod, const rs_path_t *p, int mask, rs_ent_t self, + int import, rs_found_t *out) { + rs_at_t at[RS_ROOTS_KEPT]; + int n = rs_contexts(lk, mod, import, at); + memset(out, 0, sizeof(*out)); + for (int i = 0; i < n; i++) { + rs_found_t one; + memset(&one, 0, sizeof(one)); + rs_walk(&at[i], p, mask, self, import >= 0, &one); + for (int k = 0; k < RS_NS_COUNT; k++) { + rs_set_merge(&out->ns[k], &one.ns[k]); + } + rs_set_merge(&out->owners, &one.owners); + out->ambiguous = out->ambiguous || one.ambiguous; + } +} + +/* The one thing among the candidates of `s` that were really found: RS_E_NONE + * when there is none, a gap when there are several. */ +static rs_ent_t rs_only_strong(const rs_index_t *ix, const rs_set_t *s) { + rs_set_t strong = {0}; + for (int i = 0; i < s->n; i++) { + if (!s->v[i].weak) { + rs_set_add(&strong, s->v[i]); + } + } + rs_collapse(ix, &strong); + if (s->overflow || strong.n > SKIP_ONE) { + return rs_ent(RS_E_GAP, 0); + } + return strong.n ? strong.v[0] : (rs_ent_t){0}; +} + +/* What the weak candidates of `s` come to: a gap before external. */ +static uint8_t rs_only_weak(const rs_set_t *s) { + uint8_t kind = RS_E_NONE; + for (int i = 0; i < s->n; i++) { + if (s->v[i].weak && kind != RS_E_GAP) { + kind = s->v[i].kind; + } + } + return kind; +} + +/* ── Build: dependencies ─────────────────────────────────────────── */ + +static int rs_rawdep_cmp(const void *a, const void *b) { + const rs_rawdep_t *x = (const rs_rawdep_t *)a; + const rs_rawdep_t *y = (const rs_rawdep_t *)b; + int c = strcmp(x->name, y->name); + return c ? c : (x->seq > y->seq) - (x->seq < y->seq); +} + +static int rs_dep_cmp(const void *a, const void *b) { + return strcmp(((const rs_dep_t *)a)->key, ((const rs_dep_t *)b)->key); +} + +/* What the dependency tables of a manifest say about one dependency. */ +typedef struct { + const char *name; + const char *path; /* NULL: none written */ + const char *package; /* set: `name` is a rename */ + bool workspace; /* inherited from the workspace table */ + bool conflict; /* two tables give it different paths */ +} rs_depspec_t; + +static void rs_depspec_add(rs_depspec_t *spec, const rs_rawdep_t *r) { + if (rs_is(r->attr, "path")) { + spec->conflict = spec->conflict || (spec->path && !rs_is(spec->path, r->value)); + spec->path = r->value; + } else if (rs_is(r->attr, "package")) { + spec->package = r->value; + } else if (rs_is(r->attr, "workspace") && rs_is(r->value, "true")) { + spec->workspace = true; + } +} + +/* The entry `name` of the [workspace.dependencies] table of `ws` (its raw + * records are sorted by name). false when there is none. */ +static bool rs_ws_dep(const rs_pkg_t *ws, const char *name, rs_depspec_t *spec) { + int lo = 0; + int hi = ws->nraw; + bool found = false; + while (lo < hi) { + int mid = lo + ((hi - lo) / PAIR_LEN); + if (strcmp(ws->raw[mid].name, name) < 0) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + for (int i = lo; i < ws->nraw && rs_is(ws->raw[i].name, name); i++) { + if (ws->raw[i].kind == 'w') { + found = true; + rs_depspec_add(spec, &ws->raw[i]); + } + } + return found; +} + +/* The crate a dependency brings into the extern prelude. Only a `path` + * dependency names a crate of the repository; one whose path, manifest or + * lib target is not known is a gap, never matched by name. */ +static rs_dep_t rs_dep_make(rs_index_t *ix, const rs_pkg_t *p, rs_depspec_t spec) { + rs_dep_t dep = {.key = rs_crate_name(ix, spec.name), .target = rs_ent(RS_E_GAP, 0)}; + const char *base = p->dir; + char dir[RS_PATH_BUF]; + if (spec.workspace) { + const rs_pkg_t *ws = p->ws >= 0 ? &ix->pkgs[p->ws] : NULL; + rs_depspec_t inherited = {.name = spec.name}; + if (!ws || !rs_ws_dep(ws, spec.name, &inherited)) { + return dep; + } + base = ws->dir; + spec = inherited; + } + if (spec.conflict) { + return dep; + } + if (!spec.path) { + dep.target = rs_ent(RS_E_EXTERNAL, 0); + return dep; + } + if (rs_is(spec.path, "?") || !rs_join_path(base, spec.path, dir, sizeof(dir))) { + return dep; + } + int q = rs_pkg_at(ix, dir); + if (q < 0 || !ix->pkgs[q].has_package || ix->pkgs[q].lib_root < 0) { + return dep; + } + dep.target = rs_ent(RS_E_MOD, ix->pkgs[q].lib_root); + if (!spec.package && ix->pkgs[q].lib_name) { + dep.key = ix->pkgs[q].lib_name; /* not renamed: the lib crate's own name */ + } + return dep; +} + +static void rs_pkg_deps(rs_index_t *ix, int pi) { + rs_pkg_t *p = &ix->pkgs[pi]; + p->deps = (rs_dep_t *)ix_alloc(ix, (size_t)p->nraw * sizeof(rs_dep_t)); + if (!p->deps) { + return; + } + for (int i = 0; i < p->nraw;) { + rs_depspec_t spec = {.name = p->raw[i].name}; + bool own = false; + int j = i; + for (; j < p->nraw && rs_is(p->raw[j].name, spec.name); j++) { + if (p->raw[j].kind != 'w') { + own = true; + rs_depspec_add(&spec, &p->raw[j]); + } + } + if (own) { + p->deps[p->ndeps++] = rs_dep_make(ix, p, spec); + } + i = j; + } + if (p->ndeps > SKIP_ONE) { + qsort(p->deps, (size_t)p->ndeps, sizeof(rs_dep_t), rs_dep_cmp); + } + int w = 0; /* one crate name for two targets: which one is not known */ + for (int i = 0; i < p->ndeps; i++) { + if (w > 0 && rs_is(p->deps[w - SKIP_ONE].key, p->deps[i].key)) { + if (!rs_ent_same(p->deps[w - SKIP_ONE].target, p->deps[i].target)) { + p->deps[w - SKIP_ONE].target = rs_ent(RS_E_GAP, 0); + } + continue; + } + p->deps[w++] = p->deps[i]; + } + p->ndeps = w; +} + +static void rs_deps(rs_index_t *ix) { + for (int pi = 0; pi < ix->npkgs; pi++) { + rs_pkg_t *p = &ix->pkgs[pi]; + if (p->nraw > SKIP_ONE) { + qsort(p->raw, (size_t)p->nraw, sizeof(rs_rawdep_t), rs_rawdep_cmp); + } + } + for (int pi = 0; pi < ix->npkgs; pi++) { + rs_pkg_deps(ix, pi); + } +} + +/* ── Build: bindings ─────────────────────────────────────────────── */ + +static int rs_bind_cmp(const void *a, const void *b) { + const rs_bind_t *x = (const rs_bind_t *)a; + const rs_bind_t *y = (const rs_bind_t *)b; + if (x->mod != y->mod) { + return x->mod < y->mod ? -1 : 1; + } + int c = strcmp(x->name, y->name); + return c ? c : (x->seq > y->seq) - (x->seq < y->seq); +} + +static int rs_ref_cmp(const void *a, const void *b) { + const rs_ref_t *x = (const rs_ref_t *)a; + const rs_ref_t *y = (const rs_ref_t *)b; + if (x->mod != y->mod) { + return x->mod < y->mod ? -1 : 1; + } + return (x->seq > y->seq) - (x->seq < y->seq); +} + +static int rs_member_cmp(const void *a, const void *b) { + const rs_member_t *x = (const rs_member_t *)a; + const rs_member_t *y = (const rs_member_t *)b; + int c = strcmp(x->name, y->name); + return c ? c : (x->seq > y->seq) - (x->seq < y->seq); +} + +/* The members of every type, trait and impl, by name. */ +static void rs_sort_members(rs_index_t *ix) { + for (int i = 0; i < ix->nitems; i++) { + const rs_item_t *it = &ix->items[i]; + if (it->member_count > SKIP_ONE) { + qsort(&ix->members[it->member_first], (size_t)it->member_count, sizeof(rs_member_t), + rs_member_cmp); + } + } + for (int i = 0; i < ix->nimpls; i++) { + const rs_impl_t *im = &ix->impls[i]; + if (im->member_count > SKIP_ONE) { + qsort(&ix->members[im->member_first], (size_t)im->member_count, sizeof(rs_member_t), + rs_member_cmp); + } + } +} + +static void rs_bind_put(rs_index_t *ix, int mod, const char *name, int kind, int ref, char vis) { + ix->binds[ix->nbinds] = (rs_bind_t){ + .mod = mod, .name = name, .kind = kind, .ref = ref, .seq = ix->nbinds, .vis = vis}; + ix->nbinds++; +} + +/* An item: bound in its module, and a `#[macro_export]` macro also at the + * crate roots of its file. */ +static void rs_bind_item(rs_index_t *ix, int idx) { + const rs_item_t *it = &ix->items[idx]; + const rs_file_t *f = &ix->files[it->file]; + rs_bind_put(ix, it->mod, it->name, RS_BIND_ITEM, idx, it->vis); + for (int r = 0; it->macro_export && !f->roots_top && r < f->nroots; r++) { + if (f->roots[r] != it->mod) { + rs_bind_put(ix, f->roots[r], it->name, RS_BIND_ITEM, idx, 'p'); + } + } +} + +/* Sort `refs` by module and set each module's range through `first`/`count` + * (offsets of the two fields in rs_mod_t). */ +static void rs_ref_ranges(rs_index_t *ix, rs_ref_t *refs, int n, bool globs) { + if (n > SKIP_ONE) { + qsort(refs, (size_t)n, sizeof(rs_ref_t), rs_ref_cmp); + } + for (int i = 0; i < n; i++) { + rs_mod_t *m = &ix->mods[refs[i].mod]; + int *first = globs ? &m->glob_first : &m->named_first; + int *count = globs ? &m->glob_count : &m->named_count; + if (*count == 0) { + *first = i; + } + (*count)++; + } +} + +static void rs_bindings(rs_index_t *ix) { + size_t cap = ((size_t)ix->nitems * (RS_ROOTS_KEPT + SKIP_ONE)) + (size_t)ix->ndecls + + (size_t)ix->nmods + (size_t)ix->nimports + (size_t)ix->nmaybes; + ix->binds = (rs_bind_t *)ix_alloc(ix, cap * sizeof(rs_bind_t)); + ix->globs = (rs_ref_t *)ix_alloc(ix, (size_t)ix->nimports * sizeof(rs_ref_t)); + ix->named = (rs_ref_t *)ix_alloc(ix, (size_t)ix->nimports * sizeof(rs_ref_t)); + if (!ix->binds || !ix->globs || !ix->named) { + return; + } + for (int i = 0; i < ix->nitems; i++) { + rs_bind_item(ix, i); + } + for (int m = 0; m < ix->nmods; m++) { + if (ix->mods[m].parent >= 0) { + rs_bind_put(ix, ix->mods[m].parent, ix->mods[m].name, RS_BIND_CHILD, m, + ix->mods[m].vis); + } + } + for (int d = 0; d < ix->ndecls; d++) { + const rs_decl_t *decl = &ix->decls[d]; + rs_bind_put(ix, decl->mod, decl->name, RS_BIND_CHILD, decl->child, decl->vis); + } + for (int i = 0; i < ix->nimports; i++) { + const rs_import_t *imp = &ix->imports[i]; + rs_ref_t ref = {.mod = imp->mod, .seq = imp->seq, .ref = i}; + if (imp->kind == 'g') { + ix->globs[ix->nglobs++] = ref; + } else { + ix->named[ix->nnamed++] = ref; + rs_bind_put(ix, imp->mod, imp->local, RS_BIND_IMPORT, i, imp->vis); + } + } + for (int i = 0; i < ix->nmaybes; i++) { + rs_bind_put(ix, ix->maybes[i].mod, ix->maybe_names[i], RS_BIND_MAYBE, i, 'p'); + } + if (ix->nbinds > SKIP_ONE) { + qsort(ix->binds, (size_t)ix->nbinds, sizeof(rs_bind_t), rs_bind_cmp); + } + for (int i = 0; i < ix->nbinds; i++) { + rs_mod_t *m = &ix->mods[ix->binds[i].mod]; + if (m->bind_count == 0) { + m->bind_first = i; + } + m->bind_count++; + } + rs_ref_ranges(ix, ix->globs, ix->nglobs, true); + rs_ref_ranges(ix, ix->named, ix->nnamed, false); +} + +/* ── Build: imports ──────────────────────────────────────────────── */ + +/* `extern crate name;`: the crate of that name, or `self` for the crate + * itself. */ +static void rs_import_extern(rs_look_t *lk, const rs_import_t *imp, int idx, rs_found_t *found) { + rs_at_t at[RS_ROOTS_KEPT]; + int n = rs_contexts(lk, imp->mod, idx, at); + for (int i = 0; i < n; i++) { + bool unknown = false; + if (rs_is(imp->segs[0], "self")) { + unknown = at[i].root < 0; + rs_set_add(&found->ns[0], unknown ? rs_weak(RS_E_GAP) : rs_ent(RS_E_MOD, at[i].root)); + continue; + } + rs_set_t one = {0}; + rs_extern(&at[i], imp->segs[0], &one, &unknown); + if (rs_set_empty(&one)) { + /* a crate the manifest names differently (see rs_lexical), unless + * the crate's dependencies are not known at all */ + rs_set_add(&one, rs_weak(unknown ? RS_E_GAP : RS_E_EXTERNAL)); + } + rs_set_merge(&found->ns[0], &one); + } +} + +/* Keep what an import path was found to name. An import DECLARES its name: + * whatever the path leads to, the name is bound, so a path that leads into + * something unknown binds the name to a gap (or to something external) in + * every namespace rather than leaving it unbound. */ +static void rs_import_store(rs_index_t *ix, rs_import_t *imp, const rs_found_t *found, int pass) { + bool strong = false; + uint8_t weak = RS_E_NONE; + for (int k = 0; k < RS_NS_COUNT; k++) { + uint8_t w = rs_only_weak(&found->ns[k]); + imp->res[k] = rs_only_strong(ix, &found->ns[k]); + imp->res[k].via_import = false; + strong = strong || imp->res[k].kind != RS_E_NONE; + weak = (weak == RS_E_GAP || w == RS_E_NONE) ? weak : w; + } + if (found->ambiguous || (!strong && weak != RS_E_EXTERNAL)) { + weak = RS_E_GAP; + ix->gap_imports++; + } + imp->spread = imp->kind == 'b' && (found->ambiguous || !strong); + for (int k = 0; k < RS_NS_COUNT && (found->ambiguous || !strong); k++) { + imp->res[k] = rs_ent(weak, 0); + } + imp->state = RS_STATE_DONE; + imp->done_pass = pass; +} + +/* Resolve import `idx` in pass `pass`. An import that waits for another one + * stays unresolved, unless this is the `last` attempt: then it is not + * followed. */ +static void rs_import_resolve(rs_index_t *ix, int idx, int pass, bool last) { + rs_import_t *imp = &ix->imports[idx]; + rs_look_t lk = {.ix = ix, .pass = pass}; + rs_found_t found; + memset(&found, 0, sizeof(found)); + if (imp->nsegs > 0 && imp->kind == 'x') { + rs_import_extern(&lk, imp, idx, &found); + } else if (imp->nsegs > 0) { + rs_path_t p = {.segs = imp->segs, .n = imp->nsegs, .global = imp->global}; + rs_ent_t no_self = {0}; + int mask = imp->kind == 'b' ? RS_NS_ALL : RS_NS_TYPE; + rs_resolve_path(&lk, imp->mod, &p, mask, no_self, idx, &found); + } + if (lk.blocked && !last) { + return; + } + if (lk.blocked) { + memset(&found, 0, sizeof(found)); + } + rs_import_store(ix, imp, &found, pass); +} + +/* Resolve every import. Imports depend on each other (a path can run through + * an imported name or a glob), so they are resolved in passes: a pass reads + * only what EARLIER passes resolved, which makes the result independent of + * the order of the imports. What is still waiting after the last pass (a + * cycle, or a chain longer than the passes) is not followed. */ +static void rs_imports(rs_index_t *ix) { + int pending = ix->nimports; + int pass = SKIP_ONE; + for (; pass <= RS_IMPORT_PASSES && pending > 0; pass++) { + int before = pending; + for (int i = 0; i < ix->nimports; i++) { + if (ix->imports[i].state != RS_STATE_DONE) { + rs_import_resolve(ix, i, pass, false); + pending -= ix->imports[i].state == RS_STATE_DONE; + } + } + if (pending == before) { + break; + } + } + for (int i = 0; pending > 0 && i < ix->nimports; i++) { + if (ix->imports[i].state != RS_STATE_DONE) { + rs_import_resolve(ix, i, pass + SKIP_ONE, true); + } + } +} + +/* ── Build: aliases and impls ────────────────────────────────────── */ + +/* What the type path `text`, written in module `mod`, names. */ +static rs_ent_t rs_type_path(rs_index_t *ix, int mod, const char *text) { + CBMDocLinkRsPath parsed; + const char *segs[CBM_DOCLINK_RS_MAX_SEGS]; + if (cbm_doclink_rs_parse_path(text, strlen(text), false, &parsed) != CBM_DOCLINK_RS_PATH_OK) { + return rs_ent(RS_E_GAP, 0); + } + for (int i = 0; i < parsed.nsegs; i++) { + segs[i] = rs_plain(parsed.segs[i]); + } + rs_path_t p = {.segs = segs, .n = parsed.nsegs, .global = parsed.global}; + rs_look_t lk = {.ix = ix}; + rs_found_t found; + rs_ent_t no_self = {0}; + rs_resolve_path(&lk, mod, &p, RS_NS_TYPE, no_self, CBM_NOT_FOUND, &found); + if (found.ambiguous) { + return rs_ent(RS_E_GAP, 0); + } + rs_ent_t e = rs_only_strong(ix, &found.ns[0]); + uint8_t weak = rs_only_weak(&found.ns[0]); + return (e.kind == RS_E_NONE && weak != RS_E_NONE) ? rs_ent(weak, 0) : e; +} + +static int rs_item_impl_cmp(const void *a, const void *b) { + const rs_item_impl_t *x = (const rs_item_impl_t *)a; + const rs_item_impl_t *y = (const rs_item_impl_t *)b; + if (x->item != y->item) { + return x->item < y->item ? -1 : 1; + } + return (x->impl > y->impl) - (x->impl < y->impl); +} + +static int rs_assoc_cmp(const void *a, const void *b) { + const rs_assoc_t *x = (const rs_assoc_t *)a; + const rs_assoc_t *y = (const rs_assoc_t *)b; + if (x->item != y->item) { + return x->item < y->item ? -1 : 1; + } + int c = strcmp(x->name, y->name); + if (c) { + return c; + } + return x->impl != y->impl ? (x->impl < y->impl ? -1 : 1) + : (x->member > y->member) - (x->member < y->member); +} + +/* The type an impl is for and the trait it implements. */ +static void rs_impl_types(rs_index_t *ix, rs_impl_t *im) { + rs_ent_t none = {0}; + rs_ent_t self = im->self_path ? rs_type_path(ix, im->mod, im->self_path) : none; + if (self.kind == RS_E_ITEM) { + self = rs_alias_end(ix, self.idx); + self.weak = false; + } + if (self.kind == RS_E_ITEM && !rs_item_is_type(ix->items[self.idx].kind)) { + self = rs_ent(RS_E_GAP, 0); + } + im->self_ent = self; + if (im->trait_path) { + rs_ent_t trait = rs_is(im->trait_path, "?") ? rs_ent(RS_E_GAP, 0) + : rs_type_path(ix, im->mod, im->trait_path); + im->trait_ent = trait.kind == RS_E_NONE ? rs_ent(RS_E_GAP, 0) : trait; + } +} + +/* Resolve alias targets and impl headers, and index every impl's members by + * the type the impl is for. */ +static void rs_types(rs_index_t *ix) { + int members = 0; + for (int i = 0; i < ix->nitems; i++) { + rs_item_t *it = &ix->items[i]; + if (it->alias) { + it->alias_ent = rs_type_path(ix, it->mod, it->alias); + } + } + for (int i = 0; i < ix->nimpls; i++) { + rs_impl_types(ix, &ix->impls[i]); + members += ix->impls[i].member_count; + } + ix->item_impls = (rs_item_impl_t *)ix_alloc(ix, (size_t)ix->nimpls * sizeof(rs_item_impl_t)); + ix->assoc = (rs_assoc_t *)ix_alloc(ix, (size_t)members * sizeof(rs_assoc_t)); + if (!ix->item_impls || !ix->assoc) { + return; + } + for (int i = 0; i < ix->nimpls; i++) { + const rs_impl_t *im = &ix->impls[i]; + for (int k = 0; k < im->member_count; k++) { + int m = im->member_first + k; + if (im->self_ent.kind == RS_E_ITEM) { + ix->assoc[ix->nassoc++] = (rs_assoc_t){ + .item = im->self_ent.idx, .name = ix->members[m].name, .impl = i, .member = m}; + } else if (!im->self_path) { + cbm_ht_set(ix->blanket, ix->members[m].name, (void *)(intptr_t)SKIP_ONE); + } + } + if (im->self_ent.kind == RS_E_ITEM) { + ix->item_impls[ix->nitem_impls++] = + (rs_item_impl_t){.item = im->self_ent.idx, .impl = i}; + } + } + if (ix->nitem_impls > SKIP_ONE) { + qsort(ix->item_impls, (size_t)ix->nitem_impls, sizeof(rs_item_impl_t), rs_item_impl_cmp); + } + if (ix->nassoc > SKIP_ONE) { + qsort(ix->assoc, (size_t)ix->nassoc, sizeof(rs_assoc_t), rs_assoc_cmp); + } + for (int i = 0; i < ix->nitem_impls; i++) { + rs_item_t *it = &ix->items[ix->item_impls[i].item]; + if (it->impl_count == 0) { + it->impl_first = i; + } + it->impl_count++; + } +} + +/* ── Build: line indexes of the re-extracted files ───────────────── */ + +typedef struct { + uint32_t line; + int order; /* at one line: modules are left innermost first, then entered outermost first */ + int mod; /* the innermost module from that line on */ +} rs_event_t; + +static int rs_event_cmp(const void *a, const void *b) { + const rs_event_t *x = (const rs_event_t *)a; + const rs_event_t *y = (const rs_event_t *)b; + if (x->line != y->line) { + return x->line < y->line ? -1 : 1; + } + return (x->order > y->order) - (x->order < y->order); +} + +static int rs_scope_cmp(const void *a, const void *b) { + const rs_scope_t *x = (const rs_scope_t *)a; + const rs_scope_t *y = (const rs_scope_t *)b; + if (x->start != y->start) { + return x->start < y->start ? -1 : 1; + } + return (x->end > y->end) - (x->end < y->end); +} + +static int rs_note_cmp(const void *a, const void *b) { + const rs_note_t *x = (const rs_note_t *)a; + const rs_note_t *y = (const rs_note_t *)b; + if (x->line != y->line) { + return x->line < y->line ? -1 : 1; + } + return strcmp(x->text, y->text); +} + +static int rs_line_cmp(const void *a, const void *b) { + uint32_t x = *(const uint32_t *)a; + uint32_t y = *(const uint32_t *)b; + return (x > y) - (x < y); +} + +/* The spans of file `f`: for every line, the innermost inline module. */ +static void rs_file_spans(rs_index_t *ix, rs_file_t *f, rs_event_t *ev) { + int n = 0; + for (int m = SKIP_ONE; m < f->nmods; m++) { + const rs_mod_t *mod = &ix->mods[f->mod_base + m]; + if (mod->parent < 0) { + continue; + } + ev[n++] = (rs_event_t){.line = mod->start, .order = m, .mod = f->mod_base + m}; + ev[n++] = (rs_event_t){.line = mod->end + SKIP_ONE, .order = -m, .mod = mod->parent}; + } + if (n > SKIP_ONE) { + qsort(ev, (size_t)n, sizeof(rs_event_t), rs_event_cmp); + } + f->span_first = ix->nspans; + ix->spans[ix->nspans++] = (rs_span_t){.from = 0, .mod = f->mod_base}; + for (int i = 0; i < n; i++) { + ix->spans[ix->nspans++] = (rs_span_t){.from = ev[i].line, .mod = ev[i].mod}; + } + f->span_count = ix->nspans - f->span_first; +} + +static void rs_file_lines(rs_index_t *ix, rs_file_t *f, rs_event_t *ev) { + rs_file_spans(ix, f, ev); + if (f->scope_count > SKIP_ONE) { + qsort(&ix->scopes[f->scope_first], (size_t)f->scope_count, sizeof(rs_scope_t), + rs_scope_cmp); + } + if (f->local_count > SKIP_ONE) { + qsort(&ix->locals[f->local_first], (size_t)f->local_count, sizeof(rs_note_t), rs_note_cmp); + } + if (f->unowned_count > SKIP_ONE) { + qsort(&ix->unowned[f->unowned_first], (size_t)f->unowned_count, sizeof(rs_note_t), + rs_note_cmp); + } + f->test_first = ix->ntest_lines; + for (int i = f->item_first; i < f->item_first + f->item_count; i++) { + if (ix->items[i].test) { + ix->test_lines[ix->ntest_lines++] = ix->items[i].start; + } + } + f->test_count = ix->ntest_lines - f->test_first; + if (f->test_count > SKIP_ONE) { + qsort(&ix->test_lines[f->test_first], (size_t)f->test_count, sizeof(uint32_t), rs_line_cmp); + } +} + +static void rs_line_indexes(rs_index_t *ix) { + size_t spans = 0; + size_t events = 0; + for (int fi = 0; fi < ix->nfiles; fi++) { + const rs_file_t *f = &ix->files[fi]; + if (f->kind == RS_FILE_SOURCE && f->fresh) { + size_t ev = (size_t)(f->nmods - SKIP_ONE) * PAIR_LEN; + spans += ev + SKIP_ONE; + events = ev > events ? ev : events; + } + } + ix->spans = (rs_span_t *)ix_alloc(ix, spans * sizeof(rs_span_t)); + ix->test_lines = (uint32_t *)ix_alloc(ix, (size_t)ix->nitems * sizeof(uint32_t)); + rs_event_t *ev = (rs_event_t *)ix_alloc(ix, events * sizeof(rs_event_t)); + if (!ix->spans || !ix->test_lines || !ev) { + return; + } + for (int fi = 0; fi < ix->nfiles; fi++) { + if (ix->files[fi].kind == RS_FILE_SOURCE && ix->files[fi].fresh) { + rs_file_lines(ix, &ix->files[fi], ev); + } + } +} + +/* ── Build ───────────────────────────────────────────────────────── */ + +/* The scope blob of a Rust source file, NULL when it has none this code + * reads. */ +static const char *rs_source_blob(const char *scope) { + static const char tag[] = CBM_DOCLINK_RS_SCOPE_TAG; + const size_t tl = sizeof(tag) - SKIP_ONE; + if (!scope || strncmp(scope, tag, tl) != 0 || (scope[tl] != '\n' && scope[tl] != '\0') || + rs_blob_is_manifest(scope)) { + return NULL; + } + return scope; +} + +static void rs_count_add(rs_counts_t *sum, const rs_counts_t *c) { + sum->mods += c->mods; + sum->decls += c->decls; + sum->items += c->items; + sum->members += c->members; + sum->impls += c->impls; + sum->imports += c->imports; + sum->maybes += c->maybes; + sum->locals += c->locals; + sum->unowned += c->unowned; + sum->macuses += c->macuses; + sum->macdecls += c->macdecls; +} + +/* The facts of source file `f` that need no blob, and the size of its blob. */ +static void rs_source_meta(rs_index_t *ix, rs_file_t *f, const cbm_doclink_file_t *src, + rs_counts_t *sum) { + rs_counts_t c = {0}; + /* the qualified name of an item `x` of this file, minus the `x` */ + const char *probe = cbm_fqn_compute(&ix->arena, ix->project, src->rel_path, "x"); + size_t pl = probe ? strlen(probe) : 0; + f->kind = RS_FILE_SOURCE; + f->fresh = src->run_file >= 0; + f->test_path = rs_is_test_path(src->rel_path); + f->item_prefix = pl > 0 ? ix_strndup(ix, probe, pl - SKIP_ONE) : NULL; + f->module_qn = + cbm_fqn_module_source_lang(&ix->arena, ix->project, src->rel_path, CBM_LANG_RUST); + ix->oom = ix->oom || !f->item_prefix || !f->module_qn; + rs_count_blob(rs_source_blob(src->scope), &c); + f->nmods = c.mods + SKIP_ONE; + f->mod_base = ix->nmods; + ix->nmods += f->nmods; + ix->nsources++; + rs_count_add(sum, &c); +} + +static bool rs_alloc_tables(rs_index_t *ix, const rs_counts_t *c, int manifests) { + ix->mods = (rs_mod_t *)ix_calloc(ix, (size_t)ix->nmods, sizeof(rs_mod_t)); + ix->decls = (rs_decl_t *)ix_calloc(ix, (size_t)c->decls, sizeof(rs_decl_t)); + ix->items = (rs_item_t *)ix_calloc(ix, (size_t)c->items, sizeof(rs_item_t)); + ix->members = (rs_member_t *)ix_calloc(ix, (size_t)c->members, sizeof(rs_member_t)); + ix->impls = (rs_impl_t *)ix_calloc(ix, (size_t)c->impls, sizeof(rs_impl_t)); + ix->imports = (rs_import_t *)ix_calloc(ix, (size_t)c->imports, sizeof(rs_import_t)); + ix->maybes = (rs_ref_t *)ix_calloc(ix, (size_t)c->maybes, sizeof(rs_ref_t)); + ix->maybe_names = (const char **)ix_calloc(ix, (size_t)c->maybes, sizeof(char *)); + ix->macuses = (rs_macuse_t *)ix_calloc(ix, (size_t)c->macuses, sizeof(rs_macuse_t)); + ix->macdecls = (rs_macdecl_t *)ix_calloc(ix, (size_t)c->macdecls, sizeof(rs_macdecl_t)); + ix->scopes = + (rs_scope_t *)ix_calloc(ix, (size_t)c->items + (size_t)c->impls, sizeof(rs_scope_t)); + ix->locals = (rs_note_t *)ix_calloc(ix, (size_t)c->locals, sizeof(rs_note_t)); + ix->unowned = (rs_note_t *)ix_calloc(ix, (size_t)c->unowned, sizeof(rs_note_t)); + ix->pkgs = (rs_pkg_t *)ix_calloc(ix, (size_t)manifests, sizeof(rs_pkg_t)); + return !ix->oom; +} + +/* Read every file's scope blob into the index. */ +static bool rs_load(rs_index_t *ix, const cbm_doclink_build_in_t *in) { + rs_counts_t sum = {0}; + int manifests = 0; + ix->nfiles = in->file_count; + ix->files = (rs_file_t *)ix_calloc(ix, (size_t)in->file_count, sizeof(rs_file_t)); + if (!ix->files) { + return false; + } + for (int i = 0; i < in->file_count; i++) { + const cbm_doclink_file_t *src = &in->files[i]; + rs_file_t *f = &ix->files[i]; + f->rel_path = src->rel_path; + f->root_pkg = RS_PKG_NONE; + f->pkg = RS_PKG_NONE; + cbm_ht_set(ix->file_by_path, f->rel_path, (void *)(intptr_t)(i + SKIP_ONE)); + if (src->run_file >= 0 && src->run_file < in->run_file_count) { + ix->run_to_file[src->run_file] = i; + } + if (rs_ends_with(src->rel_path, ".rs")) { + rs_source_meta(ix, f, src, &sum); + } else if (rs_is(rs_basename(src->rel_path), "Cargo.toml") && + rs_blob_is_manifest(src->scope)) { + f->kind = RS_FILE_MANIFEST; + manifests++; + } + } + if (!rs_alloc_tables(ix, &sum, manifests)) { + return false; + } + for (int i = 0; i < in->file_count; i++) { + rs_file_t *f = &ix->files[i]; + if (f->kind == RS_FILE_MANIFEST) { + rs_read_manifest(ix, i, in->files[i].scope); + } + for (int m = 0; f->kind == RS_FILE_SOURCE && m < f->nmods; m++) { + ix->mods[f->mod_base + m].file = i; + ix->mods[f->mod_base + m].parent = CBM_NOT_FOUND; + } + if (f->kind == RS_FILE_SOURCE) { + rs_read_source(ix, i, rs_source_blob(in->files[i].scope)); + } + } + for (int pi = 0; pi < ix->npkgs; pi++) { + const char *dir = ix->pkgs[pi].dir; + cbm_ht_set(ix->pkg_by_dir, dir[0] ? dir : ".", (void *)(intptr_t)(pi + SKIP_ONE)); + } + if (ix->nmacuses > SKIP_ONE) { + qsort(ix->macuses, (size_t)ix->nmacuses, sizeof(rs_macuse_t), rs_macuse_cmp); + } + if (ix->nmacdecls > SKIP_ONE) { + qsort(ix->macdecls, (size_t)ix->nmacdecls, sizeof(rs_macdecl_t), rs_macdecl_cmp); + } + return !ix->oom; +} + +static void rs_destroy(void *index) { + rs_index_t *ix = (rs_index_t *)index; + if (!ix) { + return; + } + cbm_ht_free(ix->file_by_path); + cbm_ht_free(ix->pkg_by_dir); + cbm_ht_free(ix->blanket); + cbm_free(CBM_MEM_CLASS_OTHER, ix->run_to_file); + cbm_arena_destroy(&ix->arena); + cbm_free(CBM_MEM_CLASS_OTHER, ix); +} + +static void rs_log_index(const rs_index_t *ix) { + enum { N_FILES = 0, N_PKGS, N_ROOTS, N_ORPHANS, N_MODS, N_ITEMS, N_IMPORTS, N_GAPS, N_COUNT }; + char b[N_COUNT][CBM_SZ_32]; + snprintf(b[N_FILES], sizeof(b[N_FILES]), "%d", ix->nsources); + snprintf(b[N_PKGS], sizeof(b[N_PKGS]), "%d", ix->npkgs); + snprintf(b[N_ROOTS], sizeof(b[N_ROOTS]), "%d", ix->crate_roots); + snprintf(b[N_ORPHANS], sizeof(b[N_ORPHANS]), "%d", ix->orphans); + snprintf(b[N_MODS], sizeof(b[N_MODS]), "%d", ix->nmods); + snprintf(b[N_ITEMS], sizeof(b[N_ITEMS]), "%d", ix->nitems); + snprintf(b[N_IMPORTS], sizeof(b[N_IMPORTS]), "%d", ix->nimports); + snprintf(b[N_GAPS], sizeof(b[N_GAPS]), "%d", ix->gap_imports); + cbm_log_info("doc_links.rs.index", "files", b[N_FILES], "manifests", b[N_PKGS], "crate_roots", + b[N_ROOTS], "orphan_files", b[N_ORPHANS], "modules", b[N_MODS], "items", + b[N_ITEMS], "imports", b[N_IMPORTS], "unfollowed_imports", b[N_GAPS]); +} + +static void *rs_build(const cbm_doclink_build_in_t *in) { + rs_index_t *ix = (rs_index_t *)cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*ix)); + if (!ix) { + return NULL; + } + cbm_arena_init(&ix->arena); + ix->project = in->ctx->project_name; + ix->file_by_path = cbm_ht_create(CBM_SZ_4K); + ix->pkg_by_dir = cbm_ht_create(CBM_SZ_64); + ix->blanket = cbm_ht_create(CBM_SZ_256); + ix->run_count = in->run_file_count; + ix->run_to_file = (int *)cbm_alloc( + CBM_MEM_CLASS_OTHER, + (size_t)(in->run_file_count ? in->run_file_count : SKIP_ONE) * sizeof(int)); + if (!ix->file_by_path || !ix->pkg_by_dir || !ix->blanket || !ix->run_to_file) { + cbm_log_error("doc_links.rs.error", "step", "tables", "reason", "alloc"); + rs_destroy(ix); + return NULL; + } + for (int i = 0; i < in->run_file_count; i++) { + ix->run_to_file[i] = CBM_NOT_FOUND; + } + if (rs_load(ix, in)) { + rs_crate_roots(ix); + rs_module_tree(ix); + rs_file_roots(ix); + rs_orphan_packages(ix); + rs_product(ix); + rs_deps(ix); + rs_sort_members(ix); + rs_bindings(ix); + } + if (!ix->oom) { + rs_imports(ix); + rs_types(ix); + rs_line_indexes(ix); + } + if (ix->oom) { + cbm_log_error("doc_links.rs.error", "step", "index", "reason", "alloc"); + rs_destroy(ix); + return NULL; + } + rs_log_index(ix); + return ix; +} + +/* ── One reference ───────────────────────────────────────────────── */ + +typedef struct { + const rs_index_t *ix; + const rs_file_t *f; + const CBMDocLink *link; + const char *segs[CBM_DOCLINK_RS_MAX_SEGS]; + rs_path_t path; + int dis; + int mod; /* the module the documented definition is in */ + rs_ent_t self; /* what `Self` names there */ + bool single; /* one plain name: no `::`, no keyword */ + bool file_level; + bool product; /* the documented definition is product code */ +} rs_link_t; + +static bool rs_is_keyword(const char *seg) { + return rs_is(seg, "crate") || rs_is(seg, "self") || rs_is(seg, "super") || rs_is(seg, "Self"); +} + +/* The innermost module of file `f` at `line`. */ +static int rs_mod_at(const rs_index_t *ix, const rs_file_t *f, uint32_t line) { + int lo = f->span_first; + int hi = lo + f->span_count; + while (lo < hi) { + int mid = lo + ((hi - lo) / PAIR_LEN); + if (ix->spans[mid].from <= line) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + return lo > f->span_first ? ix->spans[lo - SKIP_ONE].mod : f->mod_base; +} + +/* What `Self` names at `line`: the type or trait whose definition holds the + * line, the self type of the impl that does, or nothing. */ +static rs_ent_t rs_self_at(const rs_index_t *ix, const rs_file_t *f, uint32_t line) { + rs_ent_t none = {0}; + int lo = f->scope_first; + int hi = lo + f->scope_count; + while (lo < hi) { + int mid = lo + ((hi - lo) / PAIR_LEN); + if (ix->scopes[mid].start <= line) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + if (lo == f->scope_first || ix->scopes[lo - SKIP_ONE].end < line) { + return none; + } + const rs_scope_t *s = &ix->scopes[lo - SKIP_ONE]; + if (s->item >= 0) { + return rs_ent(RS_E_ITEM, s->item); + } + rs_ent_t self = ix->impls[s->impl].self_ent; + return self.kind == RS_E_ITEM ? self : rs_ent(RS_E_IMPL, s->impl); +} + +/* First note of notes[first, first + count) at or after `line`. */ +static int rs_note_at(const rs_note_t *notes, int first, int count, uint32_t line) { + int lo = first; + int hi = first + count; + while (lo < hi) { + int mid = lo + ((hi - lo) / PAIR_LEN); + if (notes[mid].line < line) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + return lo; +} + +/* True when the documented definition does not own its graph node: another + * definition of the file has the same qualified name and keeps the node, so + * an edge would leave from that other definition. */ +static bool rs_unowned(const rs_index_t *ix, const rs_file_t *f, const CBMDocLink *link) { + size_t pl = strlen(f->item_prefix); + const char *qn = link->source_qn ? link->source_qn : ""; + int end = f->unowned_first + f->unowned_count; + for (int i = rs_note_at(ix->unowned, f->unowned_first, f->unowned_count, link->def_line); + i < end && ix->unowned[i].line == link->def_line; i++) { + if (strncmp(qn, f->item_prefix, pl) == 0 && rs_is(qn + pl, ix->unowned[i].text)) { + return true; + } + } + return false; +} + +/* True when `name` is a parameter or type parameter of the definition the + * reference documents. */ +static bool rs_is_local(const rs_link_t *q, const char *name) { + const rs_index_t *ix = q->ix; + const rs_file_t *f = q->f; + size_t nl = strlen(name); + int end = f->local_first + f->local_count; + if (q->file_level) { + return false; + } + for (int i = rs_note_at(ix->locals, f->local_first, f->local_count, q->link->def_line); + i < end && ix->locals[i].line == q->link->def_line; i++) { + for (const char *p = ix->locals[i].text; *p;) { + const char *comma = strchr(p, ','); + size_t len = comma ? (size_t)(comma - p) : strlen(p); + const char *plain = (len > PAIR_LEN && p[0] == 'r' && p[1] == '#') ? p + PAIR_LEN : p; + size_t plen = len - (size_t)(plain - p); + if (plen == nl && memcmp(plain, name, nl) == 0) { + return true; + } + p = comma ? comma + SKIP_ONE : p + len; + } + } + return false; +} + +/* Is the documented definition product code? Not in a test-only module, not + * an item marked as a test, not inside a type marked as one. */ +static bool rs_source_product(const rs_link_t *q) { + const rs_index_t *ix = q->ix; + const rs_file_t *f = q->f; + if (!ix->mods[q->mod].product) { + return false; + } + if (q->file_level) { + return true; + } + if (q->self.kind == RS_E_ITEM && ix->items[q->self.idx].test) { + return false; + } + int lo = f->test_first; + int hi = lo + f->test_count; + while (lo < hi) { + int mid = lo + ((hi - lo) / PAIR_LEN); + if (ix->test_lines[mid] < q->link->def_line) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + return !(lo < f->test_first + f->test_count && ix->test_lines[lo] == q->link->def_line); +} + +static const char *rs_label_name(char code) { + static const struct { + char code; + const char *label; + } table[] = { + {'S', "Struct"}, {'E', "Enum"}, {'C', "Class"}, {'I', "Interface"}, {'T', "Type"}, + {'F', "Function"}, {'V', "Variable"}, {'M', "Method"}, {'D', "Field"}, + }; + for (size_t i = 0; i < sizeof(table) / sizeof(table[0]); i++) { + if (table[i].code == code) { + return table[i].label; + } + } + return NULL; +} + +/* The node of a declaration of file `fi`: looked up by qualified name, and + * taken only when its label and file are the declaration's (another + * definition can own that name: a same-named item of another kind, another + * file's module). */ +static const cbm_gbuf_node_t *rs_node(const rs_index_t *ix, const cbm_gbuf_t *graph, int fi, + const char *label, const char *qn) { + const cbm_gbuf_node_t *n = (label && qn) ? cbm_gbuf_find_by_qn(graph, qn) : NULL; + if (!n || !n->label || !n->file_path || !rs_is(n->label, label) || + !rs_is(n->file_path, ix->files[fi].rel_path)) { + return NULL; + } + return n; +} + +static const cbm_gbuf_node_t *rs_decl_node(const rs_index_t *ix, const cbm_gbuf_t *graph, int fi, + char label, const char *suffix) { + char qn[RS_QN_BUF]; + if (!label || !suffix || + snprintf(qn, sizeof(qn), "%s%s", ix->files[fi].item_prefix, suffix) >= (int)sizeof(qn)) { + return NULL; + } + return rs_node(ix, graph, fi, rs_label_name(label), qn); +} + +/* The outcome for the one entity a reference names. */ +static void rs_outcome(const rs_link_t *q, const cbm_gbuf_t *graph, rs_ent_t e, + cbm_doclink_outcome_t *out) { + const rs_index_t *ix = q->ix; + const cbm_gbuf_node_t *node = NULL; + bool product = true; + out->reason = CBM_DOCLINK_REASON_GRAPH_GAP; + if (e.kind == RS_E_ITEM) { + const rs_item_t *it = &ix->items[e.idx]; + product = ix->mods[it->mod].product && !it->test; + node = rs_decl_node(ix, graph, it->file, it->label, it->suffix); + } else if (e.kind == RS_E_MEMBER) { + const rs_member_t *m = &ix->members[e.idx]; + product = ix->mods[m->mod].product && !m->owner_test; + node = rs_decl_node(ix, graph, m->file, m->label, m->suffix); + } else if (e.kind == RS_E_MOD) { + const rs_mod_t *mod = &ix->mods[e.idx]; + product = mod->product; + node = mod->parent < 0 + ? rs_node(ix, graph, mod->file, "Module", ix->files[mod->file].module_qn) + : NULL; /* an inline module has no node */ + } else { + bool external = e.kind == RS_E_EXTERNAL || + (e.kind == RS_E_IMPL && ix->impls[e.idx].self_ent.kind == RS_E_EXTERNAL); + out->reason = external ? CBM_DOCLINK_REASON_EXTERNAL : CBM_DOCLINK_REASON_GRAPH_GAP; + return; + } + if (q->product && !product) { + out->reason = CBM_DOCLINK_REASON_TEST_ONLY; + } else if (node) { + out->kind = CBM_DOCLINK_EDGE; + out->target = node; + } +} + +/* The namespaces a disambiguator or suffix asks for. */ +static int rs_dis_mask(int dis) { + switch (dis) { + case CBM_DOCLINK_RS_DIS_NONE: + return RS_NS_ALL; + case CBM_DOCLINK_RS_DIS_CONST: + case CBM_DOCLINK_RS_DIS_STATIC: + case CBM_DOCLINK_RS_DIS_FN: + case CBM_DOCLINK_RS_DIS_FIELD: + case CBM_DOCLINK_RS_DIS_VALUE_NS: + return RS_NS_VALUE; + case CBM_DOCLINK_RS_DIS_DERIVE: + case CBM_DOCLINK_RS_DIS_MACRO_BANG: + case CBM_DOCLINK_RS_DIS_MACRO_NS: + return RS_NS_MACRO; + default: + return RS_NS_TYPE; + } +} + +/* The item kinds and member kinds a disambiguator accepts ("" for none, NULL + * for any). */ +static void rs_dis_kinds(int dis, const char **items, const char **members, bool *mod) { + *items = ""; + *members = ""; + *mod = false; + switch (dis) { + case CBM_DOCLINK_RS_DIS_NONE: + case CBM_DOCLINK_RS_DIS_TYPE_NS: + case CBM_DOCLINK_RS_DIS_VALUE_NS: + case CBM_DOCLINK_RS_DIS_MACRO_NS: + *items = NULL; + *members = NULL; + *mod = true; + break; + case CBM_DOCLINK_RS_DIS_STRUCT: + *items = "S"; + break; + case CBM_DOCLINK_RS_DIS_ENUM: + *items = "E"; + break; + case CBM_DOCLINK_RS_DIS_TRAIT: + *items = "R"; + break; + case CBM_DOCLINK_RS_DIS_UNION: + *items = "U"; + break; + case CBM_DOCLINK_RS_DIS_TYALIAS: + *items = "A"; + break; + case CBM_DOCLINK_RS_DIS_MOD: + *mod = true; + break; + case CBM_DOCLINK_RS_DIS_CONST: + *items = "C"; + *members = "c"; + break; + case CBM_DOCLINK_RS_DIS_STATIC: + *items = "G"; + break; + case CBM_DOCLINK_RS_DIS_FN: + *items = "F"; + *members = "f"; + break; + case CBM_DOCLINK_RS_DIS_DERIVE: + *items = "V"; + break; + case CBM_DOCLINK_RS_DIS_FIELD: + *members = "d"; + break; + case CBM_DOCLINK_RS_DIS_VARIANT: + *members = "v"; + break; + case CBM_DOCLINK_RS_DIS_MACRO_BANG: + *items = "MP"; + break; + default: + break; + } +} + +/* Drop the candidates of another kind than the disambiguator names. What is + * outside the repository or not placed has no known kind and stays. */ +static void rs_filter_dis(const rs_index_t *ix, rs_set_t *s, int dis) { + const char *items = NULL; + const char *members = NULL; + bool mod = false; + int w = 0; + rs_dis_kinds(dis, &items, &members, &mod); + for (int i = 0; i < s->n; i++) { + rs_ent_t e = s->v[i]; + bool keep = true; + if (e.kind == RS_E_ITEM) { + keep = !items || strchr(items, ix->items[e.idx].kind) != NULL; + } else if (e.kind == RS_E_MEMBER) { + keep = !members || strchr(members, ix->members[e.idx].kind) != NULL; + } else if (e.kind == RS_E_MOD) { + keep = mod; + } + if (keep) { + s->v[w++] = e; + } + } + s->n = w; +} + +/* rustdoc: a trait and a derive macro of one name are the trait. A derive + * macro of another crate is known only as something external of that name. */ +static void rs_trait_wins(const rs_index_t *ix, rs_set_t *s) { + if (s->overflow || s->n != PAIR_LEN) { + return; + } + for (int i = 0; i < PAIR_LEN; i++) { + rs_ent_t a = s->v[i]; + rs_ent_t b = s->v[SKIP_ONE - i]; + bool trait = a.kind == RS_E_ITEM && ix->items[a.idx].kind == 'R'; + bool derive = + b.kind == RS_E_EXTERNAL || (b.kind == RS_E_ITEM && ix->items[b.idx].kind == 'V'); + if (trait && derive) { + s->v[0] = a; + s->n = SKIP_ONE; + return; + } + } +} + +/* The candidates of a reference: what was really found in any namespace, or, + * when nothing was, the one weak answer (`*weak`). */ +static void rs_candidates(const rs_index_t *ix, const rs_found_t *found, rs_set_t *out, + bool *weak) { + uint8_t w = RS_E_NONE; + for (int k = 0; k < RS_NS_COUNT; k++) { + const rs_set_t *s = &found->ns[k]; + uint8_t wk = rs_only_weak(s); + for (int i = 0; i < s->n; i++) { + if (!s->v[i].weak) { + rs_set_add(out, s->v[i]); + } + } + out->overflow = out->overflow || s->overflow; + w = (w == RS_E_GAP || wk == RS_E_NONE) ? w : wk; + } + rs_collapse(ix, out); + *weak = rs_set_empty(out) && w != RS_E_NONE; + if (*weak) { + rs_set_add(out, rs_ent(w, 0)); + } +} + +/* True when one of the standard derives in the comma-joined list `derives` + * generates a method `name` (the methods of the trait each one implements: + * defined by the language). What a derive of another crate generates is not + * known and is not assumed. */ +static bool rs_derive_declares(const char *derives, const char *name) { + static const struct { + const char *derive; + const char *methods; /* each between commas */ + } table[] = { + {"Clone", ",clone,clone_from,"}, + {"Debug", ",fmt,"}, + {"Default", ",default,"}, + {"PartialEq", ",eq,ne,"}, + {"PartialOrd", ",partial_cmp,lt,le,gt,ge,"}, + {"Ord", ",cmp,max,min,clamp,"}, + {"Hash", ",hash,hash_slice,"}, + }; + char key[CBM_SZ_64]; + if (snprintf(key, sizeof(key), ",%s,", name) >= (int)sizeof(key)) { + return false; + } + for (const char *p = derives; *p;) { + const char *comma = strchr(p, ','); + size_t len = comma ? (size_t)(comma - p) : strlen(p); + for (size_t i = 0; i < sizeof(table) / sizeof(table[0]); i++) { + if (strlen(table[i].derive) == len && memcmp(table[i].derive, p, len) == 0 && + strstr(table[i].methods, key)) { + return true; + } + } + p = comma ? comma + SKIP_ONE : p + len; + } + return false; +} + +/* A name no associated item of type `item` carries: is it still declared by + * something the scan does not show as a member? By a standard derive of the + * type, or by an impl for no named type (`impl Trait for T`), or + * possibly among more impls than are searched. */ +static bool rs_assoc_hidden(const rs_index_t *ix, int item, const char *name) { + rs_ent_t end = rs_alias_end(ix, item); + if (end.kind != RS_E_ITEM) { + return false; + } + const rs_item_t *it = &ix->items[end.idx]; + if (it->kind != 'S' && it->kind != 'E' && it->kind != 'U') { + return false; + } + return (it->derives && rs_derive_declares(it->derives, name)) || + it->impl_count > RS_IMPL_SCAN || cbm_ht_get(ix->blanket, name) != NULL; +} + +/* A reference that names nothing that was found. */ +static void rs_link_miss(const rs_link_t *q, const rs_set_t *owners, bool mismatch, + cbm_doclink_outcome_t *out) { + const char *last = q->segs[q->path.n - SKIP_ONE]; + bool hidden = false; + for (int i = 0; !mismatch && i < owners->n && !hidden; i++) { + hidden = owners->v[i].kind == RS_E_ITEM && rs_assoc_hidden(q->ix, owners->v[i].idx, last); + } + if (hidden) { + out->reason = CBM_DOCLINK_REASON_GRAPH_GAP; + } else if (!mismatch && q->single && rs_is_local(q, q->segs[0])) { + out->kind = CBM_DOCLINK_LOCAL; + } +} + +static void rs_link_decide(const rs_link_t *q, const cbm_gbuf_t *graph, + cbm_doclink_outcome_t *out) { + const rs_index_t *ix = q->ix; + rs_look_t lk = {.ix = ix, .assoc = true}; + rs_found_t found; + rs_set_t cand = {0}; + bool weak = false; + bool prim = q->single && RS_IN(RS_PRIMITIVES, q->segs[0]); + bool any_type = q->dis == CBM_DOCLINK_RS_DIS_NONE || q->dis == CBM_DOCLINK_RS_DIS_TYPE_NS; + rs_resolve_path(&lk, q->mod, &q->path, rs_dis_mask(q->dis), q->self, CBM_NOT_FOUND, &found); + if (found.ambiguous) { + out->reason = CBM_DOCLINK_REASON_AMBIGUOUS; + return; + } + rs_candidates(ix, &found, &cand, &weak); + if (q->dis == CBM_DOCLINK_RS_DIS_PRIM || (prim && (weak || rs_set_empty(&cand)))) { + /* a primitive type: rustdoc's fallback when the path names nothing */ + out->reason = prim ? CBM_DOCLINK_REASON_EXTERNAL : CBM_DOCLINK_REASON_MISSING; + return; + } + if (prim && any_type) { + out->reason = CBM_DOCLINK_REASON_AMBIGUOUS; /* the primitive, or what the name binds */ + return; + } + bool had = !rs_set_empty(&cand); + rs_filter_dis(ix, &cand, q->dis); + rs_trait_wins(ix, &cand); + if (cand.overflow || cand.n > SKIP_ONE) { + out->reason = CBM_DOCLINK_REASON_AMBIGUOUS; + return; + } + if (cand.n == 0) { + rs_link_miss(q, &found.owners, had, out); + return; + } + rs_outcome(q, graph, cand.v[0], out); + out->exact = !q->single || cand.v[0].via_import; + if (weak && cand.v[0].kind == RS_E_GAP && q->single && rs_is_local(q, q->segs[0])) { + out->kind = CBM_DOCLINK_LOCAL; + } +} + +/* One reference of run file `run_file`. */ +static void rs_resolve(const void *index, void *state, int run_file, const CBMDocLink *link, + const cbm_gbuf_t *graph, cbm_doclink_outcome_t *out) { + (void)state; /* Rust keeps no per-file working state */ + const rs_index_t *ix = (const rs_index_t *)index; + CBMDocLinkRsPath parsed; + out->kind = CBM_DOCLINK_UNRESOLVED; + out->reason = CBM_DOCLINK_REASON_MISSING; + out->target = NULL; + out->exact = false; + if (!ix || !link || !link->raw || run_file < 0 || run_file >= ix->run_count || + ix->run_to_file[run_file] < 0) { + return; + } + const rs_file_t *f = &ix->files[ix->run_to_file[run_file]]; + if (f->kind != RS_FILE_SOURCE) { + return; + } + if (cbm_doclink_rs_parse_path(link->raw, strlen(link->raw), false, &parsed) != + CBM_DOCLINK_RS_PATH_OK) { + out->reason = CBM_DOCLINK_REASON_UNPARSEABLE; + return; + } + rs_link_t q = {.ix = ix, + .f = f, + .link = link, + .dis = parsed.dis, + .file_level = (link->flags & CBM_DOCLINK_FLAG_FILE) != 0}; + if (!q.file_level && f->err_line && link->def_line >= f->err_line) { + /* the definition lies where the file no longer parses: its scope is not known */ + out->reason = CBM_DOCLINK_REASON_GRAPH_GAP; + return; + } + for (int i = 0; i < parsed.nsegs; i++) { + q.segs[i] = rs_plain(parsed.segs[i]); + } + q.path = (rs_path_t){.segs = q.segs, .n = parsed.nsegs, .global = parsed.global}; + q.single = parsed.nsegs == SKIP_ONE && !parsed.global && !rs_is_keyword(q.segs[0]); + q.mod = q.file_level ? f->mod_base : rs_mod_at(ix, f, link->def_line); + if (!q.file_level) { + q.self = rs_self_at(ix, f, link->def_line); + } + q.product = rs_source_product(&q); + rs_link_decide(&q, graph, out); + if (out->kind == CBM_DOCLINK_EDGE && !q.file_level && rs_unowned(ix, f, link)) { + out->kind = CBM_DOCLINK_UNRESOLVED; /* no node of its own to leave from */ + out->reason = CBM_DOCLINK_REASON_GRAPH_GAP; + out->target = NULL; + out->exact = false; + } +} + +/* ── Incremental scope rules ─────────────────────────────────────── */ + +/* The next line of a blob; false at the end. */ +static bool rs_delta_line(const char **cursor, const char **line, size_t *len) { + const char *p = *cursor; + if (!p || !*p) { + return false; + } + const char *nl = strchr(p, '\n'); + *line = p; + *len = nl ? (size_t)(nl - p) : strlen(p); + *cursor = nl ? nl + SKIP_ONE : p + *len; + return true; +} + +/* The name of an `M start kind name node` line whose member no path can reach + * under another name: a function, a constant, a type or a field (not a + * variant, which `use` can rename). The name must be one the incremental + * route can find in an unresolved reference: ASCII letters, digits and `_`. */ +static bool rs_delta_member(const char *line, size_t len, const char **name, size_t *nlen) { + enum { F_KIND = 2, F_NAME = 3 }; + const char *field[F_NAME + PAIR_LEN]; + int n = 0; + field[n++] = line; + for (size_t i = 0; i < len && n < F_NAME + PAIR_LEN; i++) { + if (line[i] == '\t') { + field[n++] = line + i + SKIP_ONE; + } + } + if (line[0] != 'M' || n != F_NAME + PAIR_LEN || field[F_NAME] - field[F_KIND] != PAIR_LEN || + !strchr("fctd", field[F_KIND][0])) { + return false; + } + const char *s = field[F_NAME]; + size_t sl = (size_t)(field[F_NAME + SKIP_ONE] - s) - SKIP_ONE; + if (sl > PAIR_LEN && s[0] == 'r' && s[1] == '#') { + s += PAIR_LEN; + sl -= PAIR_LEN; + } + for (size_t i = 0; i < sl; i++) { + if (!(isalnum((unsigned char)s[i]) || s[i] == '_')) { + return false; + } + } + *name = s; + *nlen = sl; + return sl > 0; +} + +/* Compare a changed file's stored and fresh scopes line by line, in order + * (both in the persisted form: no line numbers, no per-definition records). + * + * Equal scopes leave every other file's resolution alone: other files see a + * file only through these records. + * + * One difference does too: a member (function, constant, type, field of a + * type, trait or impl) the fresh scope no longer has. A member is reached + * only by a path that ends in its name, so a reference that depended on it + * either bound it (an edge into this file: re-resolved with the closure) or + * names it in its unresolved row: the name is reported. + * + * Everything else is GLOBAL: a new or changed line; a removed item, module, + * import or variant (`use` can bring any of them into another file under + * another name, so no name finds the references that depended on it); a + * changed order (same-named declarations own their node by order); and any + * change of a manifest. */ +static int rs_scope_delta(const char *stored, const char *fresh, cbm_doclink_name_fn removed, + void *ud) { + const char *sp = stored; + const char *fp = fresh; + const char *sl = NULL; + const char *fl = NULL; + size_t slen = 0; + size_t flen = 0; + bool hs = rs_delta_line(&sp, &sl, &slen); + bool hf = rs_delta_line(&fp, &fl, &flen); + while (hs || hf) { + if (hs && hf && slen == flen && memcmp(sl, fl, slen) == 0) { + hs = rs_delta_line(&sp, &sl, &slen); + hf = rs_delta_line(&fp, &fl, &flen); + continue; + } + const char *name = NULL; + size_t nlen = 0; + if (!hs || !rs_delta_member(sl, slen, &name, &nlen)) { + return CBM_DOCLINK_DELTA_GLOBAL; + } + if (!removed || !removed(ud, name, nlen)) { + return CBM_NOT_FOUND; + } + hs = rs_delta_line(&sp, &sl, &slen); + } + return CBM_DOCLINK_DELTA_LOCAL; +} + +const cbm_doclink_resolver_t cbm_doclink_rs_resolver = { + .langs = {CBM_LANG_RUST, CBM_LANG_TOML}, + .lang_count = 2, + .scope_tag = CBM_DOCLINK_RS_SCOPE_TAG, + .build = rs_build, + .destroy = rs_destroy, + .resolve = rs_resolve, + .scope_delta = rs_scope_delta, +}; diff --git a/tests/test_doc_mentions_helpers.h b/tests/test_doc_mentions_helpers.h index 1053912c9..e37ee9368 100644 --- a/tests/test_doc_mentions_helpers.h +++ b/tests/test_doc_mentions_helpers.h @@ -438,7 +438,8 @@ static const int DM_HELD_FAMILIES[] = { CBM_DOCLINK_JAVA_SEE, CBM_DOCLINK_JAVA_THROWS, CBM_DOCLINK_JAVA_VALUE, CBM_DOCLINK_KT_KDOC_LINK, CBM_DOCLINK_KT_SEE, CBM_DOCLINK_KT_THROWS, - CBM_DOCLINK_KT_SAMPLE, + CBM_DOCLINK_KT_SAMPLE, CBM_DOCLINK_RS_INTRA_DOC, + CBM_DOCLINK_RS_LINK_DEF, }; static inline void dm_ship_held_families(void) { diff --git a/tests/test_doc_mentions_rs.c b/tests/test_doc_mentions_rs.c new file mode 100644 index 000000000..95ec05f1a --- /dev/null +++ b/tests/test_doc_mentions_rs.c @@ -0,0 +1,1734 @@ +/* + * test_doc_mentions_rs.c — Rust intra-doc links -> MENTIONS edges. + * + * Extraction (internal/cbm/doclink_rs.c): which text is a link, the path + * grammar, the scope blob of a source file and of a Cargo manifest. + * Resolution (src/pipeline/doc_links_rs.c): every pipeline test indexes a real + * fixture through cbm_pipeline_run and reads the published database (helpers: + * test_doc_mentions_helpers.h). + */ +#include "../src/foundation/compat.h" +#include "test_framework.h" +#include "test_helpers.h" +#include "test_doc_mentions_helpers.h" + +/* ── extraction: which text is a link ────────────────────────────── */ + +/* The token written as `raw` on `line`; NULL when there is none. */ +static const CBMDocLink *rs_token_at(const CBMFileResult *r, const char *raw, uint32_t line) { + for (int i = 0; i < r->doc_links.count; i++) { + const CBMDocLink *l = &r->doc_links.items[i]; + if (l->line == line && strcmp(l->raw, raw) == 0) { + return l; + } + } + return NULL; +} + +static int rs_tokens_on(const CBMFileResult *r, uint32_t line) { + int n = 0; + for (int i = 0; i < r->doc_links.count; i++) { + n += r->doc_links.items[i].line == line; + } + return n; +} + +TEST(doc_mentions_rs_tokens) { + const char *src = + "//! File doc with [`Alpha`] here.\n" /* 1 */ + "#![allow(dead_code)]\n" /* 2 */ + "//! After an attribute: [beta::Gamma].\n" /* 3 */ + "\n" /* 4 */ + "/// Inline [the text](alpha::Path) and shortcut [`Short`] and [plain::path].\n" /* 5 */ + "/// Collapsed [`Coll`][] and reference [see this][lbl] and [lbl].\n" /* 6 */ + "/// URL [site](https://example.org/a), file [guide](../guide.md), [top](#top).\n" /* 7 */ + "/// Fragment [`Frag#method.x`], image ![alt](img::Path), footnote [^1].\n" /* 8 */ + "/// Code `[NotALink]`, escaped \\[NotThis\\], prose [two words].\n" /* 9 */ + "/// Kinds [struct@Dis], [call()], [mac!], generic [Vec].\n" /* 10 */ + "/// - [ ] task\n" /* 11 */ + "/// - [x] done\n" /* 12 */ + "/// [!NOTE]\n" /* 13 */ + "/// Backticked [`weird(x)`] and nested [outer [`Inner`] text].\n" /* 14 */ + "///\n" /* 15 */ + "/// ```\n" /* 16 */ + "/// [InFence]\n" /* 17 */ + "/// ```\n" /* 18 */ + "///\n" /* 19 */ + "/// [InIndentedCode]\n" /* 20 */ + "///\n" /* 21 */ + "/// \n" /* 22 */ + "/// [lbl]: defined::Target\n" /* 23 */ + "/// [unused]: https://example.org/unused\n" /* 24 */ + "pub struct Documented;\n" /* 25 */ + "\n" /* 26 */ + "//// [FourSlashes]\n" /* 27 */ + "// [PlainComment]\n" /* 28 */ + "pub struct NotDocumented;\n" /* 29 */ + "\n" /* 30 */ + "/** Block doc with [`InBlock`]\n" /* 31 */ + " * and a second line [second::Line].\n" /* 32 */ + " */\n" /* 33 */ + "pub fn block_documented() {}\n"; /* 34 */ + CBMFileResult *r = dm_extract(src, CBM_LANG_RUST, "src/lib.rs"); + ASSERT_NOT_NULL(r); + + /* the file's own doc: marked, with the real line of every doc line (the + * attribute between them is no doc line) */ + const CBMDocLink *alpha = rs_token_at(r, "Alpha", 1); + ASSERT_NOT_NULL(alpha); + ASSERT_EQ(alpha->syntax, CBM_DOCLINK_RS_INTRA_DOC); + ASSERT_TRUE((alpha->flags & CBM_DOCLINK_FLAG_FILE) != 0); + const CBMDocLink *gamma = rs_token_at(r, "beta::Gamma", 3); + ASSERT_NOT_NULL(gamma); + ASSERT_TRUE((gamma->flags & CBM_DOCLINK_FLAG_FILE) != 0); + + /* an item's doc: inline, shortcut (backticks go), plain shortcut */ + const CBMDocLink *inl = rs_token_at(r, "alpha::Path", 5); + ASSERT_NOT_NULL(inl); + ASSERT_EQ(inl->syntax, CBM_DOCLINK_RS_INTRA_DOC); + ASSERT_EQ(inl->flags & CBM_DOCLINK_FLAG_FILE, 0); + ASSERT_STR_EQ(inl->source_qn, "p.src.lib.Documented"); + ASSERT_EQ(inl->def_line, 25); + ASSERT_NOT_NULL(rs_token_at(r, "Short", 5)); + ASSERT_NOT_NULL(rs_token_at(r, "plain::path", 5)); + ASSERT_EQ(rs_tokens_on(r, 5), 3); + + /* collapsed; a label with a definition takes the definition's destination */ + ASSERT_NOT_NULL(rs_token_at(r, "Coll", 6)); + const CBMDocLink *def = rs_token_at(r, "defined::Target", 6); + ASSERT_NOT_NULL(def); + ASSERT_EQ(def->syntax, CBM_DOCLINK_RS_LINK_DEF); + ASSERT_EQ(dm_count_tokens(r, "defined::Target"), 2); /* [see this][lbl] and [lbl] */ + ASSERT_EQ(rs_tokens_on(r, 6), 3); + + /* a URL is the core's href family; a relative file and a page anchor are + * not harvested */ + const CBMDocLink *url = rs_token_at(r, "https://example.org/a", 7); + ASSERT_NOT_NULL(url); + ASSERT_EQ(url->syntax, CBM_DOCLINK_HREF); + ASSERT_EQ(rs_tokens_on(r, 7), 1); + + /* the fragment is cut; an image and a footnote are no links */ + ASSERT_NOT_NULL(rs_token_at(r, "Frag", 8)); + ASSERT_EQ(rs_tokens_on(r, 8), 1); + + /* a code span, escaped brackets and prose hold no link */ + ASSERT_EQ(rs_tokens_on(r, 9), 0); + + /* disambiguators, suffixes and generics stay in the token */ + ASSERT_NOT_NULL(rs_token_at(r, "struct@Dis", 10)); + ASSERT_NOT_NULL(rs_token_at(r, "call()", 10)); + ASSERT_NOT_NULL(rs_token_at(r, "mac!", 10)); + ASSERT_NOT_NULL(rs_token_at(r, "Vec", 10)); + + /* task markers and an admonition are no links */ + ASSERT_EQ(rs_tokens_on(r, 11), 0); + ASSERT_EQ(rs_tokens_on(r, 12), 0); + ASSERT_EQ(rs_tokens_on(r, 13), 0); + + /* backticks make rustdoc keep what it would otherwise drop as prose; of + * nested brackets the inner pair is the link */ + ASSERT_NOT_NULL(rs_token_at(r, "weird(x)", 14)); + ASSERT_NOT_NULL(rs_token_at(r, "Inner", 14)); + ASSERT_EQ(rs_tokens_on(r, 14), 2); + + /* code blocks, an HTML comment and the definitions themselves */ + ASSERT_EQ(rs_tokens_on(r, 17), 0); + ASSERT_EQ(rs_tokens_on(r, 20), 0); + ASSERT_EQ(rs_tokens_on(r, 22), 0); + ASSERT_EQ(rs_tokens_on(r, 23), 0); + ASSERT_EQ(rs_tokens_on(r, 24), 0); + + /* `////` and `//` are comments, not documentation */ + ASSERT_NULL(dm_find_token(r, "FourSlashes")); + ASSERT_NULL(dm_find_token(r, "PlainComment")); + + /* a block doc, its decoration removed */ + const CBMDocLink *blk = rs_token_at(r, "InBlock", 31); + ASSERT_NOT_NULL(blk); + ASSERT_STR_EQ(blk->source_qn, "p.src.lib.block_documented"); + ASSERT_NOT_NULL(rs_token_at(r, "second::Line", 32)); + cbm_free_result(r); + PASS(); +} + +/* The doc of the first item is the item's, never the file's. */ +TEST(doc_mentions_rs_item_doc_is_not_file_doc) { + const char *src = "/// Outer doc of the first item: [`Target`].\n" + "pub struct First;\n"; + CBMFileResult *r = dm_extract(src, CBM_LANG_RUST, "src/first.rs"); + ASSERT_NOT_NULL(r); + const CBMDocLink *t = dm_find_token(r, "Target"); + ASSERT_NOT_NULL(t); + ASSERT_EQ(t->flags & CBM_DOCLINK_FLAG_FILE, 0); + ASSERT_STR_EQ(t->source_qn, "p.src.first.First"); + ASSERT_EQ(r->doc_links.count, 1); + cbm_free_result(r); + PASS(); +} + +/* Text nobody controls: nothing here may crash, hang or grow without bound. */ +TEST(doc_mentions_rs_untrusted_text) { + enum { DEEP = 20000, LONG = 1 << 20, LABEL = 700 }; + /* a megabyte destination, a megabyte label, a megabyte of opening brackets, + * a megabyte of backticks, and a label longer than a token carries */ + char *big = malloc((size_t)LONG * 4 + 4096); + ASSERT_NOT_NULL(big); + size_t w = (size_t)snprintf(big, 64, "/// [dest]("); + memset(big + w, 'a', LONG); + w += LONG; + w += (size_t)snprintf(big + w, 64, ")\n/// [`"); + memset(big + w, 'b', LONG); + w += LONG; + w += (size_t)snprintf(big + w, 64, "`]\n/// "); + memset(big + w, '[', LONG); + w += LONG; + w += (size_t)snprintf(big + w, 64, "\n/// ticks "); /* not at the line start: no fence */ + memset(big + w, '`', LONG); + w += LONG; + w += (size_t)snprintf(big + w, 64, "\n///\n/// [`"); + memset(big + w, 'c', LABEL); + w += LABEL; + snprintf(big + w, 64, "`]\npub struct Big;\n"); + CBMFileResult *r = dm_extract(big, CBM_LANG_RUST, "src/big.rs"); + ASSERT_NOT_NULL(r); + /* an over-long destination is one token, cut and marked: it can never + * parse, so no truncated prefix is ever resolved. A label of more than + * 999 characters is no link label at all (CommonMark). */ + ASSERT_EQ(r->doc_links.count, 2); + for (int i = 0; i < r->doc_links.count; i++) { + const char *raw = r->doc_links.items[i].raw; + size_t rl = strlen(raw); + CBMDocLinkRsPath path; + ASSERT_TRUE(rl < CBM_DOCLINK_RS_RAW_CAP); + ASSERT_TRUE(raw[0] == (i == 0 ? 'a' : 'c')); + ASSERT_STR_EQ(raw + rl - 3, "..."); + ASSERT_EQ(cbm_doclink_rs_parse_path(raw, rl, false, &path), CBM_DOCLINK_RS_PATH_MALFORMED); + } + cbm_free_result(r); + + /* deeply nested links, lists and block quotes */ + size_t pos = (size_t)snprintf(big, 64, "/// "); + for (int i = 0; i < DEEP; i++) { + big[pos++] = '['; + } + pos += (size_t)snprintf(big + pos, 64, "`Core`"); + for (int i = 0; i < DEEP; i++) { + big[pos++] = ']'; + } + pos += (size_t)snprintf(big + pos, 64, "\n/// "); + for (int i = 0; i < DEEP; i++) { + big[pos++] = '>'; + } + pos += (size_t)snprintf(big + pos, 64, " [`Quoted`]\n/// "); + for (int i = 0; i < DEEP; i++) { + big[pos++] = '-'; + big[pos++] = ' '; + } + snprintf(big + pos, 64, "[`Listed`]\npub struct Nested;\n"); + r = dm_extract(big, CBM_LANG_RUST, "src/nested.rs"); + ASSERT_NOT_NULL(r); + ASSERT_NOT_NULL(dm_find_token(r, "Core")); + cbm_free_result(r); + + /* bytes that are no UTF-8, and a NUL inside the doc */ + const char bad[] = "/// [`Caf\xC3`] [`\xFF\xFE`] [`Ok`]\n/// [`Af\0ter`]\npub struct Bytes;\n"; + r = cbm_extract_file(bad, (int)sizeof(bad) - 1, CBM_LANG_RUST, "p", "src/bytes.rs", 0, NULL, + NULL); + ASSERT_NOT_NULL(r); + ASSERT_NOT_NULL(dm_find_token(r, "Ok")); + cbm_free_result(r); + free(big); + PASS(); +} + +/* ── extraction: the path grammar (rustdoc's preprocess_link) ────── */ + +TEST(doc_mentions_rs_parse_path) { + /* where the grammar decides what is harvested: text in plain brackets + * that can be no path is prose (no token); the same text in backticks, + * and a path that is broken, is a link rustdoc reports (a token, which + * the resolver turns into an `unparseable` row) */ + CBMFileResult *r = dm_extract("/// [a+b] [`c+d`] [bogus@Foo] [Vec<] [x y] [`x y`]\n" + "pub struct Grammar;\n", + CBM_LANG_RUST, "src/g.rs"); + ASSERT_NOT_NULL(r); + ASSERT_NULL(dm_find_token(r, "a+b")); + ASSERT_NOT_NULL(dm_find_token(r, "c+d")); + ASSERT_NOT_NULL(dm_find_token(r, "bogus@Foo")); + ASSERT_NOT_NULL(dm_find_token(r, "Vec<")); + ASSERT_NULL(dm_find_token(r, "x y")); + ASSERT_EQ(r->doc_links.count, 3); + cbm_free_result(r); + + struct { + const char *raw; + bool lenient; + CBMDocLinkRsPathStatus want; + int nsegs; + const char *last; /* last segment */ + int dis; + bool global; + } cases[] = { + {"Foo", false, CBM_DOCLINK_RS_PATH_OK, 1, "Foo", CBM_DOCLINK_RS_DIS_NONE, false}, + {"a::b::C", false, CBM_DOCLINK_RS_PATH_OK, 3, "C", CBM_DOCLINK_RS_DIS_NONE, false}, + {"::std::fmt", false, CBM_DOCLINK_RS_PATH_OK, 2, "fmt", CBM_DOCLINK_RS_DIS_NONE, true}, + {" crate::x ", false, CBM_DOCLINK_RS_PATH_OK, 2, "x", CBM_DOCLINK_RS_DIS_NONE, false}, + /* generics are stripped, as rustdoc does */ + {"Vec::new", false, CBM_DOCLINK_RS_PATH_OK, 2, "new", CBM_DOCLINK_RS_DIS_NONE, false}, + {"HashMap", false, CBM_DOCLINK_RS_PATH_OK, 1, "HashMap", CBM_DOCLINK_RS_DIS_NONE, + false}, + /* disambiguators and suffixes */ + {"struct@Foo", false, CBM_DOCLINK_RS_PATH_OK, 1, "Foo", CBM_DOCLINK_RS_DIS_STRUCT, false}, + {"fn@a::run", false, CBM_DOCLINK_RS_PATH_OK, 2, "run", CBM_DOCLINK_RS_DIS_FN, false}, + {"method@T::m", false, CBM_DOCLINK_RS_PATH_OK, 2, "m", CBM_DOCLINK_RS_DIS_FN, false}, + {"mod@x", false, CBM_DOCLINK_RS_PATH_OK, 1, "x", CBM_DOCLINK_RS_DIS_MOD, false}, + {"prim@str", false, CBM_DOCLINK_RS_PATH_OK, 1, "str", CBM_DOCLINK_RS_DIS_PRIM, false}, + {"type@X", false, CBM_DOCLINK_RS_PATH_OK, 1, "X", CBM_DOCLINK_RS_DIS_TYPE_NS, false}, + {"run()", false, CBM_DOCLINK_RS_PATH_OK, 1, "run", CBM_DOCLINK_RS_DIS_FN, false}, + {"vec!", false, CBM_DOCLINK_RS_PATH_OK, 1, "vec", CBM_DOCLINK_RS_DIS_MACRO_BANG, false}, + {"vec![]", false, CBM_DOCLINK_RS_PATH_OK, 1, "vec", CBM_DOCLINK_RS_DIS_MACRO_BANG, false}, + /* a raw identifier, a tuple field, a name that is no ASCII */ + {"r#type", false, CBM_DOCLINK_RS_PATH_OK, 1, "r#type", CBM_DOCLINK_RS_DIS_NONE, false}, + {"Pair::0", false, CBM_DOCLINK_RS_PATH_OK, 2, "0", CBM_DOCLINK_RS_DIS_NONE, false}, + {"Gr\xC3\xB6\xC3\x9F" + "e", + false, CBM_DOCLINK_RS_PATH_OK, 1, + "Gr\xC3\xB6\xC3\x9F" + "e", + CBM_DOCLINK_RS_DIS_NONE, false}, + /* rustdoc's spellings of primitive types that are no identifiers */ + {"!", false, CBM_DOCLINK_RS_PATH_OK, 1, "!", CBM_DOCLINK_RS_DIS_NONE, false}, + {"&mut", false, CBM_DOCLINK_RS_PATH_OK, 1, "&mut", CBM_DOCLINK_RS_DIS_NONE, false}, + {"*const", false, CBM_DOCLINK_RS_PATH_OK, 1, "*const", CBM_DOCLINK_RS_DIS_NONE, false}, + /* prose: no link at all */ + {"two words", true, CBM_DOCLINK_RS_PATH_NOT_LINK, 0, NULL, 0, false}, + {"a+b", true, CBM_DOCLINK_RS_PATH_NOT_LINK, 0, NULL, 0, false}, + {"!NOTE", false, CBM_DOCLINK_RS_PATH_NOT_LINK, 0, NULL, 0, false}, + {"", false, CBM_DOCLINK_RS_PATH_NOT_LINK, 0, NULL, 0, false}, + /* a link, but no path */ + {"a+b", false, CBM_DOCLINK_RS_PATH_MALFORMED, 0, NULL, 0, false}, + {"weird(x)", false, CBM_DOCLINK_RS_PATH_MALFORMED, 0, NULL, 0, false}, + {"Vec<", false, CBM_DOCLINK_RS_PATH_MALFORMED, 0, NULL, 0, false}, + {"a:b", false, CBM_DOCLINK_RS_PATH_MALFORMED, 0, NULL, 0, false}, + {"a::", false, CBM_DOCLINK_RS_PATH_MALFORMED, 0, NULL, 0, false}, + {"bogus@Foo", false, CBM_DOCLINK_RS_PATH_MALFORMED, 0, NULL, 0, false}, + {"::m", false, CBM_DOCLINK_RS_PATH_MALFORMED, 0, NULL, 0, false}, + }; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + CBMDocLinkRsPath p; + CBMDocLinkRsPathStatus got = + cbm_doclink_rs_parse_path(cases[i].raw, strlen(cases[i].raw), cases[i].lenient, &p); + bool ok = got == cases[i].want; + if (ok && got == CBM_DOCLINK_RS_PATH_OK) { + ok = p.nsegs == cases[i].nsegs && p.dis == cases[i].dis && + p.global == cases[i].global && strcmp(p.segs[p.nsegs - 1], cases[i].last) == 0; + } + if (!ok) { + printf(" [%s]: status %d segs %d dis %d\n", cases[i].raw, (int)got, p.nsegs, p.dis); + FAIL("path grammar"); + } + } + /* more segments than are carried, and a segment longer than is carried */ + char many[256] = "a"; + for (int i = 0; i < CBM_DOCLINK_RS_MAX_SEGS; i++) { + strcat(many, "::a"); + } + CBMDocLinkRsPath p; + ASSERT_EQ(cbm_doclink_rs_parse_path(many, strlen(many), false, &p), + CBM_DOCLINK_RS_PATH_MALFORMED); + char wide[CBM_DOCLINK_RS_SEG_CAP + 8]; + memset(wide, 'w', sizeof(wide) - 1); + wide[sizeof(wide) - 1] = '\0'; + ASSERT_EQ(cbm_doclink_rs_parse_path(wide, strlen(wide), false, &p), + CBM_DOCLINK_RS_PATH_MALFORMED); + PASS(); +} + +/* ── extraction: the scope blob ──────────────────────────────────── */ + +/* True when the blob holds exactly this line. */ +static bool rs_blob_has(const char *blob, const char *line) { + size_t n = strlen(line); + for (const char *p = blob; p && *p;) { + const char *nl = strchr(p, '\n'); + size_t len = nl ? (size_t)(nl - p) : strlen(p); + if (len == n && memcmp(p, line, n) == 0) { + return true; + } + p = nl ? nl + 1 : NULL; + } + return false; +} + +TEST(doc_mentions_rs_scope_blob) { + const char *src = "use std::fmt::{self, Debug as Dbg};\n" /* 1 */ + "pub use crate::shapes::*;\n" /* 2 */ + "extern crate alloc as heap;\n" /* 3 */ + "mod shapes;\n" /* 4 */ + "#[cfg(test)]\n" /* 5 */ + "mod checks;\n" /* 6 */ + "#[path = \"odd/place.rs\"]\n" /* 7 */ + "pub(crate) mod odd;\n" /* 8 */ + "/// A widget: [`Gadget`], [`size`], [`T`].\n" /* 9 */ + "#[derive(Clone, serde::Serialize)]\n" /* 10 */ + "pub struct Widget {\n" /* 11 */ + " pub size: u32,\n" /* 12 */ + " inner: T,\n" /* 13 */ + "}\n" /* 14 */ + "pub enum Kind { Round, Flat }\n" /* 15 */ + "pub struct Pair(pub u32, pub u32);\n" /* 16 */ + "pub trait Draw {\n" /* 17 */ + " const SIDES: u32;\n" /* 18 */ + " type Out;\n" /* 19 */ + " fn draw(&self);\n" /* 20 */ + "}\n" /* 21 */ + "impl Widget {\n" /* 22 */ + " pub const UNIT: u32 = 1;\n" /* 23 */ + " /// Resize by [`factor`].\n" /* 24 */ + " pub fn resize(&mut self, factor: u32) {}\n" /* 25 */ + "}\n" /* 26 */ + "impl Draw for Widget {\n" /* 27 */ + " const SIDES: u32 = 0;\n" /* 28 */ + " type Out = ();\n" /* 29 */ + " fn draw(&self) {}\n" /* 30 */ + "}\n" /* 31 */ + "impl Draw for T {\n" /* 32 */ + " const SIDES: u32 = 0;\n" /* 33 */ + " type Out = ();\n" /* 34 */ + " fn draw(&self) {}\n" /* 35 */ + "}\n" /* 36 */ + "pub type Alias = shapes::Circle;\n" /* 37 */ + "pub const LIMIT: u32 = 3;\n" /* 38 */ + "pub static NAME: &str = \"\";\n" /* 39 */ + "pub(super) fn helper() {}\n" /* 40 */ + "#[macro_export]\n" /* 41 */ + "macro_rules! declare {\n" /* 42 */ + " ($n:ident) => { pub struct Knob; pub struct $n; };\n" /* 43 */ + "}\n" /* 44 */ + "declare!(Dial);\n" /* 45 */ + "lazy_static! { pub static ref TABLE: u32 = 0; }\n" /* 46 */ + "#[cfg(test)]\n" /* 47 */ + "mod tests {\n" /* 48 */ + " use super::*;\n" /* 49 */ + " fn helper() {}\n" /* 50 */ + "}\n"; /* 51 */ + CBMFileResult *r = dm_extract(src, CBM_LANG_RUST, "src/lib.rs"); + ASSERT_NOT_NULL(r); + ASSERT_NOT_NULL(r->doc_scope); + const char *b = r->doc_scope; + ASSERT_TRUE(strncmp(b, CBM_DOCLINK_RS_SCOPE_TAG "\n", 4) == 0); + /* imports: a `self` import, a renamed one, a glob, an extern crate */ + ASSERT_TRUE(rs_blob_has(b, "U\t0\t-\ts\tfmt\tstd::fmt")); + ASSERT_TRUE(rs_blob_has(b, "U\t0\t-\tb\tDbg\tstd::fmt::Debug")); + ASSERT_TRUE(rs_blob_has(b, "U\t0\tp\tg\t-\tcrate::shapes")); + ASSERT_TRUE(rs_blob_has(b, "U\t0\t-\tx\theap\talloc")); + /* `mod x;`: plain, test-only, with a path */ + ASSERT_TRUE(rs_blob_has(b, "D\t0\t-\tshapes\t-")); + ASSERT_TRUE(rs_blob_has(b, "D\t0\t-t\tchecks\t-")); + ASSERT_TRUE(rs_blob_has(b, "D\t0\tc\todd\todd/place.rs")); + /* items with their node, members with theirs, derive names */ + ASSERT_TRUE(rs_blob_has(b, "T\t0\t11\t14\tp\tS\tWidget\tS=\tClone,Serialize")); + ASSERT_TRUE(rs_blob_has(b, "M\t12\td\tsize\tD:Widget.size")); + ASSERT_TRUE(rs_blob_has(b, "M\t13\td\tinner\tD:Widget.inner")); + ASSERT_TRUE(rs_blob_has(b, "T\t0\t15\t15\tp\tE\tKind\tE=\t-")); + ASSERT_TRUE(rs_blob_has(b, "M\t15\tv\tRound\t!")); /* a variant has no node */ + ASSERT_TRUE(rs_blob_has(b, "M\t0\td\t0\t!")); /* nor has a tuple field */ + ASSERT_TRUE(rs_blob_has(b, "T\t0\t17\t21\tp\tR\tDraw\tI=\t-")); + ASSERT_TRUE(rs_blob_has(b, "M\t19\tt\tOut\t!")); + ASSERT_TRUE(rs_blob_has(b, "M\t20\tf\tdraw\tM:Draw.draw")); + /* impls: inherent, of a trait, and for no named type */ + ASSERT_TRUE(rs_blob_has(b, "I\t0\t22\t26\tWidget\t-")); + ASSERT_TRUE(rs_blob_has(b, "M\t23\tc\tUNIT\t!")); /* an impl's constant has no node */ + ASSERT_TRUE(rs_blob_has(b, "M\t25\tf\tresize\tM:Widget.resize")); + ASSERT_TRUE(rs_blob_has(b, "I\t0\t27\t31\tWidget\tDraw")); + ASSERT_TRUE(rs_blob_has(b, "I\t0\t32\t36\t?\tDraw")); + ASSERT_TRUE(rs_blob_has(b, "T\t0\t37\t37\tp\tA\tAlias\tT=\tshapes::Circle")); + ASSERT_TRUE(rs_blob_has(b, "T\t0\t38\t38\tp\tC\tLIMIT\tV=\t-")); + ASSERT_TRUE(rs_blob_has(b, "T\t0\t39\t39\tp\tG\tNAME\tV=\t-")); + /* the module-level helper lost its node to the one in `mod tests` (the + * extractor flattens inline modules) */ + ASSERT_TRUE(rs_blob_has(b, "T\t0\t40\t40\ts\tF\thelper\t!\t-")); + ASSERT_TRUE(rs_blob_has(b, "T\t1\t50\t50\t-\tF\thelper\tF=\t-")); + ASSERT_TRUE(rs_blob_has(b, "N\t1\t0\t48\t51\t-t\ttests")); + ASSERT_TRUE(rs_blob_has(b, "U\t1\t-\tg\t-\tsuper")); + /* macros: an exported definition and what its body declares by name (the + * metavariable is no name); the invocation; the name written at it; a + * `static ref` of a macro body */ + ASSERT_TRUE(rs_blob_has(b, "T\t0\t42\t44\t-e\tM\tdeclare\t!\t-")); + ASSERT_TRUE(rs_blob_has(b, "W\tdeclare\tKnob")); + ASSERT_TRUE(rs_blob_has(b, "V\t0\tdeclare")); + ASSERT_TRUE(rs_blob_has(b, "Q\t0\tDial")); + ASSERT_TRUE(rs_blob_has(b, "V\t0\tlazy_static")); + ASSERT_TRUE(rs_blob_has(b, "Q\t0\tTABLE")); + /* the names local to a documented definition that carries references */ + ASSERT_TRUE(rs_blob_has(b, "P\t11\tT")); + ASSERT_TRUE(rs_blob_has(b, "P\t25\tT,factor")); + ASSERT_FALSE(rs_blob_has(b, "B")); + + /* the persisted form: no line numbers, no per-definition records */ + char *portable = cbm_doclink_portable_scope(b); + ASSERT_NOT_NULL(portable); + ASSERT_TRUE(rs_blob_has(portable, "T\t0\t0\t0\tp\tS\tWidget\tS=\tClone,Serialize")); + ASSERT_TRUE(rs_blob_has(portable, "M\t0\tf\tresize\tM:Widget.resize")); + ASSERT_TRUE(rs_blob_has(portable, "N\t1\t0\t0\t0\t-t\ttests")); + ASSERT_NULL(strstr(portable, "\nP\t")); + ASSERT_NULL(strstr(portable, "\nZ\t")); + cbm_free(CBM_MEM_CLASS_OTHER, portable); + cbm_free_result(r); + PASS(); +} + +/* A parse error: what follows it has no scope, unless the error stayed inside + * a function body with its delimiters paired. */ +TEST(doc_mentions_rs_scope_parse_errors) { + /* syntax the grammar does not know, inside a body: the items around it + * are where the tree puts them */ + const char *contained = "pub fn odd() {\n" + " let x = 1 @@ 2;\n" + "}\n" + "pub struct After;\n"; + CBMFileResult *r = dm_extract(contained, CBM_LANG_RUST, "src/a.rs"); + ASSERT_NOT_NULL(r); + ASSERT_NOT_NULL(r->doc_scope); + ASSERT_TRUE(rs_blob_has(r->doc_scope, "T\t0\t4\t4\tp\tS\tAfter\tS=\t-")); + ASSERT_FALSE(rs_blob_has(r->doc_scope, "B")); + ASSERT_NULL(strstr(r->doc_scope, "\nE\t")); + cbm_free_result(r); + + /* an unpaired brace in a body can move every later item boundary: the + * parser gave up on the whole file, and nothing of it is trusted */ + const char *open = "pub struct Before;\n" + "pub fn half() {\n" + " if true {\n" + "}\n" + "pub struct Lost;\n"; + r = dm_extract(open, CBM_LANG_RUST, "src/b.rs"); + ASSERT_NOT_NULL(r); + ASSERT_NOT_NULL(r->doc_scope); + ASSERT_TRUE(rs_blob_has(r->doc_scope, "E\t1")); + ASSERT_TRUE(rs_blob_has(r->doc_scope, "B")); + ASSERT_NULL(strstr(r->doc_scope, "\nT\t")); + cbm_free_result(r); + + /* an error between items: what stands before it keeps its scope, what + * follows is only a name the file may declare somewhere */ + const char *between = "pub struct One;\n" + "pub struct {\n" + "pub struct Two;\n"; + r = dm_extract(between, CBM_LANG_RUST, "src/c.rs"); + ASSERT_NOT_NULL(r); + ASSERT_NOT_NULL(r->doc_scope); + ASSERT_TRUE(rs_blob_has(r->doc_scope, "T\t0\t1\t1\tp\tS\tOne\tS=\t-")); + ASSERT_TRUE(rs_blob_has(r->doc_scope, "E\t2")); + ASSERT_TRUE(rs_blob_has(r->doc_scope, "Q\t*\tTwo")); + ASSERT_TRUE(rs_blob_has(r->doc_scope, "B")); + cbm_free_result(r); + PASS(); +} + +TEST(doc_mentions_rs_manifest_blob) { + const char *toml = "[package]\n" + "name = \"acme-core\"\n" + "edition.workspace = true\n" + "build = \"gen/build.rs\"\n" + "autobins = false\n" + "\n" + "[lib]\n" + "name = \"acme\"\n" + "path = \"lib/root.rs\"\n" + "\n" + "[[bin]]\n" + "name = \"tool\"\n" + "path = \"cli/tool.rs\"\n" + "\n" + "[dependencies]\n" + "serde = \"1\"\n" + "util = { path = \"../util\" }\n" + "alias = { package = \"real-name\", path = \"../real\" }\n" + "shared.workspace = true\n" + "\n" + "[dev-dependencies.helper]\n" + "path = \"../helper\"\n" + "\n" + "[target.'cfg(unix)'.dependencies]\n" + "nix = \"0.29\"\n" + "\n" + "[workspace]\n" + "members = [\"crates/*\"]\n" + "\n" + "[workspace.package]\n" + "edition = \"2021\"\n" + "\n" + "[workspace.dependencies]\n" + "shared = { path = \"crates/shared\" }\n"; + CBMFileResult *r = dm_extract(toml, CBM_LANG_TOML, "crates/core/Cargo.toml"); + ASSERT_NOT_NULL(r); + ASSERT_NOT_NULL(r->doc_scope); + const char *b = r->doc_scope; + ASSERT_TRUE(strncmp(b, CBM_DOCLINK_RS_SCOPE_TAG "\nC\n", 6) == 0); + ASSERT_TRUE(rs_blob_has(b, "K\tpackage\tname\tacme-core")); + ASSERT_TRUE(rs_blob_has(b, "K\tpackage\tedition.workspace\ttrue")); + ASSERT_TRUE(rs_blob_has(b, "K\tpackage\tbuild\tgen/build.rs")); + ASSERT_TRUE(rs_blob_has(b, "K\tpackage\tautobins\tfalse")); + ASSERT_TRUE(rs_blob_has(b, "K\tlib\tname\tacme")); + ASSERT_TRUE(rs_blob_has(b, "K\tlib\tpath\tlib/root.rs")); + ASSERT_TRUE(rs_blob_has(b, "A\tbin")); + ASSERT_TRUE(rs_blob_has(b, "a\tpath\tcli/tool.rs")); + ASSERT_TRUE(rs_blob_has(b, "X\td\tserde\t-\t-")); + ASSERT_TRUE(rs_blob_has(b, "X\td\tutil\tpath\t../util")); + ASSERT_TRUE(rs_blob_has(b, "X\td\talias\tpackage\treal-name")); + ASSERT_TRUE(rs_blob_has(b, "X\td\talias\tpath\t../real")); + ASSERT_TRUE(rs_blob_has(b, "X\td\tshared\tworkspace\ttrue")); + ASSERT_TRUE(rs_blob_has(b, "X\tv\thelper\tpath\t../helper")); + ASSERT_TRUE(rs_blob_has(b, "X\td\tnix\t-\t-")); + ASSERT_TRUE(rs_blob_has(b, "W")); + ASSERT_TRUE(rs_blob_has(b, "K\tworkspace.package\tedition\t2021")); + ASSERT_TRUE(rs_blob_has(b, "X\tw\tshared\tpath\tcrates/shared")); + cbm_free_result(r); + + /* any other TOML file has no scope */ + r = dm_extract("[package]\nname = \"x\"\n", CBM_LANG_TOML, "config/settings.toml"); + ASSERT_NOT_NULL(r); + ASSERT_NULL(r->doc_scope); + cbm_free_result(r); + PASS(); +} + +/* ── resolution: the fixture workspace ───────────────────────────── */ + +typedef struct { + const char *path; + const char *text; +} rs_fixture_file_t; + +static void rs_write_files(const char *root, const rs_fixture_file_t *files, size_t n) { + for (size_t i = 0; i < n; i++) { + th_write_file(TH_PATH(root, files[i].path), files[i].text); + } +} + +/* A workspace of five packages. `core` is the one under test; `util` and + * `extra` are path dependencies of it; `twin` has a lib and two binaries that + * share a module; `old` is an edition-2015 crate. */ +static void rs_write_workspace(const char *root) { + static const rs_fixture_file_t files[] = { + {"Cargo.toml", + "[workspace]\n" + "members = [\"crates/core\", \"crates/util\", \"crates/extra\", \"crates/twin\",\n" + " \"crates/old\"]\n" + "\n" + "[workspace.package]\n" + "edition = \"2021\"\n" + "\n" + "[workspace.dependencies]\n" + "util = { path = \"crates/util\" }\n"}, + + /* util: a path dependency, inherited through the workspace table */ + {"crates/util/Cargo.toml", "[package]\nname = \"util\"\nedition = \"2021\"\n"}, + {"crates/util/src/lib.rs", "//! Utilities.\n" + "mod inner;\n" + "pub use crate::inner::Hidden as Shown;\n" + "/// A tool.\n" + "pub struct Tool;\n" + "impl Tool {\n" + " /// Runs.\n" + " pub fn run(&self) {}\n" + "}\n"}, + {"crates/util/src/inner.rs", "/// Hidden away.\npub struct Hidden;\n"}, + + /* extra: its lib target is named unlike its package */ + {"crates/extra/Cargo.toml", "[package]\nname = \"acme-extra\"\nedition = \"2021\"\n" + "\n[lib]\nname = \"extra_lib\"\n"}, + {"crates/extra/src/lib.rs", "/// Extra.\npub struct Extra;\n"}, + + {"crates/core/Cargo.toml", "[package]\n" + "name = \"acme-core\"\n" + "edition.workspace = true\n" + "\n" + "[dependencies]\n" + "util.workspace = true\n" + "serde = \"1\"\n" + "acme-extra = { path = \"../extra\" }\n" + "Option = { package = \"acme-extra\", path = \"../extra\" }\n"}, + {"crates/core/src/lib.rs", + "//! Crate doc: [`Gadget`] and [shapes::Circle], also [the tool][tool].\n" /* 1 */ + "//! Rotten: [`NoSuchThing`].\n" /* 2 */ + "//!\n" + "//! [tool]: util::Tool\n" + "\n" + "pub mod shapes;\n" + "pub mod deep;\n" + "pub mod fake;\n" + "pub mod shadow;\n" + "pub mod levels;\n" + "pub mod char;\n" + "pub mod first;\n" + "pub mod chain;\n" + "pub mod strict;\n" + "pub mod broken;\n" + "#[cfg(test)]\n" + "mod tests_support;\n" + "#[path = \"elsewhere/odd.rs\"]\n" + "pub mod odd;\n" + "pub mod outer_inline {\n" + " pub mod leaf;\n" + " /// In an inline module.\n" + " pub struct InInline;\n" + "}\n" + "#[cfg(unix)]\n" + "#[path = \"imp_unix.rs\"]\n" + "mod imp;\n" + "#[cfg(windows)]\n" + "#[path = \"imp_win.rs\"]\n" + "mod imp;\n" + "\n" + "use fake::*;\n" + "pub use shapes::Circle;\n" + "use shapes::Circle as Disc;\n" + "use util::Tool as Hammer;\n" + "pub use paint_derive::Paintable;\n" + "\n" + "/// A gadget.\n" + "pub struct Gadget;\n" + "/// Own shape (the glob brings another one).\n" + "pub struct Shape;\n" + "/// A name that is no ASCII.\n" + "pub struct Gr\xC3\xB6\xC3\x9F" + "e;\n" + "/// A trait named like a derive macro.\n" + "pub trait Paintable {}\n" + "/// Both a function and a macro.\n" + "pub fn dual() {}\n" + "#[macro_export]\n" + "macro_rules! dual {\n" + " () => {};\n" + "}\n" + "/// Unix.\n" + "#[cfg(unix)]\n" + "pub fn plat() {}\n" + "/// Windows.\n" + "#[cfg(windows)]\n" + "pub fn plat() {}\n" + "/// One of two declarations under cfg.\n" + "#[cfg(unix)]\n" + "pub struct Twin;\n" + "/// The other.\n" + "#[cfg(not(unix))]\n" + "pub struct Twin;\n" + "/// Only for tests.\n" + "#[cfg(test)]\n" + "pub fn only_in_tests() {}\n" + "macro_rules! declare_knob {\n" + " () => {\n" + " pub struct Knob;\n" + " };\n" + "}\n" + "declare_knob!();\n" + "define_things! {\n" + " pub struct Made;\n" + "}\n" + "\n" + "/// [`Gadget`]\n" + "pub struct SrcShortcut;\n" + "/// [the circle](shapes::Circle)\n" + "pub struct SrcInline;\n" + "/// [`Circle`][]\n" + "pub struct SrcCollapsed;\n" + "/// [the hammer][ham]\n" + "///\n" + "/// [ham]: Hammer\n" + "pub struct SrcLinkDef;\n" + "/// [`Disc`]\n" + "pub struct SrcRenamed;\n" + "/// [site](https://example.org/docs) and [guide](../guide.md)\n" + "pub struct SrcUrl;\n" + "/// [`Shape`]\n" + "pub struct SrcOwnOverGlob;\n" + "/// [`OnlyInFake`] [`CrateWide`] [`Secret`]\n" + "pub struct SrcGlob;\n" + "/// [util::Tool::run] [extra_lib::Extra] [Option::Extra] [serde::Serialize]\n" + "/// [std::fmt::Debug] [util::Shown]\n" + "pub struct SrcExtern;\n" + "/// [`String`] [`u8`] [str::len] [`char`] [prim@char] [mod@char] [prim@Gadget]\n" + "pub struct SrcStd;\n" + "/// [crate::shapes::Kind] [self::Gadget] [crate]\n" + "pub struct SrcKeywords;\n" + "/// [Circle::area] [Circle::area()] [Circle::radius] [Circle::UNIT]\n" + "/// [shapes::Kind::Round] [shapes::Draw::draw] [Circle::draw] [shapes::Ring::area]\n" + "/// [shapes::Ring] [shapes::Pair] [shapes::Pair()] [shapes::Pair::0] [shapes::Bits]\n" + "/// [Circle::no_such] [Circle::clone] [shapes::Lamp::glow]\n" + "pub struct SrcMembers;\n" + "/// [`dual`] [`dual()`] [`dual!`] [fn@dual] [macro@dual]\n" + "pub struct SrcNamespaces;\n" + "/// [struct@Gadget] [enum@Gadget] [fn@Gadget] [type@Gadget] [`Paintable`] [`plat`]\n" + "/// [`Twin`]\n" + "pub struct SrcKinds;\n" + "/// [`NoSuchName`] [shapes::NoSuch] [`Gr\xC3\xB6\xC3\x9F" + "e`] [`\xC3\x9C" + "bel`]\n" + "/// [outer_inline] [outer_inline::InInline] [`Knob`] [`Made`]\n" + "/// [tests_support::Fixture] [`only_in_tests`] [`Circle::area(x)`] [Circle#a#b]\n" + "/// [imp::Thing]\n" + "pub struct SrcReasons;\n" + "/// [deep::pit::Hole] [shapes::round::Ball] [odd::Odd] [outer_inline::leaf::Leaf]\n" + "pub struct SrcModules;\n" + "/// [`count`] and [`T`] and [`Gadget`]\n" + "pub fn src_local(count: u32) {}\n"}, + {"crates/core/src/shapes.rs", + "pub mod round;\n" + "/// A circle. [`Self`] [`Self::area`]\n" + "#[derive(Clone)]\n" + "pub struct Circle {\n" + " /// The radius; see [`Circle::area`].\n" + " pub radius: f64,\n" + "}\n" + "impl Circle {\n" + " /// The unit.\n" + " pub const UNIT: f64 = 1.0;\n" + " /// Area from [`Self::radius`]; see [`Draw::draw`], [`Circle::draw`],\n" + " /// [`Circle::outline`], [`Self::UNIT`].\n" + " pub fn area(&self) -> f64 {\n" + " 0.0\n" + " }\n" + "}\n" + "/// Kinds.\n" + "pub enum Kind {\n" + " Round,\n" + " Flat,\n" + "}\n" + "/// Draws.\n" + "pub trait Draw {\n" + " /// Draw; see [`Self::outline`].\n" + " fn draw(&self);\n" + " /// Outline.\n" + " fn outline(&self) {}\n" + "}\n" + "impl Draw for Circle {\n" + " /// Draws a circle: [`Self::area`].\n" + " fn draw(&self) {}\n" + "}\n" + "/// Alias.\n" + "pub type Ring = Circle;\n" + "/// Tuple.\n" + "pub struct Pair(pub u32, pub u32);\n" + "/// Bits.\n" + "pub union Bits {\n" + " pub a: u32,\n" + " pub b: f32,\n" + "}\n" + "/// A field and a method of one name: the method is what a path names.\n" + "pub struct Lamp {\n" + " pub glow: u32,\n" + "}\n" + "impl Lamp {\n" + " /// Glows.\n" + " pub fn glow(&self) -> u32 {\n" + " self.glow\n" + " }\n" + "}\n"}, + {"crates/core/src/shapes/round.rs", + "/// A ball: [super::Circle] [super::super::Gadget] [`super::Kind`]\n" + "pub struct Ball;\n"}, + {"crates/core/src/deep/mod.rs", "pub mod pit;\n"}, + {"crates/core/src/deep/pit.rs", "/// A hole.\npub struct Hole;\n"}, + {"crates/core/src/fake.rs", "/// Decoy shape.\n" + "pub struct Shape;\n" + "/// Only here.\n" + "pub struct OnlyInFake;\n" + "/// Crate wide.\n" + "pub(crate) struct CrateWide;\n" + "struct Secret;\n"}, + {"crates/core/src/shadow.rs", "pub mod util;\n"}, + {"crates/core/src/shadow/util.rs", "/// The shadowing tool.\npub struct Tool;\n"}, + {"crates/core/src/levels.rs", "use crate::shadow::*;\n" + "/// Own Vec.\n" + "pub struct Vec;\n" + "/// [util::Tool] [`Vec`]\n" + "pub struct LevelGlob;\n"}, + {"crates/core/src/char.rs", "/// In a module named like a primitive type.\n" + "pub struct Unit;\n"}, + {"crates/core/src/first.rs", "/// [`crate::Gadget`]\npub struct First;\n"}, + {"crates/core/src/chain.rs", "mod a {\n" + " /// Deep.\n" + " pub struct Deep;\n" + "}\n" + "mod b {\n" + " pub use super::a::Deep as Mid;\n" + "}\n" + "pub use self::b::Mid as Top;\n" + "mod g1 {\n" + " pub use super::g2::*;\n" + "}\n" + "mod g2 {\n" + " /// Via globs.\n" + " pub struct ViaGlobs;\n" + "}\n" + "use self::g1::*;\n" + "mod c1 {\n" + " pub use super::c2::Loop;\n" + "}\n" + "mod c2 {\n" + " pub use super::c1::Loop;\n" + "}\n" + "/// [`Top`] [`ViaGlobs`] [c1::Loop]\n" + "pub struct ChainUser;\n"}, + {"crates/core/src/strict.rs", "use serde::*;\n" + "use util::Tool as Wrench;\n" + "/// [util::Tool] [`Wrench`]\n" + "pub struct StrictUser;\n"}, + {"crates/core/src/broken.rs", "/// Before the error: [`crate::Gadget`] [`Vec`]\n" + "pub struct Whole;\n" + "pub struct {\n" + "/// After the error: [`crate::Shape`]\n" + "pub struct Cut;\n"}, + /* declared by no `mod`: an orphan */ + {"crates/core/src/orphan.rs", "/// [crate::Gadget] [`String`] [serde::Serialize]\n" + "pub struct Orphan;\n"}, + {"crates/core/src/tests_support.rs", + "/// A fixture: [`crate::Gadget`] [`super::only_in_tests`]\n" + "pub struct Fixture;\n"}, + {"crates/core/src/elsewhere/odd.rs", "/// Odd.\npub struct Odd;\n"}, + {"crates/core/src/outer_inline/leaf.rs", "/// Leaf.\npub struct Leaf;\n"}, + {"crates/core/src/imp_unix.rs", "/// Unix thing.\npub struct Thing;\n"}, + {"crates/core/src/imp_win.rs", "/// Windows thing.\npub struct Thing;\n"}, + {"crates/core/tests/it.rs", "/// [acme_core::Gadget] [`acme_core::shapes::Circle::area`]\n" + "fn it_works() {}\n"}, + + /* twin: one module file under two crate roots, and a second binary */ + {"crates/twin/Cargo.toml", "[package]\nname = \"twin\"\nedition = \"2021\"\n"}, + {"crates/twin/src/lib.rs", "mod common;\n" + "/// Only in the lib.\n" + "pub struct Only;\n" + "/// In both roots.\n" + "pub struct Both;\n"}, + {"crates/twin/src/main.rs", "mod common;\n" + "/// In both roots.\n" + "pub struct Both;\n" + "fn main() {}\n"}, + {"crates/twin/src/common.rs", "/// [crate::Only] [crate::Both]\npub struct Shared;\n"}, + {"crates/twin/src/bin/tool.rs", "/// [twin::Only]\npub fn run_tool() {}\n"}, + + /* old: no edition key means 2015, where a `use` path starts at the root */ + {"crates/old/Cargo.toml", "[package]\nname = \"old\"\n"}, + {"crates/old/src/lib.rs", "mod a;\nmod b;\n"}, + {"crates/old/src/a.rs", "/// Thing.\npub struct Thing;\n"}, + {"crates/old/src/b.rs", "use a::Thing;\n/// [`Thing`]\npub struct UsesThing;\n"}, + }; + rs_write_files(root, files, sizeof(files) / sizeof(files[0])); +} + +/* One edge from the definition ending in `src` to the one ending in `tgt`, + * with every fragment of `want` (separated by '|') in its properties. */ +static bool rs_edge_is(const char *db, const char *src, const char *tgt, const char *want) { + char props[512]; + char frags[256]; + int n = 0; + dm_edge(db, src, tgt, props, sizeof(props), &n); + if (n != 1) { + printf(" edge %s -> %s: %d found\n", src, tgt, n); + return false; + } + snprintf(frags, sizeof(frags), "%s", want); + for (char *f = strtok(frags, "|"); f; f = strtok(NULL, "|")) { + if (!strstr(props, f)) { + printf(" edge %s -> %s: %s lacks %s\n", src, tgt, props, f); + return false; + } + } + return true; +} + +static bool rs_no_edge(const char *db, const char *src, const char *tgt) { + char props[512]; + int n = 0; + dm_edge(db, src, tgt, props, sizeof(props), &n); + if (n != 0) { + printf(" edge %s -> %s: %d found, none expected\n", src, tgt, n); + } + return n == 0; +} + +/* The unresolved row of `raw` in `rel` has this reason ("" for no row). */ +static bool rs_row_is(const char *db, const char *rel, const char *raw, const char *want) { + char reason[64]; + dm_row(db, rel, raw, reason, sizeof(reason), NULL, 0); + if (strcmp(reason, want) != 0) { + printf(" row %s [%s]: \"%s\", expected \"%s\"\n", rel, raw, reason, want); + return false; + } + return true; +} + +#define RS_SYNTAX_INTRA "\"syntax\":\"intra_doc\"" +#define RS_SYNTAX_DEF "\"syntax\":\"link_def\"" +#define RS_EXACT "\"tier\":\"exact\"" +#define RS_UNIQUE "\"tier\":\"unique\"" + +/* One indexed copy of the workspace for the resolver tests. */ +typedef struct { + char tmp[256]; + char db[512]; +} rs_ws_t; + +static bool rs_ws_open(rs_ws_t *ws) { + snprintf(ws->tmp, sizeof(ws->tmp), "/tmp/cbm_dmrs_XXXXXX"); + if (!cbm_mkdtemp(ws->tmp)) { + return false; + } + rs_write_workspace(ws->tmp); + snprintf(ws->db, sizeof(ws->db), "%s/ws.db", ws->tmp); + return dm_index(ws->tmp, ws->db, NULL) == 0; +} + +static void rs_ws_close(rs_ws_t *ws) { + dm_unlink_db(ws->db); + th_rmtree(ws->tmp); +} + +static const char RS_LIB[] = "crates/core/src/lib.rs"; + +/* The two families, both tiers, and what is no reference at all. */ +TEST(doc_mentions_rs_families_and_tiers) { + rs_ws_t ws; + ASSERT_TRUE(rs_ws_open(&ws)); + const char *db = ws.db; + char syntax[64]; + char reason[64]; + /* intra_doc: the destination is written at the link */ + ASSERT_TRUE(rs_edge_is(db, "lib.SrcShortcut", "lib.Gadget", + "\"via\":\"doc_comment\"|" RS_SYNTAX_INTRA "|" RS_UNIQUE + "|\"count\":1")); + ASSERT_TRUE(rs_edge_is(db, "lib.SrcInline", "shapes.Circle", RS_SYNTAX_INTRA "|" RS_EXACT)); + /* a name bound by a named import is an exact naming */ + ASSERT_TRUE(rs_edge_is(db, "lib.SrcCollapsed", "shapes.Circle", RS_SYNTAX_INTRA "|" RS_EXACT)); + ASSERT_TRUE(rs_edge_is(db, "lib.SrcRenamed", "shapes.Circle", RS_EXACT)); + /* link_def: the destination comes from a link reference definition */ + ASSERT_TRUE(rs_edge_is(db, "lib.SrcLinkDef", "util.src.lib.Tool", RS_SYNTAX_DEF "|" RS_EXACT)); + /* a URL is the core's href family; a relative file is not harvested */ + dm_row(db, RS_LIB, "https://example.org/docs", reason, sizeof(reason), syntax, sizeof(syntax)); + ASSERT_STR_EQ(reason, "external"); + ASSERT_STR_EQ(syntax, "href"); + ASSERT_TRUE(rs_row_is(db, RS_LIB, "../guide.md", "")); + ASSERT_EQ(dm_mentions_from(db, "lib.SrcUrl"), 0); + /* the definition's own parameter and type parameter: no edge, no row */ + ASSERT_TRUE(rs_edge_is(db, "lib.src_local", "lib.Gadget", RS_UNIQUE)); + ASSERT_EQ(dm_mentions_from(db, "lib.src_local"), 1); + ASSERT_TRUE(rs_row_is(db, RS_LIB, "count", "")); + ASSERT_TRUE(rs_row_is(db, RS_LIB, "T", "")); + rs_ws_close(&ws); + PASS(); +} + +/* A file's own doc: the File node is the source. */ +TEST(doc_mentions_rs_file_level_docs) { + rs_ws_t ws; + ASSERT_TRUE(rs_ws_open(&ws)); + const char *db = ws.db; + const char *file = "core.src.lib.rs.__file__"; + ASSERT_TRUE( + rs_edge_is(db, file, "core.src.lib.Gadget", RS_SYNTAX_INTRA "|" RS_UNIQUE "|\"line\":1")); + ASSERT_TRUE(rs_edge_is(db, file, "shapes.Circle", RS_SYNTAX_INTRA "|" RS_EXACT)); + ASSERT_TRUE(rs_edge_is(db, file, "util.src.lib.Tool", RS_SYNTAX_DEF "|" RS_EXACT)); + ASSERT_EQ(dm_mentions_from(db, file), 3); + /* an unresolved one: a row with the file's path and the reference's line */ + ASSERT_TRUE(rs_row_is(db, RS_LIB, "NoSuchThing", "missing")); + ASSERT_EQ(dm_count(db, "SELECT line FROM doc_link_unresolved WHERE raw = 'NoSuchThing'"), 2); + /* the outer doc of a file's first item belongs to the item */ + ASSERT_TRUE(rs_edge_is(db, "first.First", "core.src.lib.Gadget", RS_EXACT)); + ASSERT_EQ(dm_mentions_from(db, "first.rs.__file__"), 0); + rs_ws_close(&ws); + PASS(); +} + +/* The lookup levels, each with a decoy one level further out. */ +TEST(doc_mentions_rs_lookup_levels) { + rs_ws_t ws; + ASSERT_TRUE(rs_ws_open(&ws)); + const char *db = ws.db; + /* level 1 before level 2: the module's own item, not the glob's */ + ASSERT_TRUE(rs_edge_is(db, "lib.SrcOwnOverGlob", "core.src.lib.Shape", RS_UNIQUE)); + ASSERT_TRUE(rs_no_edge(db, "lib.SrcOwnOverGlob", "fake.Shape")); + /* level 2: a glob import brings in what the importing module can see */ + ASSERT_TRUE(rs_edge_is(db, "lib.SrcGlob", "fake.OnlyInFake", RS_UNIQUE)); + ASSERT_TRUE(rs_edge_is(db, "lib.SrcGlob", "fake.CrateWide", RS_UNIQUE)); + ASSERT_TRUE(rs_row_is(db, RS_LIB, "Secret", "missing")); /* private to `fake` */ + /* level 2 before level 3: the module a glob brings in, not the crate of + * the extern prelude named like it */ + ASSERT_TRUE(rs_edge_is(db, "levels.LevelGlob", "shadow.util.Tool", RS_EXACT)); + ASSERT_TRUE(rs_no_edge(db, "levels.LevelGlob", "util.src.lib.Tool")); + /* level 1 before level 4: the module's own `Vec`, not the prelude's */ + ASSERT_TRUE(rs_edge_is(db, "levels.LevelGlob", "levels.Vec", RS_UNIQUE)); + /* level 3: a path dependency (through the workspace table), a lib target + * named unlike its package, and a re-export of the dependency */ + ASSERT_TRUE(rs_edge_is(db, "lib.SrcExtern", "util.src.lib.Tool.run", RS_EXACT)); + ASSERT_TRUE(rs_edge_is(db, "lib.SrcExtern", "extra.src.lib.Extra", RS_EXACT "|\"count\":2")); + ASSERT_TRUE(rs_edge_is(db, "lib.SrcExtern", "util.src.inner.Hidden", RS_EXACT)); + /* level 3 before level 4: a dependency renamed `Option` is that crate, + * not the prelude's type (the second of the two references above) */ + ASSERT_TRUE(rs_row_is(db, RS_LIB, "Option::Extra", "")); + /* a dependency without a path, and the standard library */ + ASSERT_TRUE(rs_row_is(db, RS_LIB, "serde::Serialize", "external")); + ASSERT_TRUE(rs_row_is(db, RS_LIB, "std::fmt::Debug", "external")); + /* level 4: the std prelude and the primitive types */ + ASSERT_TRUE(rs_row_is(db, RS_LIB, "String", "external")); + ASSERT_TRUE(rs_row_is(db, RS_LIB, "u8", "external")); + ASSERT_TRUE(rs_row_is(db, RS_LIB, "str::len", "external")); + /* a module named like a primitive type: rustdoc wants a disambiguator */ + ASSERT_TRUE(rs_row_is(db, RS_LIB, "char", "ambiguous")); + ASSERT_TRUE(rs_row_is(db, RS_LIB, "prim@char", "external")); + ASSERT_TRUE(rs_edge_is(db, "lib.SrcStd", "core.src.char", RS_UNIQUE)); + ASSERT_TRUE(rs_row_is(db, RS_LIB, "prim@Gadget", "missing")); + /* a doc-link path does not look past a glob of something outside the + * repository (its names would silently win); an import path does */ + ASSERT_TRUE(rs_row_is(db, "crates/core/src/strict.rs", "util::Tool", "external")); + ASSERT_TRUE(rs_edge_is(db, "strict.StrictUser", "util.src.lib.Tool", RS_EXACT)); + rs_ws_close(&ws); + PASS(); +} + +/* `crate`, `self`, `super`, `Self`, and what a path walks through. */ +TEST(doc_mentions_rs_paths_and_modules) { + rs_ws_t ws; + ASSERT_TRUE(rs_ws_open(&ws)); + const char *db = ws.db; + ASSERT_TRUE(rs_edge_is(db, "lib.SrcKeywords", "shapes.Kind", RS_EXACT)); + ASSERT_TRUE(rs_edge_is(db, "lib.SrcKeywords", "core.src.lib.Gadget", RS_EXACT)); + ASSERT_TRUE(rs_edge_is(db, "lib.SrcKeywords", "core.src.lib", RS_EXACT)); /* the root module */ + /* `super`, twice: the file that declares the module, then the crate root */ + ASSERT_TRUE(rs_edge_is(db, "round.Ball", "shapes.Circle", RS_EXACT)); + ASSERT_TRUE(rs_edge_is(db, "round.Ball", "core.src.lib.Gadget", RS_EXACT)); + ASSERT_TRUE(rs_edge_is(db, "round.Ball", "shapes.Kind", RS_EXACT)); + /* `Self`: the documented type (a reference to itself is no edge), the + * impl's self type, the trait */ + ASSERT_TRUE(rs_edge_is(db, "shapes.Circle", "shapes.Circle.area", RS_EXACT)); + ASSERT_TRUE(rs_no_edge(db, "shapes.Circle", "shapes.Circle")); + ASSERT_TRUE(rs_row_is(db, "crates/core/src/shapes.rs", "Self", "")); + ASSERT_TRUE(rs_edge_is(db, "shapes.Circle.area", "shapes.Circle.radius", RS_EXACT)); + ASSERT_TRUE(rs_edge_is(db, "shapes.Draw.draw", "shapes.Draw.outline", RS_EXACT)); + ASSERT_TRUE(rs_edge_is(db, "shapes.Circle.draw", "shapes.Circle.area", RS_EXACT)); + /* a field's doc */ + ASSERT_TRUE(rs_edge_is(db, "shapes.Circle.radius", "shapes.Circle.area", RS_EXACT)); + /* where `mod x;` files are: beside a mod.rs, under the directory of any + * other file, at a `#[path]`, under an inline module's directory */ + ASSERT_TRUE(rs_edge_is(db, "lib.SrcModules", "deep.pit.Hole", RS_EXACT)); + ASSERT_TRUE(rs_edge_is(db, "lib.SrcModules", "shapes.round.Ball", RS_EXACT)); + ASSERT_TRUE(rs_edge_is(db, "lib.SrcModules", "elsewhere.odd.Odd", RS_EXACT)); + ASSERT_TRUE(rs_edge_is(db, "lib.SrcModules", "outer_inline.leaf.Leaf", RS_EXACT)); + /* two files for one module name under different cfg */ + ASSERT_TRUE(rs_row_is(db, RS_LIB, "imp::Thing", "ambiguous")); + /* imports: a chain of renaming re-exports, a chain of globs, a cycle */ + ASSERT_TRUE(rs_edge_is(db, "chain.ChainUser", "chain.Deep", RS_EXACT)); + ASSERT_TRUE(rs_edge_is(db, "chain.ChainUser", "chain.ViaGlobs", RS_UNIQUE)); + ASSERT_TRUE(rs_row_is(db, "crates/core/src/chain.rs", "c1::Loop", "graph_gap")); + /* one module file under two crate roots: what one root declares binds, + * what both declare is ambiguous */ + ASSERT_TRUE(rs_edge_is(db, "common.Shared", "twin.src.lib.Only", RS_EXACT)); + ASSERT_TRUE(rs_row_is(db, "crates/twin/src/common.rs", "crate::Both", "ambiguous")); + /* a package's other targets see its lib by name */ + ASSERT_TRUE(rs_edge_is(db, "bin.tool.run_tool", "twin.src.lib.Only", RS_EXACT)); + ASSERT_TRUE(rs_edge_is(db, "tests.it.it_works", "core.src.lib.Gadget", RS_EXACT)); + ASSERT_TRUE(rs_edge_is(db, "tests.it.it_works", "shapes.Circle.area", RS_EXACT)); + /* edition 2015: a `use` path starts at the crate root */ + ASSERT_TRUE(rs_edge_is(db, "old.src.b.UsesThing", "old.src.a.Thing", RS_EXACT)); + rs_ws_close(&ws); + PASS(); +} + +/* Associated items, namespaces, disambiguators. */ +TEST(doc_mentions_rs_members_and_kinds) { + rs_ws_t ws; + ASSERT_TRUE(rs_ws_open(&ws)); + const char *db = ws.db; + /* a method, with and without `()`, and through a type alias: one edge */ + ASSERT_TRUE(rs_edge_is(db, "lib.SrcMembers", "shapes.Circle.area", RS_EXACT "|\"count\":3")); + ASSERT_TRUE(rs_edge_is(db, "lib.SrcMembers", "shapes.Circle.radius", RS_EXACT)); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM nodes WHERE label = 'Field' AND " + "qualified_name LIKE '%.shapes.Circle.radius'"), + 1); + ASSERT_TRUE(rs_edge_is(db, "lib.SrcMembers", "shapes.Draw.draw", RS_EXACT)); + ASSERT_TRUE(rs_edge_is(db, "lib.SrcMembers", "shapes.Ring", RS_EXACT)); + ASSERT_TRUE(rs_edge_is(db, "lib.SrcMembers", "shapes.Pair", RS_EXACT)); + ASSERT_TRUE(rs_edge_is(db, "lib.SrcMembers", "shapes.Bits", RS_EXACT)); + /* a field and a method of one name: the method (rustdoc's order) */ + ASSERT_TRUE(rs_edge_is(db, "lib.SrcMembers", "shapes.Lamp.glow", RS_EXACT)); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM nodes WHERE label = 'Method' AND " + "qualified_name LIKE '%.shapes.Lamp.glow'"), + 1); + /* an item of a trait impl needs its trait in scope where the link is + * written: not in lib.rs, but in shapes.rs, where a default item of the + * trait is found through the impl as well */ + ASSERT_TRUE(rs_row_is(db, RS_LIB, "Circle::draw", "missing")); + ASSERT_TRUE(rs_edge_is(db, "shapes.Circle.area", "shapes.Circle.draw", RS_EXACT)); + ASSERT_TRUE(rs_edge_is(db, "shapes.Circle.area", "shapes.Draw.outline", RS_EXACT)); + ASSERT_TRUE(rs_edge_is(db, "shapes.Circle.area", "shapes.Draw.draw", RS_EXACT)); + /* declared, but without a graph node */ + ASSERT_TRUE(rs_row_is(db, RS_LIB, "Circle::UNIT", "graph_gap")); /* impl constant */ + ASSERT_TRUE(rs_row_is(db, RS_LIB, "shapes::Kind::Round", "graph_gap")); /* variant */ + ASSERT_TRUE(rs_row_is(db, RS_LIB, "shapes::Pair::0", "graph_gap")); /* tuple field */ + ASSERT_TRUE(rs_row_is(db, RS_LIB, "Circle::clone", "graph_gap")); /* #[derive(Clone)] */ + /* not declared at all; a tuple struct is no function */ + ASSERT_TRUE(rs_row_is(db, RS_LIB, "Circle::no_such", "missing")); + ASSERT_TRUE(rs_row_is(db, RS_LIB, "shapes::Pair()", "missing")); + /* a function and a macro of one name */ + ASSERT_TRUE(rs_row_is(db, RS_LIB, "dual", "ambiguous")); + ASSERT_TRUE(rs_edge_is(db, "lib.SrcNamespaces", "core.src.lib.dual", "\"count\":2")); + ASSERT_TRUE(rs_row_is(db, RS_LIB, "dual!", "graph_gap")); + ASSERT_TRUE(rs_row_is(db, RS_LIB, "macro@dual", "graph_gap")); + /* a disambiguator of another kind is no match */ + ASSERT_TRUE(rs_edge_is(db, "lib.SrcKinds", "core.src.lib.Gadget", "\"count\":2")); + ASSERT_TRUE(rs_row_is(db, RS_LIB, "enum@Gadget", "missing")); + ASSERT_TRUE(rs_row_is(db, RS_LIB, "fn@Gadget", "missing")); + /* a trait and a derive macro of one name are the trait */ + ASSERT_TRUE(rs_edge_is(db, "lib.SrcKinds", "core.src.lib.Paintable", RS_UNIQUE)); + /* declared twice under cfg: one definition with its variants on one node, + * so the item binds */ + ASSERT_TRUE(rs_row_is(db, RS_LIB, "plat", "")); + ASSERT_TRUE(rs_edge_is(db, "lib.SrcKinds", "core.src.lib.plat", RS_UNIQUE)); + ASSERT_TRUE(rs_edge_is(db, "lib.SrcKinds", "core.src.lib.Twin", RS_UNIQUE)); + rs_ws_close(&ws); + PASS(); +} + +/* Every reason, with the cases that are easy to mislabel. */ +TEST(doc_mentions_rs_reasons) { + rs_ws_t ws; + ASSERT_TRUE(rs_ws_open(&ws)); + const char *db = ws.db; + /* missing: nothing of that name where the language looks */ + ASSERT_TRUE(rs_row_is(db, RS_LIB, "NoSuchName", "missing")); + ASSERT_TRUE(rs_row_is(db, RS_LIB, "shapes::NoSuch", "missing")); + /* a name that is no ASCII is a name like any other */ + ASSERT_TRUE(rs_edge_is(db, "lib.SrcReasons", + "lib.Gr\xC3\xB6\xC3\x9F" + "e", + RS_UNIQUE)); + ASSERT_TRUE(rs_row_is(db, RS_LIB, + "\xC3\x9C" + "bel", + "missing")); + /* graph_gap: an inline module has no node, its items have */ + ASSERT_TRUE(rs_row_is(db, RS_LIB, "outer_inline", "graph_gap")); + ASSERT_TRUE(rs_edge_is(db, "lib.SrcReasons", "lib.InInline", RS_EXACT)); + /* graph_gap: declared by a macro, in its body or at the invocation */ + ASSERT_TRUE(rs_row_is(db, RS_LIB, "Knob", "graph_gap")); + ASSERT_TRUE(rs_row_is(db, RS_LIB, "Made", "graph_gap")); + /* test_only_target: product code never binds test code ... */ + ASSERT_TRUE(rs_row_is(db, RS_LIB, "tests_support::Fixture", "test_only_target")); + ASSERT_TRUE(rs_row_is(db, RS_LIB, "only_in_tests", "test_only_target")); + /* ... test code binds both */ + ASSERT_TRUE(rs_edge_is(db, "tests_support.Fixture", "core.src.lib.Gadget", RS_EXACT)); + ASSERT_TRUE(rs_edge_is(db, "tests_support.Fixture", "lib.only_in_tests", RS_EXACT)); + /* unparseable: a link whose destination is no path */ + ASSERT_TRUE(rs_row_is(db, RS_LIB, "Circle::area(x)", "unparseable")); + ASSERT_TRUE(rs_row_is(db, RS_LIB, "Circle#a#b", "unparseable")); + /* ambiguous */ + ASSERT_TRUE(rs_row_is(db, RS_LIB, "imp::Thing", "ambiguous")); + /* external */ + ASSERT_TRUE(rs_row_is(db, RS_LIB, "serde::Serialize", "external")); + /* a file with a parse error: what stands before the error resolves what + * it can name for sure; a name the file does not show is a gap, not the + * prelude's; what follows the error has no scope */ + ASSERT_TRUE(rs_edge_is(db, "broken.Whole", "core.src.lib.Gadget", RS_EXACT)); + ASSERT_TRUE(rs_row_is(db, "crates/core/src/broken.rs", "Vec", "graph_gap")); + ASSERT_TRUE(rs_row_is(db, "crates/core/src/broken.rs", "crate::Shape", "graph_gap")); + /* a file no crate root reaches: its `crate` is not known, its package is */ + ASSERT_TRUE(rs_row_is(db, "crates/core/src/orphan.rs", "crate::Gadget", "graph_gap")); + ASSERT_TRUE(rs_row_is(db, "crates/core/src/orphan.rs", "String", "external")); + ASSERT_TRUE(rs_row_is(db, "crates/core/src/orphan.rs", "serde::Serialize", "external")); + /* no reason outside the core's set, and none left as an unknown one */ + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM doc_link_unresolved WHERE reason NOT IN " + "('missing','ambiguous','external','test_only_target','graph_gap'," + "'unparseable')"), + 0); + rs_ws_close(&ws); + PASS(); +} + +/* ── resolution: structure nobody controls ───────────────────────── */ + +/* Deep nesting, long chains, cycles and crowds: every one resolves or ends + * in a row; none may hang, crash or pick something at random. */ +TEST(doc_mentions_rs_untrusted_structure) { + enum { DEPTH = 600, CHAIN = 100, CROWD = 3000, RING = 40, BUF = 1 << 20 }; + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dmrs_un_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char *src = malloc(BUF); + ASSERT_NOT_NULL(src); + th_write_file(TH_PATH(tmp, "Cargo.toml"), "[package]\nname = \"wild\"\nedition = \"2021\"\n"); + th_write_file(TH_PATH(tmp, "src/lib.rs"), + "pub mod nest;\npub mod globs;\npub mod ring;\npub mod crowd;\n" + "/// The root's target.\npub struct Top;\n"); + + /* inline modules nested DEPTH deep; the innermost names the root through + * `crate`, and through more `super`s than a path carries */ + size_t w = 0; + for (int i = 0; i < DEPTH; i++) { + w += (size_t)snprintf(src + w, BUF - w, "pub mod m {\n"); + } + w += (size_t)snprintf(src + w, BUF - w, "/// [crate::Top] ["); + for (int i = 0; i < CBM_DOCLINK_RS_MAX_SEGS + 4; i++) { + w += (size_t)snprintf(src + w, BUF - w, "super::"); + } + w += (size_t)snprintf(src + w, BUF - w, "Top]\npub struct Innermost;\n"); + for (int i = 0; i < DEPTH; i++) { + w += (size_t)snprintf(src + w, BUF - w, "}\n"); + } + th_write_file(TH_PATH(tmp, "src/nest.rs"), src); + + /* a chain of re-exporting globs longer than is followed, and a short one */ + w = (size_t)snprintf(src, BUF, "use self::g0::*;\nuse self::s0::*;\n"); + for (int i = 0; i < CHAIN; i++) { + w += (size_t)snprintf(src + w, BUF - w, "mod g%d { pub use super::g%d::*; }\n", i, i + 1); + } + w += (size_t)snprintf(src + w, BUF - w, + "mod g%d { /// Far.\n pub struct Far; }\n" + "mod s0 { pub use super::s1::*; }\n" + "mod s1 { /// Near.\n pub struct Near; }\n" + "/// [`Far`] [`Near`]\npub struct GlobUser;\n", + CHAIN); + th_write_file(TH_PATH(tmp, "src/globs.rs"), src); + + /* a ring of imports, each waiting for the next */ + w = 0; + for (int i = 0; i < RING; i++) { + w += (size_t)snprintf(src + w, BUF - w, "mod r%d { pub use super::r%d::Round; }\n", i, + (i + 1) % RING); + } + w += (size_t)snprintf(src + w, BUF - w, "/// [r0::Round]\npub struct RingUser;\n"); + th_write_file(TH_PATH(tmp, "src/ring.rs"), src); + + /* thousands of declarations of one name, and of impls of one type */ + w = 0; + for (int i = 0; i < CROWD; i++) { + w += (size_t)snprintf(src + w, BUF - w, "#[cfg(c%d)]\npub fn many() {}\n", i); + } + w += (size_t)snprintf(src + w, BUF - w, "/// One type, many impls.\npub struct Hub;\n"); + for (int i = 0; i < CROWD; i++) { + w += (size_t)snprintf(src + w, BUF - w, + "impl From<[u8; %d]> for Hub {\n" + " fn from(_: [u8; %d]) -> Self { Hub }\n}\n", + i, i); + } + w += (size_t)snprintf(src + w, BUF - w, + "/// [`many`] [`Hub::from`] [`Hub`]\n" + "pub struct CrowdUser;\n"); + th_write_file(TH_PATH(tmp, "src/crowd.rs"), src); + free(src); + + char db[512]; + snprintf(db, sizeof(db), "%s/wild.db", tmp); + ASSERT_EQ(dm_index(tmp, db, NULL), 0); + /* depth is no limit for `crate`; a path of more segments than are carried + * is no path */ + ASSERT_TRUE(rs_edge_is(db, "nest.Innermost", "src.lib.Top", RS_EXACT)); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM doc_link_unresolved WHERE rel_path = " + "'src/nest.rs' AND reason = 'unparseable'"), + 1); + /* a glob chain that is followed, and one that is not: a gap, never a miss */ + ASSERT_TRUE(rs_edge_is(db, "globs.GlobUser", "globs.Near", RS_UNIQUE)); + ASSERT_TRUE(rs_row_is(db, "src/globs.rs", "Far", "graph_gap")); + /* a cycle of imports is not followed */ + ASSERT_TRUE(rs_row_is(db, "src/ring.rs", "r0::Round", "graph_gap")); + /* a crowd of one name is ambiguous, whatever its size */ + ASSERT_TRUE(rs_row_is(db, "src/crowd.rs", "many", "ambiguous")); + ASSERT_TRUE(rs_row_is(db, "src/crowd.rs", "Hub::from", "ambiguous")); + ASSERT_TRUE(rs_edge_is(db, "crowd.CrowdUser", "crowd.Hub", RS_UNIQUE)); + dm_unlink_db(db); + th_rmtree(tmp); + PASS(); +} + +/* ── which scope changes are repairable file by file ─────────────── */ + +static int rs_delta(const char *before, const char *after, char *names, size_t cap) { + return dm_scope_delta(CBM_LANG_RUST, "src/s.rs", before, after, names, cap); +} + +#define RS_D_USE "use std::fmt;\n" +#define RS_D_STRUCT "pub struct W {\n pub size: u32,\n pub name: u32,\n}\n" +#define RS_D_IMPL_OPEN "impl W {\n" +#define RS_D_RUN " pub fn run(&self) {}\n" +#define RS_D_STOP " pub fn stop(&self) {}\n" +#define RS_D_LIMIT " pub const LIMIT: u32 = 1;\n" +#define RS_D_IMPL_CLOSE "}\n" +#define RS_D_ENUM "pub enum Kind {\n A,\n B,\n}\n" +#define RS_D_FREE "pub fn free() {}\n" +#define RS_D_OTHER "pub struct Other;\n" +#define RS_D_IMPL RS_D_IMPL_OPEN RS_D_RUN RS_D_STOP RS_D_LIMIT RS_D_IMPL_CLOSE +#define RS_D_TAIL RS_D_ENUM RS_D_FREE RS_D_OTHER + +TEST(doc_mentions_rs_scope_delta_rules) { + const char *base = RS_D_USE RS_D_STRUCT RS_D_IMPL RS_D_TAIL; + struct { + const char *what; + const char *after; + int want; + const char *names; + } cases[] = { + {"unchanged", base, CBM_DOCLINK_DELTA_LOCAL, ""}, + /* the persisted scope carries no line number, no body and no doc */ + {"lines moved", "// moved\n\n" RS_D_USE RS_D_STRUCT RS_D_IMPL RS_D_TAIL, + CBM_DOCLINK_DELTA_LOCAL, ""}, + {"body edit", + RS_D_USE RS_D_STRUCT RS_D_IMPL_OPEN + " pub fn run(&self) {\n let _ = 1;\n }\n" RS_D_STOP RS_D_LIMIT RS_D_IMPL_CLOSE + RS_D_TAIL, + CBM_DOCLINK_DELTA_LOCAL, ""}, + {"doc edit", + RS_D_USE "/// Now documented: [`Other`], [`size`].\n" RS_D_STRUCT RS_D_IMPL RS_D_TAIL, + CBM_DOCLINK_DELTA_LOCAL, ""}, + /* a member's own visibility is no part of what a path can name */ + {"method made private", + RS_D_USE RS_D_STRUCT RS_D_IMPL_OPEN + " fn run(&self) {}\n" RS_D_STOP RS_D_LIMIT RS_D_IMPL_CLOSE RS_D_TAIL, + CBM_DOCLINK_DELTA_LOCAL, ""}, + /* a removed member is reported by name */ + {"method removed", + RS_D_USE RS_D_STRUCT RS_D_IMPL_OPEN RS_D_RUN RS_D_LIMIT RS_D_IMPL_CLOSE RS_D_TAIL, + CBM_DOCLINK_DELTA_LOCAL, "stop"}, + {"field removed", RS_D_USE "pub struct W {\n pub size: u32,\n}\n" RS_D_IMPL RS_D_TAIL, + CBM_DOCLINK_DELTA_LOCAL, "name"}, + {"constant removed", + RS_D_USE RS_D_STRUCT RS_D_IMPL_OPEN RS_D_RUN RS_D_STOP RS_D_IMPL_CLOSE RS_D_TAIL, + CBM_DOCLINK_DELTA_LOCAL, "LIMIT"}, + {"two members removed", + RS_D_USE "pub struct W {\n pub size: u32,\n}\n" RS_D_IMPL_OPEN RS_D_RUN RS_D_LIMIT + RS_D_IMPL_CLOSE RS_D_TAIL, + CBM_DOCLINK_DELTA_LOCAL, "name,stop"}, + /* everything else can re-route references of files with no edge here */ + {"method added", + RS_D_USE RS_D_STRUCT RS_D_IMPL_OPEN RS_D_RUN RS_D_STOP RS_D_LIMIT + " pub fn more(&self) {}\n" RS_D_IMPL_CLOSE RS_D_TAIL, + CBM_DOCLINK_DELTA_GLOBAL, NULL}, + {"method renamed", + RS_D_USE RS_D_STRUCT RS_D_IMPL_OPEN + " pub fn go(&self) {}\n" RS_D_STOP RS_D_LIMIT RS_D_IMPL_CLOSE RS_D_TAIL, + CBM_DOCLINK_DELTA_GLOBAL, NULL}, + {"members reordered", + RS_D_USE RS_D_STRUCT RS_D_IMPL_OPEN RS_D_STOP RS_D_RUN RS_D_LIMIT RS_D_IMPL_CLOSE + RS_D_TAIL, + CBM_DOCLINK_DELTA_GLOBAL, NULL}, + /* a variant can be imported under another name: no name finds the + * references that depended on it */ + {"variant removed", + RS_D_USE RS_D_STRUCT RS_D_IMPL "pub enum Kind {\n A,\n}\n" RS_D_FREE RS_D_OTHER, + CBM_DOCLINK_DELTA_GLOBAL, NULL}, + /* ... and so can every item */ + {"function removed", RS_D_USE RS_D_STRUCT RS_D_IMPL RS_D_ENUM RS_D_OTHER, + CBM_DOCLINK_DELTA_GLOBAL, NULL}, + {"type removed", RS_D_USE RS_D_STRUCT RS_D_IMPL RS_D_ENUM RS_D_FREE, + CBM_DOCLINK_DELTA_GLOBAL, NULL}, + {"type added", RS_D_USE RS_D_STRUCT RS_D_IMPL RS_D_TAIL "pub struct New;\n", + CBM_DOCLINK_DELTA_GLOBAL, NULL}, + {"item made private", RS_D_USE RS_D_STRUCT RS_D_IMPL RS_D_ENUM RS_D_FREE "struct Other;\n", + CBM_DOCLINK_DELTA_GLOBAL, NULL}, + {"item made test-only", + RS_D_USE RS_D_STRUCT RS_D_IMPL RS_D_ENUM RS_D_FREE "#[cfg(test)]\n" RS_D_OTHER, + CBM_DOCLINK_DELTA_GLOBAL, NULL}, + {"derive added", RS_D_USE "#[derive(Clone)]\n" RS_D_STRUCT RS_D_IMPL RS_D_TAIL, + CBM_DOCLINK_DELTA_GLOBAL, NULL}, + {"import added", RS_D_USE "use std::io;\n" RS_D_STRUCT RS_D_IMPL RS_D_TAIL, + CBM_DOCLINK_DELTA_GLOBAL, NULL}, + {"import removed", RS_D_STRUCT RS_D_IMPL RS_D_TAIL, CBM_DOCLINK_DELTA_GLOBAL, NULL}, + {"module declared", RS_D_USE "mod extra;\n" RS_D_STRUCT RS_D_IMPL RS_D_TAIL, + CBM_DOCLINK_DELTA_GLOBAL, NULL}, + {"impl header changed", + RS_D_USE RS_D_STRUCT + "impl fmt::Debug for W {\n" RS_D_RUN RS_D_STOP RS_D_LIMIT RS_D_IMPL_CLOSE RS_D_TAIL, + CBM_DOCLINK_DELTA_GLOBAL, NULL}, + /* an impl is more than its members: through it a type takes the + * default items of a trait, which live in another file */ + {"impl removed", RS_D_USE RS_D_STRUCT RS_D_TAIL, CBM_DOCLINK_DELTA_GLOBAL, NULL}, + /* a parse error where there was none */ + {"parse error", RS_D_USE RS_D_STRUCT RS_D_IMPL RS_D_TAIL "pub struct {\n", + CBM_DOCLINK_DELTA_GLOBAL, NULL}, + {"file deleted", NULL, CBM_DOCLINK_DELTA_GLOBAL, NULL}, + }; + char names[256]; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + int got = rs_delta(base, cases[i].after, names, sizeof(names)); + if (got != cases[i].want || (cases[i].names && strcmp(names, cases[i].names) != 0)) { + printf(" %s: delta %d [%s], want %d [%s]\n", cases[i].what, got, names, cases[i].want, + cases[i].names ? cases[i].names : "any"); + FAIL("scope delta"); + } + } + /* a member whose name the incremental route cannot look for in a row + * (it splits rows at every byte that is no ASCII letter, digit or `_`) */ + const char *wide = "pub struct W {\n pub gr\xC3\xB6\xC3\x9F" + "e: u32,\n pub b: u32,\n}\n"; + ASSERT_EQ(rs_delta(wide, "pub struct W {\n pub b: u32,\n}\n", names, sizeof(names)), + CBM_DOCLINK_DELTA_GLOBAL); + /* a raw identifier is reported as the name a reference writes */ + ASSERT_EQ(rs_delta("pub struct W {\n pub r#type: u32,\n pub b: u32,\n}\n", + "pub struct W {\n pub b: u32,\n}\n", names, sizeof(names)), + CBM_DOCLINK_DELTA_LOCAL); + ASSERT_STR_EQ(names, "type"); + /* a scope that appears is as global as one that leaves */ + ASSERT_EQ(rs_delta(NULL, base, names, sizeof(names)), CBM_DOCLINK_DELTA_GLOBAL); + + /* a manifest: only what the resolver reads of it counts, and every such + * change is global (it re-scopes the files of its package) */ + const char *toml = "[package]\nname = \"p\"\nedition = \"2021\"\n\n" + "[dependencies]\nserde = \"1\"\nutil = { path = \"../util\" }\n"; + struct { + const char *what; + const char *after; + int want; + } manifests[] = { + {"unchanged", toml, CBM_DOCLINK_DELTA_LOCAL}, + {"comment and version", + "# a comment\n[package]\nname = \"p\"\nedition = \"2021\"\nversion = \"2.0.0\"\n\n" + "[dependencies]\nserde = \"2\"\nutil = { path = \"../util\" }\n", + CBM_DOCLINK_DELTA_LOCAL}, + {"dependency added", + "[package]\nname = \"p\"\nedition = \"2021\"\n\n" + "[dependencies]\nserde = \"1\"\nutil = { path = \"../util\" }\nmore = \"1\"\n", + CBM_DOCLINK_DELTA_GLOBAL}, + {"path changed", + "[package]\nname = \"p\"\nedition = \"2021\"\n\n" + "[dependencies]\nserde = \"1\"\nutil = { path = \"../other\" }\n", + CBM_DOCLINK_DELTA_GLOBAL}, + {"edition changed", + "[package]\nname = \"p\"\nedition = \"2018\"\n\n" + "[dependencies]\nserde = \"1\"\nutil = { path = \"../util\" }\n", + CBM_DOCLINK_DELTA_GLOBAL}, + }; + for (size_t i = 0; i < sizeof(manifests) / sizeof(manifests[0]); i++) { + int got = dm_scope_delta(CBM_LANG_TOML, "Cargo.toml", toml, manifests[i].after, names, + sizeof(names)); + if (got != manifests[i].want) { + printf(" manifest %s: delta %d, want %d\n", manifests[i].what, got, manifests[i].want); + FAIL("manifest scope delta"); + } + } + PASS(); +} + +/* ── incremental == full ─────────────────────────────────────────── */ + +static const char RS_INC_LIB[] = "//! The crate: [`Target`].\n" + "mod a;\n" + "mod b;\n" + "mod user;\n" + "mod rows;\n" + "pub use a::Target;\n" + "use b::Other;\n"; +static const char RS_INC_LIB_DOC[] = "//! The crate: [`Target`] and [`Other`].\n" + "mod a;\n" + "mod b;\n" + "mod user;\n" + "mod rows;\n" + "pub use a::Target;\n" + "use b::Other;\n"; +#define RS_INC_A_HEAD "/// The target.\npub struct Target;\nimpl Target {\n" +#define RS_INC_A_GO " /// Go.\n pub fn go(&self) {}\n" +#define RS_INC_A_STOP " /// Stop.\n pub fn stop(&self) {}\n" +/* two impl blocks with a method of one name: the later one owns the node */ +#define RS_INC_A_TWIN1 \ + "impl Target {\n /// Twin.\n pub fn twin(&self) {}\n /// Kept.\n" \ + " pub fn keep(&self) {}\n}\n" +#define RS_INC_A_KEEP "impl Target {\n /// Kept.\n pub fn keep(&self) {}\n}\n" +#define RS_INC_A_TWIN2 "impl Target {\n /// Twin.\n pub fn twin(&self) {}\n}\n" +#define RS_INC_A_DUP "/// A duplicate.\npub struct Dup;\n" +static const char RS_INC_A[] = + RS_INC_A_HEAD RS_INC_A_GO RS_INC_A_STOP "}\n" RS_INC_A_TWIN1 RS_INC_A_TWIN2 RS_INC_A_DUP; +static const char RS_INC_A_BODY[] = + RS_INC_A_HEAD " /// Go.\n pub fn go(&self) {\n let _ = 1;\n }\n" RS_INC_A_STOP + "}\n" RS_INC_A_TWIN1 RS_INC_A_TWIN2 RS_INC_A_DUP; +static const char RS_INC_A_NO_STOP[] = RS_INC_A_HEAD + " /// Go.\n pub fn go(&self) {\n let _ = 1;\n }\n}\n" RS_INC_A_TWIN1 + RS_INC_A_TWIN2 RS_INC_A_DUP; +static const char RS_INC_A_ONE_TWIN[] = RS_INC_A_HEAD + " /// Go.\n pub fn go(&self) {\n let _ = 1;\n }\n}\n" RS_INC_A_KEEP + RS_INC_A_TWIN2 RS_INC_A_DUP; +static const char RS_INC_A_RENAMED[] = + "/// The target.\npub struct Renamed;\nimpl Renamed {\n" RS_INC_A_GO "}\n" RS_INC_A_DUP; +static const char RS_INC_B[] = "/// A duplicate.\npub struct Dup;\n/// Other.\npub struct Other;\n"; +static const char RS_INC_B_NO_DUP[] = "/// Other.\npub struct Other;\n"; +static const char RS_INC_USER[] = + "use crate::a::*;\n" + "use crate::b::*;\n" + "/// [`Target`] [`Target::go`] [`Target::stop`] [`Dup`] [`Missing`]\n" + "pub struct User;\n"; +static const char RS_INC_USER_DOC[] = + "use crate::a::*;\n" + "use crate::b::*;\n" + "/// [`Target`] [`Target::go`] [`Target::stop`] [`Dup`] [`Missing`] [`Other`]\n" + "pub struct User;\n"; +/* its only reference is unresolved: it has no edge into the file it names */ +static const char RS_INC_ROWS[] = "/// [crate::a::Target::twin]\npub struct Rows;\n"; + +TEST(doc_mentions_rs_incremental_equals_full) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dmrs_inc_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[400]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + th_write_file(TH_PATH(repo, "Cargo.toml"), "[package]\nname = \"inc\"\nedition = \"2021\"\n"); + th_write_file(TH_PATH(repo, "src/lib.rs"), RS_INC_LIB); + th_write_file(TH_PATH(repo, "src/a.rs"), RS_INC_A); + th_write_file(TH_PATH(repo, "src/b.rs"), RS_INC_B); + th_write_file(TH_PATH(repo, "src/user.rs"), RS_INC_USER); + th_write_file(TH_PATH(repo, "src/rows.rs"), RS_INC_ROWS); + char inc_db[512]; + char full_db[512]; + snprintf(inc_db, sizeof(inc_db), "%s/inc.db", tmp); + snprintf(full_db, sizeof(full_db), "%s/full.db", tmp); + ASSERT_EQ(dm_index(repo, inc_db, NULL), 0); + const char *file = "src.lib.rs.__file__"; + const char *user_rs = "src/user.rs"; + /* the preconditions the steps below move away from */ + ASSERT_TRUE(rs_edge_is(inc_db, file, "a.Target", RS_EXACT)); + ASSERT_TRUE(rs_edge_is(inc_db, "user.User", "a.Target", RS_UNIQUE)); + ASSERT_TRUE(rs_edge_is(inc_db, "user.User", "a.Target.go", RS_EXACT)); + ASSERT_TRUE(rs_edge_is(inc_db, "user.User", "a.Target.stop", RS_EXACT)); + ASSERT_TRUE(rs_row_is(inc_db, user_rs, "Dup", "ambiguous")); /* a::Dup and b::Dup */ + ASSERT_TRUE(rs_row_is(inc_db, user_rs, "Missing", "missing")); + ASSERT_TRUE(rs_row_is(inc_db, "src/rows.rs", "crate::a::Target::twin", "ambiguous")); + + /* a doc edit re-extracts the file alone */ + th_write_file(TH_PATH(repo, "src/user.rs"), RS_INC_USER_DOC); + ASSERT_EQ(dm_step(repo, inc_db, full_db, "doc edit", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0); + ASSERT_TRUE(rs_edge_is(inc_db, "user.User", "b.Other", RS_UNIQUE)); + + /* a reference in the FILE's own doc, added by an edit of that doc alone */ + th_write_file(TH_PATH(repo, "src/lib.rs"), RS_INC_LIB_DOC); + ASSERT_EQ(dm_step(repo, inc_db, full_db, "file doc edit", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), + 0); + ASSERT_TRUE(rs_edge_is(inc_db, file, "b.Other", RS_EXACT)); + ASSERT_TRUE(rs_edge_is(inc_db, file, "a.Target", RS_EXACT)); + + /* a body edit in a file others point into: its scope is unchanged, the + * edges into it stand (from an item and from a file's own doc) */ + th_write_file(TH_PATH(repo, "src/a.rs"), RS_INC_A_BODY); + ASSERT_EQ( + dm_step(repo, inc_db, full_db, "target body edit", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), + 0); + ASSERT_TRUE(rs_edge_is(inc_db, "user.User", "a.Target.go", RS_EXACT)); + ASSERT_TRUE(rs_edge_is(inc_db, file, "a.Target", RS_EXACT)); + + /* a removed member: the file with an edge to it gets a row instead */ + th_write_file(TH_PATH(repo, "src/a.rs"), RS_INC_A_NO_STOP); + ASSERT_EQ( + dm_step(repo, inc_db, full_db, "member removed", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0); + ASSERT_TRUE(rs_no_edge(inc_db, "user.User", "a.Target.stop")); + ASSERT_TRUE(rs_row_is(inc_db, user_rs, "Target::stop", "missing")); + + /* ... and a file with NO edge into the changed one, whose row names the + * removed member, is resolved again: two candidates shrink to one */ + th_write_file(TH_PATH(repo, "src/a.rs"), RS_INC_A_ONE_TWIN); + ASSERT_EQ(dm_step(repo, inc_db, full_db, "twin removed", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), + 0); + ASSERT_TRUE(rs_edge_is(inc_db, "rows.Rows", "a.Target.twin", RS_EXACT)); + + /* a removed item is not repaired file by file: one of two ambiguous + * candidates goes, the other binds */ + th_write_file(TH_PATH(repo, "src/b.rs"), RS_INC_B_NO_DUP); + ASSERT_EQ(dm_step(repo, inc_db, full_db, "item removed", CBM_INCREMENTAL_ROUTE_FORCED_FULL), 0); + ASSERT_TRUE(rs_edge_is(inc_db, "user.User", "a.Dup", RS_UNIQUE)); + + /* renaming a target */ + th_write_file(TH_PATH(repo, "src/a.rs"), RS_INC_A_RENAMED); + ASSERT_EQ(dm_step(repo, inc_db, full_db, "rename", CBM_INCREMENTAL_ROUTE_FORCED_FULL), 0); + ASSERT_TRUE(rs_row_is(inc_db, user_rs, "Target", "missing")); + ASSERT_TRUE(rs_row_is(inc_db, "src/lib.rs", "Target", "graph_gap")); /* `pub use a::Target` */ + ASSERT_TRUE(rs_edge_is(inc_db, file, "b.Other", RS_EXACT)); + + /* the manifest scopes every file of its package */ + th_write_file( + TH_PATH(repo, "Cargo.toml"), + "[package]\nname = \"inc\"\nedition = \"2021\"\n\n[dependencies]\nserde = \"1\"\n"); + ASSERT_EQ(dm_step(repo, inc_db, full_db, "manifest", CBM_INCREMENTAL_ROUTE_FORCED_FULL), 0); + + /* deleting a file others point into */ + ASSERT_EQ(unlink(TH_PATH(repo, "src/b.rs")), 0); + ASSERT_EQ(dm_step(repo, inc_db, full_db, "delete file", CBM_INCREMENTAL_ROUTE_FORCED_FULL), 0); + ASSERT_TRUE(rs_no_edge(inc_db, "user.User", "b.Other")); + ASSERT_EQ(dm_mentions_from(inc_db, file), 0); + + dm_unlink_db(inc_db); + dm_unlink_db(full_db); + th_rmtree(tmp); + PASS(); +} + +/* ── the parallel path == the sequential path ────────────────────── */ + +enum { RS_RING = 64 }; /* above MIN_FILES_FOR_PARALLEL */ + +/* Every module documents its successor (through `crate`), a shared target (by + * a glob, by a path into its methods) and a name nothing declares; the root + * documents the hub in its own doc. */ +static void rs_write_ring_fixture(const char *repo) { + char path[512]; + char body[1024]; + char lib[8192]; + size_t w = (size_t)snprintf(lib, sizeof(lib), "//! The ring's [`hub::Hub`].\npub mod hub;\n"); + for (int i = 0; i < RS_RING; i++) { + w += (size_t)snprintf(lib + w, sizeof(lib) - w, "pub mod m%02d;\n", i); + snprintf(path, sizeof(path), "%s/src/m%02d.rs", repo, i); + snprintf(body, sizeof(body), + "use crate::hub::*;\n" + "/// Next [crate::m%02d::C%02d], shared [`Hub`], [`Hub::run`], [`Nothing%02d`].\n" + "pub struct C%02d;\n", + (i + 1) % RS_RING, (i + 1) % RS_RING, i, i); + th_write_file(path, body); + } + th_write_file(TH_PATH(repo, "src/lib.rs"), lib); + th_write_file(TH_PATH(repo, "src/hub.rs"), "/// The hub.\n" + "pub struct Hub;\n" + "impl Hub {\n" + " /// Runs.\n" + " pub fn run(&self) {}\n" + "}\n"); + th_write_file(TH_PATH(repo, "Cargo.toml"), "[package]\nname = \"ring\"\nedition = \"2021\"\n"); +} + +TEST(doc_mentions_rs_parallel_equals_sequential) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dmrs_par_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[400]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + rs_write_ring_fixture(repo); + char par_db[512]; + char seq_db[512]; + snprintf(par_db, sizeof(par_db), "%s/par.db", tmp); + snprintf(seq_db, sizeof(seq_db), "%s/seq.db", tmp); + ASSERT_EQ(dm_workers_agree(repo, par_db, seq_db), 0); + /* per module: its successor, the hub, the hub's method; the root: the hub */ + ASSERT_EQ(dm_count(par_db, "SELECT COUNT(*) FROM edges WHERE type = 'MENTIONS'"), + RS_RING * 3 + 1); + /* per module: the undeclared name */ + ASSERT_EQ(dm_count(par_db, "SELECT COUNT(*) FROM doc_link_unresolved"), RS_RING); + ASSERT_EQ(dm_count(par_db, "SELECT COUNT(*) FROM doc_link_unresolved WHERE reason = 'missing'"), + RS_RING); + dm_unlink_db(par_db); + dm_unlink_db(seq_db); + th_rmtree(tmp); + PASS(); +} + +SUITE(doc_mentions_rs) { + dm_ship_held_families(); + RUN_TEST(doc_mentions_rs_tokens); + RUN_TEST(doc_mentions_rs_item_doc_is_not_file_doc); + RUN_TEST(doc_mentions_rs_untrusted_text); + RUN_TEST(doc_mentions_rs_parse_path); + RUN_TEST(doc_mentions_rs_scope_blob); + RUN_TEST(doc_mentions_rs_scope_parse_errors); + RUN_TEST(doc_mentions_rs_manifest_blob); + RUN_TEST(doc_mentions_rs_families_and_tiers); + RUN_TEST(doc_mentions_rs_file_level_docs); + RUN_TEST(doc_mentions_rs_lookup_levels); + RUN_TEST(doc_mentions_rs_paths_and_modules); + RUN_TEST(doc_mentions_rs_members_and_kinds); + RUN_TEST(doc_mentions_rs_reasons); + RUN_TEST(doc_mentions_rs_untrusted_structure); + RUN_TEST(doc_mentions_rs_scope_delta_rules); + RUN_TEST(doc_mentions_rs_incremental_equals_full); + RUN_TEST(doc_mentions_rs_parallel_equals_sequential); +} diff --git a/tests/test_main.c b/tests/test_main.c index 746ee2bc3..ae29520be 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -1827,6 +1827,7 @@ extern void suite_doc_links_rst(void); extern void suite_doc_links_adoc(void); extern void suite_doc_mentions_go(void); extern void suite_doc_mentions_jvm(void); +extern void suite_doc_mentions_rs(void); extern void suite_infrascan(void); extern void suite_cli(void); extern void suite_agent_clients(void); @@ -2300,6 +2301,7 @@ int main(int argc, char **argv) { RUN_SELECTED_SUITE(doc_links_adoc); RUN_SELECTED_SUITE(doc_mentions_go); RUN_SELECTED_SUITE(doc_mentions_jvm); + RUN_SELECTED_SUITE(doc_mentions_rs); /* Infrastructure scanning */ RUN_SELECTED_SUITE(infrascan); From ef5359e3eedce846ddef028fc716faadecfeef1e Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 10 Oct 2026 11:58:37 +0200 Subject: [PATCH 04/17] extract(perl): a module's POD that no sub holds is its File node's doc A POD section whose =head / =item names a sub is that sub's doc (#2459). Everything else (NAME, DESCRIPTION, SEE ALSO, the text of a section around the subs it holds, sections that name an attribute or helper without a node) was stored nowhere, so nothing could read it. It is now the file's module doc, stored on the File node as Go package comments and Rust inner docs are: every POD block with the sections a sub's doc holds cut out, each remaining run as it stands in the source, in document order and joined by a blank line (a sub's doc joins its sections the same way); a run of only =cut, =pod, =over and =back lines is dropped. A section is marked when a sub's doc takes it, so the module doc is built after the definitions and the two never hold the same text. Its first line is noted for the doc-link layer. Mojolicious at 93f591742a7d: L< codes in stored docs 624 (subs, unchanged) + 1,595 (File nodes of 112 modules) = 2,219, the count of L<> codes in the POD of its .pm and .pl files, none stored twice. Test: doc_perl_module_doc_is_the_pod_no_sub_holds (the module doc's exact text, the sub's doc unchanged, no module doc when every section is a sub's); red at its first assertion with the module doc not built. Signed-off-by: Martin Vogel --- internal/cbm/extract_defs.c | 150 ++++++++++++++++++++++++++++++++++-- tests/test_extraction.c | 59 ++++++++++++++ 2 files changed, 203 insertions(+), 6 deletions(-) diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index 2a3b9d14d..5917c0804 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -2327,6 +2327,7 @@ typedef struct { size_t name_len; uint32_t sb; uint32_t eb; + bool used; /* a sub's doc holds it (pod_doc_by_name): not the module's */ } pod_sec_t; typedef struct { @@ -2587,15 +2588,16 @@ static void pod_index_block(CBMExtractCtx *ctx, pod_index_t *ix, TSNode pod) { continue; } uint32_t se = pod_section_end(cmds, n, ci, pe); - pod_sec_t sec = {nm, nl, cmds[ci].sb, pod_trim_end(ctx->source, cmds[ci].sb, se)}; + pod_sec_t sec = {nm, nl, cmds[ci].sb, pod_trim_end(ctx->source, cmds[ci].sb, se), false}; pod_index_push(a, ix, &sec); } } -/* The file's POD sections by name, built on the first Perl sub. */ -static const pod_index_t *pod_index(CBMExtractCtx *ctx) { +/* The file's POD sections by name, built on the first Perl sub (or for the + * module doc of a file without one). */ +static pod_index_t *pod_index(CBMExtractCtx *ctx) { if (ctx->doc_pod_index) { - return (const pod_index_t *)ctx->doc_pod_index; + return (pod_index_t *)ctx->doc_pod_index; } pod_index_t *ix = (pod_index_t *)cbm_arena_alloc(doc_scratch(ctx), sizeof(pod_index_t)); if (!ix) { @@ -2623,7 +2625,7 @@ static bool pod_sec_names(const pod_sec_t *s, const char *name, size_t nl) { /* Every POD section whose =head / =item names the sub exactly, joined in * document order; NULL when none does. */ static const char *pod_doc_by_name(CBMExtractCtx *ctx, const char *name) { - const pod_index_t *ix = pod_index(ctx); + pod_index_t *ix = pod_index(ctx); if (!ix || !name || !name[0]) { return NULL; } @@ -2643,10 +2645,11 @@ static const char *pod_doc_by_name(CBMExtractCtx *ctx, const char *name) { } size_t w = 0; for (int i = 0; i < ix->count; i++) { - const pod_sec_t *s = &ix->items[i]; + pod_sec_t *s = &ix->items[i]; if (!pod_sec_names(s, name, nl)) { continue; } + s->used = true; if (w > 0) { buf[w++] = '\n'; buf[w++] = '\n'; @@ -2658,6 +2661,137 @@ static const char *pod_doc_by_name(CBMExtractCtx *ctx, const char *name) { return buf; } +/* True when src[b, e) holds a paragraph that is text, or a =head / =item + * command: not only =cut, =pod, =over and =back lines. */ +static bool pod_run_has_text(const char *src, uint32_t b, uint32_t e) { + uint32_t pos = b; + uint32_t sb = 0; + uint32_t eb = 0; + int depth = 0; + while (pod_next_para(src, &pos, e, &sb, &eb)) { + if (src[sb] != '=' || sb + SKIP_ONE >= eb || !isalpha((unsigned char)src[sb + SKIP_ONE])) { + return true; + } + pod_cmd_t c = pod_classify(src, sb, eb, &depth); + if (c.type == POD_HEAD || c.type == POD_ITEM || c.type == POD_OTHER) { + return true; + } + } + return false; +} + +typedef struct { + CBMExtractCtx *ctx; + char *buf; + size_t w; + uint32_t first_line; /* source line of the first run kept, 0 before it */ + TSNode block; + int next_sec; /* the first index section not yet passed: blocks go in order */ +} pod_module_t; + +/* Append src[b, e) of the current block to the module doc, after a blank + * line, unless it holds no text. */ +static void pod_module_run(pod_module_t *m, uint32_t b, uint32_t e) { + const char *src = m->ctx->source; + while (b < e) { /* leading blank lines */ + uint32_t le = b; + while (le < e && src[le] != '\n') { + le++; + } + if (!pod_line_is_blank(src, b, le)) { + break; + } + b = le < e ? le + SKIP_ONE : e; + } + e = pod_trim_end(src, b, e); + if (b >= e || !pod_run_has_text(src, b, e)) { + return; + } + if (m->first_line == 0) { + uint32_t line = ts_node_start_point(m->block).row + SKIP_ONE; + for (uint32_t k = ts_node_start_byte(m->block); k < b; k++) { + line += src[k] == '\n' ? SKIP_ONE : 0; + } + m->first_line = line; + } + if (m->w > 0) { + m->buf[m->w++] = '\n'; + m->buf[m->w++] = '\n'; + } + memcpy(m->buf + m->w, src + b, e - b); + m->w += e - b; +} + +/* Perl's module doc, for the File node: the file's POD outside the sections a + * sub's doc holds, so that no reference is read twice. Every POD block with + * those sections cut out, each remaining run as it is in the source, in + * document order and joined by a blank line (a sub's doc joins its sections + * the same way); a run of only =cut / =pod / =over / =back lines is dropped. + * Built after the definitions, which mark the sections they take. NULL when + * nothing is left. */ +static const char *doc_perl_module(CBMExtractCtx *ctx) { + pod_index_t *ix = pod_index(ctx); + if (!ix) { + return NULL; + } + size_t total = 0; + int blocks = 0; + TSTreeCursor cur = ts_tree_cursor_new(ctx->root); + if (ts_tree_cursor_goto_first_child(&cur)) { + do { + TSNode ch = ts_tree_cursor_current_node(&cur); + if (strcmp(ts_node_type(ch), "pod") == 0) { + total += ts_node_end_byte(ch) - ts_node_start_byte(ch); + blocks++; + } + } while (ts_tree_cursor_goto_next_sibling(&cur)); + } + if (blocks == 0) { + ts_tree_cursor_delete(&cur); + return NULL; + } + /* each block yields at most one run more than the sections it loses */ + size_t runs = (size_t)blocks + (size_t)ix->count; + pod_module_t m = {.ctx = ctx}; + m.buf = (char *)cbm_arena_alloc(ctx->arena, total + (runs * PAIR_LEN) + SKIP_ONE); + if (!m.buf) { + ts_tree_cursor_delete(&cur); + return NULL; + } + ts_tree_cursor_reset(&cur, ctx->root); + if (ts_tree_cursor_goto_first_child(&cur)) { + do { + TSNode ch = ts_tree_cursor_current_node(&cur); + if (strcmp(ts_node_type(ch), "pod") != 0) { + continue; + } + m.block = ch; + uint32_t pb = ts_node_start_byte(ch); + uint32_t pe = ts_node_end_byte(ch); + uint32_t at = pb; + /* the index holds the blocks' sections in document order */ + for (; m.next_sec < ix->count && ix->items[m.next_sec].sb < pe; m.next_sec++) { + const pod_sec_t *s = &ix->items[m.next_sec]; + if (!s->used || s->sb < pb) { + continue; + } + if (s->sb > at) { + pod_module_run(&m, at, s->sb); + } + at = s->eb > at ? s->eb : at; + } + pod_module_run(&m, at, pe); + } while (ts_tree_cursor_goto_next_sibling(&cur)); + } + ts_tree_cursor_delete(&cur); + if (m.w == 0) { + return NULL; + } + m.buf[m.w] = '\0'; + cbm_doclink_note_doc_line(ctx, m.buf, m.first_line); + return m.buf; +} + /* ── Doc entry points ── */ /* go/doc: a type spec's own doc, else its declaration's (the doc of a @@ -12065,6 +12199,10 @@ void cbm_extract_definitions(CBMExtractCtx *ctx) { ctx->result->module_doc = extract_module_doc(ctx); cbm_extract_definitions_without_module(ctx); + if (ctx->language == CBM_LANG_PERL) { + /* after the subs: the POD sections they hold are not the module's */ + ctx->result->module_doc = doc_perl_module(ctx); + } if (ctx->language == CBM_LANG_JAVASCRIPT || ctx->language == CBM_LANG_TYPESCRIPT || ctx->language == CBM_LANG_TSX || ctx->language == CBM_LANG_ARKTS) { diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 1431cefe7..4c62bddc7 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -8610,6 +8610,64 @@ TEST(doc_perl_pod_section_by_sub_name) { PASS(); } +/* A Perl file's module doc (its File node's docstring) is the POD that no + * sub's doc holds: each run as it stands, joined by a blank line, so that a + * reference is read once. */ +TEST(doc_perl_module_doc_is_the_pod_no_sub_holds) { + CBMFileResult *r = extract("package Knob;\n" + "\n" + "=head1 NAME\n" + "\n" + "Knob - see L.\n" + "\n" + "=cut\n" + "\n" + "sub turn { return 1; }\n" + "\n" + "1;\n" + "__END__\n" + "\n" + "=head1 METHODS\n" + "\n" + "The knob's methods.\n" + "\n" + "=head2 turn\n" + "\n" + "Turns the knob once.\n" + "\n" + "=head2 spin\n" + "\n" + "Not a sub here.\n" + "\n" + "=cut\n", + CBM_LANG_PERL, "t", "Knob.pm"); + ASSERT_NOT_NULL(r); + ASSERT_STR_EQ(r->module_doc, "=head1 NAME\n\nKnob - see L.\n\n=cut\n\n" + "=head1 METHODS\n\nThe knob's methods.\n\n" + "=head2 spin\n\nNot a sub here.\n\n=cut"); + const CBMDefinition *d = doc_def(r, "Function", "turn"); + ASSERT_NOT_NULL(d); + ASSERT_STR_EQ(d->docstring, "=head2 turn\n\nTurns the knob once."); + cbm_free_result(r); + + /* all of the POD is a sub's: no module doc */ + r = extract("package Knob;\n" + "sub turn { return 1; }\n" + "1;\n" + "__END__\n" + "\n" + "=head2 turn\n" + "\n" + "Turns.\n" + "\n" + "=cut\n", + CBM_LANG_PERL, "t", "Knob.pm"); + ASSERT_NOT_NULL(r); + ASSERT_NULL(r->module_doc); + cbm_free_result(r); + PASS(); +} + TEST(doc_long_comment_kept_whole) { /* One 12-line block comment well past the old 500-byte cut, kept byte for * byte. */ @@ -10847,6 +10905,7 @@ SUITE(extraction) { RUN_TEST(doc_kotlin_kdoc_folded_after_imports); RUN_TEST(doc_c_typedef_struct); RUN_TEST(doc_perl_pod_section_by_sub_name); + RUN_TEST(doc_perl_module_doc_is_the_pod_no_sub_holds); RUN_TEST(doc_long_comment_kept_whole); RUN_TEST(doc_divider_only_run_is_not_doc); RUN_TEST(extract_ts_decorators_survive_interleaved_comment); From f4277ae783ded09d914930f7771ae8fdfeca0a2f Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 10 Oct 2026 13:19:21 +0200 Subject: [PATCH 05/17] doc-links(perl): POD L<> codes become MENTIONS edges, held until their audit A sub's POD section and the module's own POD (the File node's doc) name code with L<> codes: L, L, L, with a `text|` prefix or not. doclink_perl.c reads them the way perlpodspec and Pod::Simple do (verbatim paragraphs and data regions hold none, an L inside an L is no link, a URL is the core's href family) and writes each file's scope: its packages, its subs and its headings. Lines are exact: a sub's doc is matched against the file's sections, and the module doc's runs (extract_defs.c) are found again in the source in order; a module doc is POD from its start, so a run that begins with text is read too. doc_links_perl.c resolves them: - a module is the file that declares its package; of several, the one the library layout names (.../Name/Space.pm), else ambiguous. The target is the file's File node, as the field test's audited prototype has it (a Module node is missing where a same-named Folder holds its qualified name, lib/Mojo.pm next to lib/Mojo/). - L: the sub its heading names when the module's file defines that sub; any other section is a place in the module's documentation, the module. - L: a sub the heading names; a heading that names no sub is the page itself (local, neither edge nor row); a section the page has no heading for is missing. - a module the repository does not declare (CPAN, the core, a .pod page: discovery keeps no .pod file) and a man page are external; test code (t/, xt/, *.t, and cbm_is_test_file, which has no Perl rule) binds only from test code. There is no scope_delta hook yet: a change to a file's packages, subs or headings re-resolves every Perl reference. The family is held (ships=false) until its held-out audit passes; the suite ships it through the test seam. Tests (doc_mentions_perl): tokens with their lines, file-level and sub-level, each read once; every resolution rule; incremental equals full (a module doc edit and an added module-level reference repair the file, a heading change is a full run); 60 modules give the same edges and rows with one worker and several. Each rule red with it reverted (the module doc's line mapping, a module run read from its start, the missing section, the library layout). Mojolicious at 93f591742a7d (the family shipped, a measurement build) against the field test's prototype records of its .pm/.pl files: 950 (source, target) pairs both have; the 197 only the prototype has are all references within one file (570), which the core makes no edge of; 8 only here come from the extensionless scripts the prototype did not read. Every prototype row has its row here: external 102, missing 1, and the 172 targets in .pod pages external by the ruling for .pod pages. Signed-off-by: Martin Vogel --- Makefile.cbm | 4 +- internal/cbm/doclink.c | 6 + internal/cbm/doclink.h | 44 + internal/cbm/doclink_perl.c | 1677 +++++++++++++++++++++++++++++ src/pipeline/doc_links.c | 2 +- src/pipeline/doc_links.h | 1 + src/pipeline/doc_links_perl.c | 489 +++++++++ tests/test_doc_mentions_helpers.h | 2 +- tests/test_doc_mentions_perl.c | 292 +++++ tests/test_main.c | 2 + 10 files changed, 2516 insertions(+), 3 deletions(-) create mode 100644 internal/cbm/doclink_perl.c create mode 100644 src/pipeline/doc_links_perl.c create mode 100644 tests/test_doc_mentions_perl.c diff --git a/Makefile.cbm b/Makefile.cbm index 43999c35c..b8aa4f71e 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -389,6 +389,7 @@ EXTRACTION_SRCS = \ $(CBM_DIR)/doclink_go.c \ $(CBM_DIR)/doclink_jvm.c \ $(CBM_DIR)/doclink_rs.c \ + $(CBM_DIR)/doclink_perl.c \ $(CBM_DIR)/sql_values.c \ $(CBM_DIR)/helpers.c \ $(CBM_DIR)/cpp_branch_views.c \ @@ -518,6 +519,7 @@ PIPELINE_SRCS = \ src/pipeline/doc_links_go.c \ src/pipeline/doc_links_jvm.c \ src/pipeline/doc_links_rs.c \ + src/pipeline/doc_links_perl.c \ src/pipeline/doc_links_msbuild.c \ src/pipeline/pass_route_nodes.c \ src/pipeline/pass_enrichment.c \ @@ -780,7 +782,7 @@ TEST_DISCOVER_SRCS = \ TEST_GRAPH_BUFFER_SRCS = tests/test_graph_buffer.c -TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_pipeline_conventions.c tests/test_pipeline_frozen.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_doclinks.c tests/test_doc_mentions.c tests/test_doc_links_md.c tests/test_doc_links_pdf.c tests/test_doc_links_rst.c tests/test_doc_links_adoc.c tests/test_doc_mentions_go.c tests/test_doc_mentions_jvm.c tests/test_doc_mentions_rs.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c +TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_pipeline_conventions.c tests/test_pipeline_frozen.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_doclinks.c tests/test_doc_mentions.c tests/test_doc_links_md.c tests/test_doc_links_pdf.c tests/test_doc_links_rst.c tests/test_doc_links_adoc.c tests/test_doc_mentions_go.c tests/test_doc_mentions_jvm.c tests/test_doc_mentions_rs.c tests/test_doc_mentions_perl.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c TEST_WATCHER_SRCS = tests/test_watcher.c diff --git a/internal/cbm/doclink.c b/internal/cbm/doclink.c index 92892d6e3..4957e5489 100644 --- a/internal/cbm/doclink.c +++ b/internal/cbm/doclink.c @@ -211,6 +211,12 @@ static const doclink_lang_t DOCLINK_LANGS[] = { .scan_scope = cbm_doclink_rs_manifest_scan_scope, .scope_tag = CBM_DOCLINK_RS_SCOPE_TAG, .portable_scope = cbm_doclink_rs_portable_scope}, + /* Perl: POD L<> codes; the scope blob carries no lines, so it persists as + * it is */ + {.lang = CBM_LANG_PERL, + .parse_doc = cbm_doclink_perl_parse_doc, + .scan_scope = cbm_doclink_perl_scan_scope, + .scope_tag = CBM_DOCLINK_PERL_SCOPE_TAG}, }; static const doclink_lang_t *doclink_lang(CBMLanguage lang) { diff --git a/internal/cbm/doclink.h b/internal/cbm/doclink.h index 4a5127437..479d104ff 100644 --- a/internal/cbm/doclink.h +++ b/internal/cbm/doclink.h @@ -98,6 +98,8 @@ /* rust (doclink_rs.c): intra-doc links; held until their audit passes */ \ X(CBM_DOCLINK_RS_INTRA_DOC, CBM_LANG_RUST, "intra_doc", false, false, "MENTIONS") \ X(CBM_DOCLINK_RS_LINK_DEF, CBM_LANG_RUST, "link_def", false, false, "MENTIONS") \ + /* perl (doclink_perl.c): POD L<> codes; held until their audit passes */ \ + X(CBM_DOCLINK_PERL_POD_L, CBM_LANG_PERL, "pod_L", false, false, "MENTIONS") \ /* markdown (doclink_md.c): what a document's text names explicitly */ \ X(CBM_DOCLINK_MD_LINK, CBM_LANG_MARKDOWN, "link", false, true, "MENTIONS") \ X(CBM_DOCLINK_MD_PATH, CBM_LANG_MARKDOWN, "path", false, true, "MENTIONS") \ @@ -535,5 +537,47 @@ typedef enum { * are read. */ CBMDocLinkRsPathStatus cbm_doclink_rs_parse_path(const char *raw, size_t len, bool lenient, CBMDocLinkRsPath *out); +/* ── Perl POD (doclink_perl.c) ─────────────────────────────────────── */ + +void cbm_doclink_perl_parse_doc(CBMExtractCtx *ctx, const CBMDefinition *def, const char *doc, + uint32_t doc_line); +const char *cbm_doclink_perl_scan_scope(CBMExtractCtx *ctx); + +/* The Perl scope blob starts with this tag line. */ +#define CBM_DOCLINK_PERL_SCOPE_TAG "perl1" + +enum { + /* Longest L<> content kept as a token's raw text; a longer one is cut and + * marked malformed (its raw text then starts with `L<`). */ + CBM_DOCLINK_PERL_RAW_MAX = 512, + /* Buffer for a page name, a section text or a heading key. */ + CBM_DOCLINK_PERL_KEY_MAX = 256, +}; + +/* A POD text as it renders: formatting codes replaced by their content + * (X<> and Z<> by nothing, E<> by its character), white-space runs as one + * space, none at the ends. Headings and the section part of a link are + * compared in this form. Writes a NUL-terminated string into `out` (cut at + * cap - 1, never inside a UTF-8 character) and returns its length. */ +size_t cbm_doclink_perl_plain_text(const char *in, size_t len, char *out, size_t cap); + +typedef enum { + CBM_PERL_LINK_UNPARSEABLE = 0, /* malformed, nested L<>, or no page name */ + CBM_PERL_LINK_PAGE, /* L or L */ + CBM_PERL_LINK_SECTION, /* L, L<"section">: the same page */ + CBM_PERL_LINK_MAN, /* L: a man page */ +} CBMPerlLinkKind; + +/* One pod_L token's raw text, split the way perlpodspec splits it. */ +typedef struct { + CBMPerlLinkKind kind; + char page[CBM_DOCLINK_PERL_KEY_MAX]; /* "" for a section of the same page */ + char section[CBM_DOCLINK_PERL_KEY_MAX]; /* "" when only a page is named */ +} CBMPerlLink; + +/* Split a pod_L token's raw text: the first top-level `|` ends the link text, + * the first top-level `/` after it separates page and section, quotes around + * a section are dropped. False (kind UNPARSEABLE) for anything else. */ +bool cbm_doclink_perl_parse_link(const char *raw, CBMPerlLink *out); #endif /* CBM_DOCLINK_H */ diff --git a/internal/cbm/doclink_perl.c b/internal/cbm/doclink_perl.c new file mode 100644 index 000000000..42bdb72de --- /dev/null +++ b/internal/cbm/doclink_perl.c @@ -0,0 +1,1677 @@ +/* + * doclink_perl.c — Perl POD `L<>` references and the Perl doc-link scope. + * + * Tokens. A sub's stored doc is every POD section whose =headN / =item heading + * names it (extract_defs.c), verbatim and joined by a blank line. Its L<> + * formatting codes are the references, read the way perlpodspec and + * Pod::Simple read them: + * - a paragraph is a run of non-blank lines. Codes are looked for in ordinary + * paragraphs and in the text of =headN and =item. A verbatim paragraph + * (first byte a space or tab), a =for paragraph and a =begin .. =end + * region of a format without a leading colon are data and hold no code; + * - a code is a capital letter and `<`. `X<< text >>` (two or more brackets + * and whitespace) ends at whitespace and as many `>`. Codes nest; an L + * inside an L is not a link (perlpodspec: links must not nest); + * - an L<> whose target (what follows `text|`) matches `\w+:[^:\s]\S*` is a + * URL: the core's href family. Every other one is a pod_L token whose raw + * text is the code's content as written, white space squeezed. A code the + * paragraph ends before, and one longer than CBM_DOCLINK_PERL_RAW_MAX, + * keeps its opening `L<` in the raw text: nothing parses that, so the + * resolver records it as unparseable. + * A doc that does not start with a POD command is a `#` comment and has no + * formatting codes at all. + * + * Lines. The stored doc carries no position, and several sections can be + * joined in it, so every section of the file is found again here (the same + * paragraph, command and section rules as extract_defs.c) and the doc is + * matched against them byte for byte: a reference then gets the source line it + * is written on. A part of a doc that matches no section keeps the line the + * core passed (the definition's). That needs the file's sections once per + * file, and parse_doc is called per definition: the call for the file's first + * documented definition harvests the whole file, the later ones return. + * + * Scope blob (one record per line, tab-separated, first line "perl1"; no line + * numbers, so it is persisted as it is): + * E the tree has a parse error: which package a sub belongs to + * is not trusted + * P name a package, class or role the file declares; document + * order, each name once + * S pkg name a sub that has a definition, in definition order; pkg is + * the 0-based index of its P record, `-` before any package + * (main), `?` when it could not be placed + * H sub key a POD heading (=headN / =item): `key` is its text with + * formatting codes reduced to what they render + * (cbm_doclink_perl_plain_text) and an item's bullet + * dropped; `sub` is the identifier the heading names by + * extract_defs.c's rule, `-` when it names none + * Packages and the placement of subs come from the tree; the subs themselves + * from the file's definitions (so a sub listed here has a node); the headings + * from a line scan of the file, which is what perldoc reads. + */ +#include "doclink.h" + +#include "arena.h" +#include "foundation/constants.h" +#include "tree_sitter/api.h" + +#include +#include +#include +#include + +enum { + PL_CODE_DEPTH = 32, /* open formatting codes; deeper nesting is malformed */ + PL_INIT_CAP = 32, + PL_SB_INIT = 1024, + PL_ENT_MAX = 12, /* longest E<> body that is decoded */ + PL_MALFORMED = 2, /* strlen("L<"): what a malformed code's raw text starts with */ + PL_UTF8_CONT = 0x80, /* 10xxxxxx */ + PL_UTF8_CONT_MASK = 0xC0, + PL_UTF8_LEAD2 = 0xC0, + PL_UTF8_LEAD3 = 0xE0, + PL_UTF8_LEAD4 = 0xF0, + PL_UTF8_LEAD_END = 0xF8, + PL_ASCII_DEL = 0x7F, + PL_HEAD_WORD = 5, /* strlen("headN") */ + PL_NAME_MAX = 255, +}; + +/* ── Characters ──────────────────────────────────────────────────── */ + +static bool pl_blank(char c) { + return c == ' ' || c == '\t' || c == '\r'; +} + +static bool pl_space(char c) { + return c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '\f' || c == '\v'; +} + +static bool pl_word(char c) { + return isalnum((unsigned char)c) || c == '_'; +} + +static bool pl_upper(char c) { + return c >= 'A' && c <= 'Z'; +} + +/* A control byte that is not white space: never copied into a blob record or + * a raw text (a NUL would end either). */ +static bool pl_control(char c) { + unsigned char u = (unsigned char)c; + return (u < ' ' && !pl_space(c)) || u == PL_ASCII_DEL; +} + +static CBMArena *pl_scratch(CBMExtractCtx *ctx) { + return ctx->scratch ? ctx->scratch : ctx->arena; +} + +/* ── Paragraphs ──────────────────────────────────────────────────── */ + +/* Next paragraph (a maximal run of non-blank lines) of s[*pos, end): [*pb, + * *pe). The same rule as extract_defs.c's pod_next_para. */ +static bool pl_next_para(const char *s, size_t *pos, size_t end, size_t *pb, size_t *pe) { + size_t i = *pos; + bool in = false; + size_t start = 0; + while (i < end) { + const char *nl = (const char *)memchr(s + i, '\n', end - i); + size_t le = nl ? (size_t)(nl - s) : end; + bool blank = true; + for (size_t k = i; k < le && blank; k++) { + blank = pl_blank(s[k]); + } + if (!blank && !in) { + in = true; + start = i; + } + if (blank && in) { + *pb = start; + *pe = i; + *pos = i; + return true; + } + i = le < end ? le + SKIP_ONE : end; + } + *pos = end; + if (in) { + *pb = start; + *pe = end; + } + return in; +} + +/* True when the paragraph s[pb, pe) is a command (`=` and a letter). */ +static bool pl_is_command(const char *s, size_t pb, size_t pe) { + return pb + SKIP_ONE < pe && s[pb] == '=' && isalpha((unsigned char)s[pb + SKIP_ONE]); +} + +/* End of the command word that follows the `=` at s[pb]. */ +static size_t pl_command_word_end(const char *s, size_t pb, size_t pe) { + size_t wb = pb + SKIP_ONE; + while (wb < pe && pl_word(s[wb])) { + wb++; + } + return wb; +} + +static bool pl_word_is(const char *w, size_t n, const char *lit) { + return n == strlen(lit) && memcmp(w, lit, n) == 0; +} + +/* `headN`: the level, 0 for another word. */ +static int pl_head_level(const char *w, size_t n) { + static const char head[] = "head"; + size_t hl = sizeof(head) - SKIP_ONE; + if (n != PL_HEAD_WORD || memcmp(w, head, hl) != 0 || !isdigit((unsigned char)w[hl])) { + return 0; + } + return w[hl] - '0'; +} + +/* ── Formatting codes ────────────────────────────────────────────── */ + +typedef struct { + uint8_t width[PL_CODE_DEPTH]; /* brackets of each open code: 1 simple, more complex */ + int depth; + bool overflow; /* nested deeper than PL_CODE_DEPTH */ +} pl_codes_t; + +enum { PL_EV_TEXT = 0, PL_EV_OPEN, PL_EV_CLOSE }; + +/* A code opens at t[i]: push it and return where its content starts. */ +static size_t pl_code_open(pl_codes_t *st, const char *t, size_t i, size_t e, bool *pushed) { + size_t j = i + SKIP_ONE; + while (j < e && t[j] == '<') { + j++; + } + size_t brackets = j - i - SKIP_ONE; + size_t content = i + PAIR_LEN; + size_t width = SKIP_ONE; + if (brackets >= PAIR_LEN && j < e && pl_space(t[j])) { + while (j < e && pl_space(t[j])) { + j++; + } + content = j; + width = brackets > UINT8_MAX ? UINT8_MAX : brackets; + } + *pushed = st->depth < PL_CODE_DEPTH; + if (*pushed) { + st->width[st->depth++] = (uint8_t)width; + } else { + st->overflow = true; + } + return content; +} + +/* One lexing step at t[i], i < e (Pod::Simple's treelet tokens). + * PL_EV_OPEN a code opened at t[i]; its content starts at *next + * PL_EV_CLOSE the innermost code's content ended at i; *next follows its + * closing brackets + * PL_EV_TEXT t[i, *next) is literal */ +static int pl_code_step(pl_codes_t *st, const char *t, size_t i, size_t e, size_t *next) { + char c = t[i]; + if (pl_upper(c) && i + SKIP_ONE < e && t[i + SKIP_ONE] == '<') { + bool pushed = false; + *next = pl_code_open(st, t, i, e, &pushed); + return pushed ? PL_EV_OPEN : PL_EV_TEXT; + } + size_t need = st->depth > 0 ? st->width[st->depth - SKIP_ONE] : 0; + if (need == SKIP_ONE && c == '>') { + st->depth--; + *next = i + SKIP_ONE; + return PL_EV_CLOSE; + } + if (need > SKIP_ONE && pl_space(c)) { + size_t k = i; + while (k < e && pl_space(t[k])) { + k++; + } + size_t m = k; + while (m < e && t[m] == '>') { + m++; + } + if (m - k >= need) { + st->depth--; + *next = k + need; + return PL_EV_CLOSE; + } + *next = m; /* white space and too few brackets: literal */ + return PL_EV_TEXT; + } + *next = i + SKIP_ONE; + return PL_EV_TEXT; +} + +/* ── Bounded text output ─────────────────────────────────────────── */ + +typedef struct { + char *buf; + size_t cap; /* bytes of text that fit (the buffer holds one more) */ + size_t len; + bool space; /* a white-space run is pending */ + bool cut; /* something did not fit */ +} pl_out_t; + +/* Bytes of the UTF-8 character that `lead` starts; 1 for anything else. */ +static size_t pl_utf8_len(unsigned char lead) { + if (lead >= PL_UTF8_LEAD4 && lead < PL_UTF8_LEAD_END) { + return 4; + } + if (lead >= PL_UTF8_LEAD3 && lead < PL_UTF8_LEAD4) { + return 3; + } + if (lead >= PL_UTF8_LEAD2 && lead < PL_UTF8_LEAD3) { + return PAIR_LEN; + } + return SKIP_ONE; +} + +/* Append t[0, n): white-space runs become one space (none at the start), + * control bytes are dropped, and a character that does not fit whole ends + * the output. */ +static void pl_out_text(pl_out_t *o, const char *t, size_t n) { + for (size_t i = 0; i < n && !o->cut; i++) { + char c = t[i]; + if (pl_space(c)) { + o->space = o->len > 0; + continue; + } + if (pl_control(c)) { + continue; + } + size_t need = pl_utf8_len((unsigned char)c) + (o->space ? SKIP_ONE : 0); + if (o->len + need > o->cap) { + o->cut = true; + return; + } + if (o->space) { + o->buf[o->len++] = ' '; + o->space = false; + } + o->buf[o->len++] = c; + } +} + +static void pl_out_end(pl_out_t *o) { + o->buf[o->len] = '\0'; +} + +/* t[b, e) squeezed into out (cap + 1 bytes); *cut when it did not fit. */ +static size_t pl_squeeze(const char *t, size_t b, size_t e, char *out, size_t cap, bool *cut) { + pl_out_t o = {.buf = out, .cap = cap}; + pl_out_text(&o, t + b, e - b); + pl_out_end(&o); + *cut = o.cut; + return o.len; +} + +/* ── Plain text ──────────────────────────────────────────────────── */ + +/* Append the UTF-8 form of `cp`; false for what is no character. */ +static bool pl_out_codepoint(pl_out_t *o, unsigned long cp) { + enum { + CP_1 = 0x80, + CP_2 = 0x800, + CP_3 = 0x10000, + CP_MAX = 0x10FFFF, + CP_SURR_LO = 0xD800, + CP_SURR_HI = 0xDFFF, + SHIFT = 6, + LOW = 0x3F, + }; + if (cp == 0 || cp > CP_MAX || (cp >= CP_SURR_LO && cp <= CP_SURR_HI)) { + return false; + } + char u[4]; + size_t n = 0; + if (cp < CP_1) { + u[n++] = (char)cp; + } else if (cp < CP_2) { + u[n++] = (char)(PL_UTF8_LEAD2 | (cp >> SHIFT)); + u[n++] = (char)(PL_UTF8_CONT | (cp & LOW)); + } else if (cp < CP_3) { + u[n++] = (char)(PL_UTF8_LEAD3 | (cp >> (SHIFT + SHIFT))); + u[n++] = (char)(PL_UTF8_CONT | ((cp >> SHIFT) & LOW)); + u[n++] = (char)(PL_UTF8_CONT | (cp & LOW)); + } else { + u[n++] = (char)(PL_UTF8_LEAD4 | (cp >> (SHIFT + SHIFT + SHIFT))); + u[n++] = (char)(PL_UTF8_CONT | ((cp >> (SHIFT + SHIFT)) & LOW)); + u[n++] = (char)(PL_UTF8_CONT | ((cp >> SHIFT) & LOW)); + u[n++] = (char)(PL_UTF8_CONT | (cp & LOW)); + } + pl_out_text(o, u, n); + return true; +} + +/* The number an E<> body writes (decimal, 0x hex, 0 octal); false for a name. */ +static bool pl_entity_number(const char *e, size_t n, unsigned long *cp) { + enum { DEC = 10, HEX = 16, OCT = 8 }; + int base = DEC; + size_t i = 0; + if (n > PAIR_LEN && e[0] == '0' && (e[1] == 'x' || e[1] == 'X')) { + base = HEX; + i = PAIR_LEN; + } else if (n > SKIP_ONE && e[0] == '0') { + base = OCT; + i = SKIP_ONE; + } + unsigned long v = 0; + for (; i < n; i++) { + int d = isdigit((unsigned char)e[i]) ? e[i] - '0' + : isxdigit((unsigned char)e[i]) ? tolower((unsigned char)e[i]) - 'a' + DEC + : base; + if (d >= base) { + return false; + } + v = (v * (unsigned long)base) + (unsigned long)d; + } + *cp = v; + return n > 0; +} + +/* Append what E renders; an unknown body stays as written. */ +static void pl_out_entity(pl_out_t *o, const char *body, size_t n, bool closed) { + static const struct { + const char *name; + char ch; + } ents[] = {{"lt", '<'}, {"gt", '>'}, {"verbar", '|'}, + {"sol", '/'}, {"quot", '"'}, {"amp", '&'}}; + for (size_t i = 0; closed && i < sizeof(ents) / sizeof(ents[0]); i++) { + if (pl_word_is(body, n, ents[i].name)) { + pl_out_text(o, &ents[i].ch, SKIP_ONE); + return; + } + } + unsigned long cp = 0; + if (closed && pl_entity_number(body, n, &cp) && pl_out_codepoint(o, cp)) { + return; + } + pl_out_text(o, "E<", PL_MALFORMED); + pl_out_text(o, body, n); + if (closed) { + pl_out_text(o, ">", SKIP_ONE); + } +} + +size_t cbm_doclink_perl_plain_text(const char *in, size_t len, char *out, size_t cap) { + if (!out || cap == 0) { + return 0; + } + pl_out_t o = {.buf = out, .cap = cap - SKIP_ONE}; + pl_codes_t st = {.depth = 0}; + int drop_depth = 0; /* an X<> or Z<> is open at this depth: no text */ + int ent_depth = 0; /* an E<> is open at this depth */ + char ent[PL_ENT_MAX]; + size_t ent_len = 0; + size_t i = 0; + while (in && i < len && !st.overflow) { + size_t next = i; + int ev = pl_code_step(&st, in, i, len, &next); + if (ev == PL_EV_OPEN && drop_depth == 0 && ent_depth == 0) { + if (in[i] == 'X' || in[i] == 'Z') { + drop_depth = st.depth; + } else if (in[i] == 'E') { + ent_depth = st.depth; + ent_len = 0; + } + } else if (ev == PL_EV_CLOSE && drop_depth != 0) { + drop_depth = st.depth < drop_depth ? 0 : drop_depth; + } else if (ev == PL_EV_CLOSE && ent_depth != 0 && st.depth < ent_depth) { + pl_out_entity(&o, ent, ent_len, true); + ent_depth = 0; + } else if (ev == PL_EV_TEXT && drop_depth == 0 && ent_depth == 0) { + pl_out_text(&o, in + i, next - i); + } else if (ev == PL_EV_TEXT && drop_depth == 0) { + for (size_t k = i; k < next && ent_len < sizeof(ent); k++) { + ent[ent_len++] = in[k]; + } + } + i = next; + } + if (ent_depth != 0) { + pl_out_entity(&o, ent, ent_len, false); + } + if (in && i < len) { + pl_out_text(&o, in + i, len - i); /* nested too deep: the rest as written */ + } + pl_out_end(&o); + return o.len; +} + +/* ── Link grammar (perlpodspec; Pod::Simple's _treat_Ls) ─────────── */ + +typedef struct { + size_t bar; /* first top-level `|`; SIZE_MAX: none */ + size_t slash; /* first top-level `/` after it; SIZE_MAX: none */ + bool space; /* a top-level space in a link that has no `|` */ + bool well_formed; /* every code closes, none nests too deep, no L inside */ +} pl_shape_t; + +/* Where the top-level `|` and `/` of a link's content are. */ +static pl_shape_t pl_link_shape(const char *t, size_t n) { + pl_shape_t sh = {.bar = SIZE_MAX, .slash = SIZE_MAX, .well_formed = true}; + pl_codes_t st = {.depth = 0}; + size_t i = 0; + while (i < n && sh.well_formed) { + size_t next = i; + int ev = pl_code_step(&st, t, i, n, &next); + sh.well_formed = !st.overflow && !(ev == PL_EV_OPEN && t[i] == 'L'); + for (size_t k = i; ev == PL_EV_TEXT && st.depth == 0 && k < next; k++) { + if (t[k] == '|' && sh.bar == SIZE_MAX) { + sh.bar = k; + sh.slash = SIZE_MAX; /* the name and section follow the text */ + } else if (t[k] == '/' && sh.slash == SIZE_MAX) { + sh.slash = k; + } else if (t[k] == ' ' && sh.bar == SIZE_MAX) { + sh.space = true; + } + } + i = next; + } + sh.well_formed = sh.well_formed && st.depth == 0; + return sh; +} + +/* perlpodspec: a link whose target matches \A\w+:[^:\s]\S*\z is a URL. */ +static bool pl_is_url(const char *t, size_t n) { + size_t i = 0; + while (i < n && pl_word(t[i])) { + i++; + } + if (i == 0 || i + SKIP_ONE >= n || t[i] != ':' || t[i + SKIP_ONE] == ':') { + return false; + } + for (size_t k = i + SKIP_ONE; k < n; k++) { + if (pl_space(t[k])) { + return false; + } + } + return true; +} + +/* The start of t[b, e) after its leading spaces. */ +static size_t pl_lead_spaces_end(const char *t, size_t b, size_t e) { + while (b < e && t[b] == ' ') { + b++; + } + return b; +} + +/* The end of t[b, e) before its trailing spaces. */ +static size_t pl_trail_spaces_start(const char *t, size_t b, size_t e) { + while (e > b && t[e - SKIP_ONE] == ' ') { + e--; + } + return e; +} + +/* Trim t[*b, *e) and drop one pair of surrounding double quotes. */ +static void pl_trim(const char *t, size_t *b, size_t *e, bool *quoted) { + size_t lo = pl_lead_spaces_end(t, *b, *e); + size_t hi = pl_trail_spaces_start(t, lo, *e); + *quoted = hi - lo >= PAIR_LEN && t[lo] == '"' && t[hi - SKIP_ONE] == '"'; + *b = *quoted ? lo + SKIP_ONE : lo; + *e = *quoted ? hi - SKIP_ONE : hi; +} + +/* `name(3)`: a man page. */ +static bool pl_is_man_page(const char *p) { + size_t n = strlen(p); + if (n < 4 || p[n - SKIP_ONE] != ')') { + return false; + } + size_t open = n - SKIP_ONE; + while (open > 0 && p[open] != '(') { + open--; + } + if (open == 0 || open + SKIP_ONE == n - SKIP_ONE) { + return false; + } + for (size_t k = open + SKIP_ONE; k < n - SKIP_ONE; k++) { + if (!isalnum((unsigned char)p[k]) && p[k] != '-') { + return false; + } + } + return memchr(p, '/', open) == NULL; +} + +/* `Ident(::Ident)*`: a module name. */ +static bool pl_is_module_name(const char *p) { + if (!(isalpha((unsigned char)p[0]) || p[0] == '_')) { + return false; + } + for (size_t i = 0; p[i]; i++) { + if (pl_word(p[i])) { + continue; + } + if (p[i] != ':' || p[i + SKIP_ONE] != ':' || !pl_word(p[i + PAIR_LEN])) { + return false; + } + i++; + } + return true; +} + +bool cbm_doclink_perl_parse_link(const char *raw, CBMPerlLink *out) { + if (!out) { + return false; + } + memset(out, 0, sizeof(*out)); + out->kind = CBM_PERL_LINK_UNPARSEABLE; + size_t n = raw ? strlen(raw) : 0; + if (n == 0 || n > CBM_DOCLINK_PERL_RAW_MAX || (raw[0] == 'L' && raw[1] == '<')) { + return false; /* empty, or kept as written because it is malformed */ + } + pl_shape_t sh = pl_link_shape(raw, n); + if (!sh.well_formed) { + return false; + } + size_t nb = sh.bar == SIZE_MAX ? 0 : sh.bar + SKIP_ONE; + size_t ne = sh.slash == SIZE_MAX ? n : sh.slash; + size_t sb = sh.slash == SIZE_MAX ? n : sh.slash + SKIP_ONE; + size_t se = n; + bool name_quoted = false; + bool sec_quoted = false; + pl_trim(raw, &nb, &ne, &name_quoted); + pl_trim(raw, &sb, &se, &sec_quoted); + bool section_only = sh.slash == SIZE_MAX && (name_quoted || sh.space); + if (section_only) { /* L<"sec"> and the old L name a section */ + sb = nb; + se = ne; + ne = nb; + } else if (name_quoted) { + nb--; /* the quotes of a page name are part of it */ + ne++; + } + cbm_doclink_perl_plain_text(raw + nb, ne - nb, out->page, sizeof(out->page)); + cbm_doclink_perl_plain_text(raw + sb, se - sb, out->section, sizeof(out->section)); + if (!out->page[0]) { + out->kind = out->section[0] ? CBM_PERL_LINK_SECTION : CBM_PERL_LINK_UNPARSEABLE; + } else if (pl_is_man_page(out->page)) { + out->kind = CBM_PERL_LINK_MAN; + } else if (pl_is_module_name(out->page)) { + out->kind = CBM_PERL_LINK_PAGE; + } + return out->kind != CBM_PERL_LINK_UNPARSEABLE; +} + +/* ── Tokens ──────────────────────────────────────────────────────── */ + +typedef struct { + CBMExtractCtx *ctx; + const CBMDefinition *def; + const char *text; /* what is being read: the file, or a doc of unknown position */ + size_t line_pos; /* lines are counted from here ... */ + uint32_t line; /* ... which is on this line */ + bool fixed_line; /* every token gets `line` */ +} pl_emit_t; + +typedef struct { + size_t at; /* the code's letter */ + size_t cb; /* its content */ + size_t ce; + bool closed; /* false: the paragraph ended first */ +} pl_code_t; + +static uint32_t pl_line_at(pl_emit_t *em, size_t at) { + if (em->fixed_line) { + return em->line; + } + for (size_t k = em->line_pos; k < at; k++) { + em->line += em->text[k] == '\n'; + } + em->line_pos = at > em->line_pos ? at : em->line_pos; + return em->line; +} + +static void pl_push(pl_emit_t *em, const char *raw, size_t n, int syntax, uint32_t line) { + char *copy = cbm_arena_strndup(em->ctx->arena, raw, n); + if (!copy) { + return; + } + CBMDocLink link = { + .source_qn = em->def->qualified_name, + .raw = copy, + .line = line, + .def_line = em->def->start_line, + .syntax = (uint16_t)syntax, + }; + cbm_doclinks_push(&em->ctx->result->doc_links, em->ctx->arena, link); +} + +/* One L<> code of em->text -> one token (none for an empty code). */ +static void pl_emit_link(pl_emit_t *em, const pl_code_t *code) { + uint32_t line = pl_line_at(em, code->at); + char buf[PL_MALFORMED + CBM_DOCLINK_PERL_RAW_MAX + SKIP_ONE] = "L<"; + char *content = buf + PL_MALFORMED; + bool cut = false; + size_t n = pl_squeeze(em->text, code->cb, code->ce, content, CBM_DOCLINK_PERL_RAW_MAX, &cut); + if (!code->closed || cut) { + size_t keep = n > CBM_DOCLINK_PERL_RAW_MAX - PL_MALFORMED + ? CBM_DOCLINK_PERL_RAW_MAX - PL_MALFORMED + : n; + while (keep > 0 && ((unsigned char)content[keep] & PL_UTF8_CONT_MASK) == PL_UTF8_CONT) { + keep--; /* not in the middle of a character */ + } + pl_push(em, buf, PL_MALFORMED + keep, CBM_DOCLINK_PERL_POD_L, line); + return; + } + if (n == 0) { + return; /* Pod::Simple: "An empty L<>" is text */ + } + pl_shape_t sh = pl_link_shape(content, n); + size_t tb = sh.bar == SIZE_MAX ? 0 : sh.bar + SKIP_ONE; + while (tb < n && content[tb] == ' ') { + tb++; + } + if (sh.well_formed && pl_is_url(content + tb, n - tb)) { + pl_push(em, content + tb, n - tb, CBM_DOCLINK_HREF, line); + return; + } + pl_push(em, content, n, CBM_DOCLINK_PERL_POD_L, line); +} + +/* The L<> codes of one paragraph's text em->text[b, e). */ +static void pl_para_links(pl_emit_t *em, size_t b, size_t e) { + const char *t = em->text; + pl_codes_t st = {.depth = 0}; + pl_code_t link = {0}; + int link_depth = 0; /* stack depth of the open L; 0: none */ + bool harvest = false; /* the open L is a reference */ + size_t i = b; + while (i < e && !st.overflow) { + size_t next = i; + int ev = pl_code_step(&st, t, i, e, &next); + if (ev == PL_EV_OPEN && link_depth == 0 && t[i] == 'L') { + link_depth = st.depth; + harvest = !(i > b && pl_word(t[i - SKIP_ONE])); + link = (pl_code_t){.at = i, .cb = next, .ce = next, .closed = false}; + } else if (ev == PL_EV_CLOSE && link_depth != 0 && st.depth < link_depth) { + link.ce = i; + link.closed = true; + if (harvest) { + pl_emit_link(em, &link); + } + link_depth = 0; + } + i = next; + } + if (link_depth != 0 && harvest) { + link.ce = e; /* never closed, or nested too deep */ + pl_emit_link(em, &link); + } +} + +/* First byte after the white space that follows s[i]. */ +static size_t pl_skip_blank(const char *s, size_t i, size_t e) { + while (i < e && pl_space(s[i])) { + i++; + } + return i; +} + +/* The L<> codes of the POD text em->text[b, e): from its first command on, or + * from its start when `in_pod` (a run of the module doc, which is POD only). */ +static void pl_harvest_pod(pl_emit_t *em, size_t b, size_t e, bool in_pod) { + const char *t = em->text; + size_t pos = b; + size_t pb = 0; + size_t pe = 0; + int begin_depth = 0; /* open =begin regions */ + int data_from = 0; /* depth of the outermost data region; 0: none */ + while (pl_next_para(t, &pos, e, &pb, &pe)) { + bool cmd = pl_is_command(t, pb, pe); + if (!cmd) { + if (in_pod && data_from == 0 && t[pb] != ' ' && t[pb] != '\t') { + pl_para_links(em, pb, pe); + } + continue; + } + size_t wb = pl_command_word_end(t, pb, pe); + const char *w = t + pb + SKIP_ONE; + size_t wl = wb - pb - SKIP_ONE; + in_pod = !pl_word_is(w, wl, "cut"); + if (pl_word_is(w, wl, "begin")) { + size_t f = pl_skip_blank(t, wb, pe); + begin_depth++; + if (data_from == 0 && !(f < pe && t[f] == ':')) { + data_from = begin_depth; + } + } else if (pl_word_is(w, wl, "end") && begin_depth > 0) { + data_from = begin_depth == data_from ? 0 : data_from; + begin_depth--; + } else if (data_from == 0 && (pl_head_level(w, wl) > 0 || pl_word_is(w, wl, "item"))) { + pl_para_links(em, wb, pe); + } + } +} + +/* ── The file's POD sections ─────────────────────────────────────── */ + +enum { PL_HEAD = 1, PL_ITEM, PL_OVER, PL_BACK, PL_CUT, PL_OTHER }; + +typedef struct { + uint32_t sb; /* command paragraph */ + uint32_t eb; + uint32_t wb; /* end of the command word */ + uint8_t type; + uint8_t level; /* =headN: N; =item and =back: list depth */ +} pl_cmd_t; + +typedef struct { + uint32_t sb; /* the heading's command paragraph */ + uint32_t eb; /* section end, trailing white space trimmed */ + uint32_t hb; /* heading text */ + uint32_t he; + uint32_t line; /* 1-based line of sb */ + const char *name; /* the identifier the heading names; NULL: none */ + uint32_t name_len; + bool item; +} pl_sec_t; + +typedef struct { + CBMArena *a; + const char *src; + uint32_t len; + pl_sec_t *secs; + int nsecs; + int cap_secs; + pl_cmd_t *cmds; /* of the block being read */ + int ncmds; + int cap_cmds; + uint32_t line_pos; /* lines are counted from here ... */ + uint32_t line; /* ... which is on this line */ + bool failed; +} pl_scan_t; + +/* Room for one more element of `size` bytes in an arena array. */ +static void *pl_grow(pl_scan_t *s, void *items, int count, int *cap, size_t size) { + if (count < *cap) { + return items; + } + int ncap = *cap ? *cap * PAIR_LEN : PL_INIT_CAP; + void *grown = cbm_arena_alloc(s->a, (size_t)ncap * size); + if (!grown) { + s->failed = true; + return NULL; + } + if (count > 0) { + memcpy(grown, items, (size_t)count * size); + } + *cap = ncap; + return grown; +} + +/* Classify one command paragraph; `depth` is the =over nesting + * (extract_defs.c's pod_classify). */ +static pl_cmd_t pl_classify(const char *src, uint32_t sb, uint32_t eb, int *depth) { + uint32_t wb = (uint32_t)pl_command_word_end(src, sb, eb); + const char *w = src + sb + SKIP_ONE; + size_t wl = wb - sb - SKIP_ONE; + pl_cmd_t c = {sb, eb, wb, PL_OTHER, 0}; + if (pl_head_level(w, wl) > 0) { + c.type = PL_HEAD; + c.level = (uint8_t)pl_head_level(w, wl); + } else if (pl_word_is(w, wl, "item")) { + c.type = PL_ITEM; + c.level = (uint8_t)*depth; + } else if (pl_word_is(w, wl, "over")) { + c.type = PL_OVER; + (*depth)++; + } else if (pl_word_is(w, wl, "back")) { + c.type = PL_BACK; + c.level = (uint8_t)*depth; + if (*depth > 0) { + (*depth)--; + } + } else if (pl_word_is(w, wl, "cut")) { + c.type = PL_CUT; + } + return c; +} + +/* The command paragraphs of the POD block src[pb, pe) into s->cmds. */ +static void pl_block_commands(pl_scan_t *s, uint32_t pb, uint32_t pe) { + s->ncmds = 0; + int depth = 0; + size_t pos = pb; + size_t sb = 0; + size_t eb = 0; + while (!s->failed && pl_next_para(s->src, &pos, pe, &sb, &eb)) { + if (!pl_is_command(s->src, sb, eb)) { + continue; + } + pl_cmd_t *cmds = (pl_cmd_t *)pl_grow(s, s->cmds, s->ncmds, &s->cap_cmds, sizeof(pl_cmd_t)); + if (!cmds) { + return; + } + s->cmds = cmds; + s->cmds[s->ncmds++] = pl_classify(s->src, (uint32_t)sb, (uint32_t)eb, &depth); + } +} + +/* End byte of the section that cmds[ci] (a =head or =item) opens + * (extract_defs.c's pod_section_end). */ +static uint32_t pl_section_end(const pl_cmd_t *cmds, int n, int ci, uint32_t pe) { + const pl_cmd_t *c = &cmds[ci]; + for (int j = ci + SKIP_ONE; j < n; j++) { + const pl_cmd_t *d = &cmds[j]; + if (d->type == PL_CUT) { + return d->sb; + } + if (d->type == PL_HEAD && (c->type == PL_ITEM || d->level <= c->level)) { + return d->sb; + } + if (c->type == PL_ITEM && (d->type == PL_ITEM || d->type == PL_BACK) && + d->level == c->level) { + return d->sb; + } + } + return pe; +} + +/* A heading's text with formatting codes removed and newlines and tabs as + * spaces (extract_defs.c's pod_plain_heading, byte for byte: the name it + * yields decides which sub a section is stored on). */ +static char *pl_plain_heading(CBMArena *a, const char *src, uint32_t b, uint32_t e) { + char *buf = (char *)cbm_arena_alloc(a, (size_t)(e - b) + SKIP_ONE); + if (!buf) { + return NULL; + } + size_t w = 0; + for (uint32_t i = b; i < e; i++) { + char c = src[i]; + if (pl_upper(c) && i + SKIP_ONE < e && src[i + SKIP_ONE] == '<') { + uint32_t j = i + SKIP_ONE; + while (j < e && src[j] == '<') { + j++; + } + if (j - i > PAIR_LEN && j < e && (src[j] == ' ' || src[j] == '\t')) { + j++; /* X<< text >> */ + } + i = j - SKIP_ONE; + continue; + } + if (c == '>' && !(w > 0 && buf[w - SKIP_ONE] == '-')) { + continue; + } + buf[w++] = (c == '\n' || c == '\r' || c == '\t') ? ' ' : c; + } + buf[w] = '\0'; + return buf; +} + +static const char *pl_skip_spaces(const char *p) { + while (*p == ' ') { + p++; + } + return p; +} + +/* The sub a heading names: after a list bullet ("*", "1."), a `$obj->` + * receiver and a sigil, the identifier that follows; its length, 0 for none + * (extract_defs.c's pod_heading_name). */ +static size_t pl_heading_name(CBMArena *a, const char *src, uint32_t b, uint32_t e, + const char **name) { + const char *buf = pl_plain_heading(a, src, b, e); + if (!buf) { + return 0; + } + const char *p = pl_skip_spaces(buf); + if (*p == '*') { + p++; + } else if (isdigit((unsigned char)*p)) { + while (isdigit((unsigned char)*p)) { + p++; + } + if (*p == '.') { + p++; + } + } + p = pl_skip_spaces(p); + if (*p == '$') { + const char *q = p + SKIP_ONE; + while (pl_word(*q)) { + q++; + } + q = pl_skip_spaces(q); + if (q > p + SKIP_ONE && q[0] == '-' && q[SKIP_ONE] == '>') { + p = pl_skip_spaces(q + PAIR_LEN); + } + } + if (*p == '$' || *p == '@' || *p == '%' || *p == '&') { + p++; + } + if (!(isalpha((unsigned char)*p) || *p == '_')) { + return 0; + } + const char *s = p; + while (pl_word(*p)) { + p++; + } + *name = s; + return (size_t)(p - s); +} + +static uint32_t pl_trim_end(const char *src, uint32_t sb, uint32_t eb) { + while (eb > sb && pl_space(src[eb - SKIP_ONE])) { + eb--; + } + return eb; +} + +/* The 1-based line of src[at]; `at` never decreases between calls. */ +static uint32_t pl_scan_line(pl_scan_t *s, uint32_t at) { + for (uint32_t k = s->line_pos; k < at; k++) { + s->line += s->src[k] == '\n'; + } + s->line_pos = at > s->line_pos ? at : s->line_pos; + return s->line; +} + +/* The =head / =item sections of the POD block src[pb, pe). */ +static void pl_block_sections(pl_scan_t *s, uint32_t pb, uint32_t pe) { + pl_block_commands(s, pb, pe); + for (int ci = 0; ci < s->ncmds && !s->failed; ci++) { + const pl_cmd_t *c = &s->cmds[ci]; + if (c->type != PL_HEAD && c->type != PL_ITEM) { + continue; + } + pl_sec_t *secs = (pl_sec_t *)pl_grow(s, s->secs, s->nsecs, &s->cap_secs, sizeof(pl_sec_t)); + if (!secs) { + return; + } + s->secs = secs; + pl_sec_t sec = {.sb = c->sb, .hb = c->wb, .he = c->eb, .item = c->type == PL_ITEM}; + sec.eb = pl_trim_end(s->src, c->sb, pl_section_end(s->cmds, s->ncmds, ci, pe)); + sec.line = pl_scan_line(s, c->sb); + sec.name_len = (uint32_t)pl_heading_name(s->a, s->src, c->wb, c->eb, &sec.name); + s->secs[s->nsecs++] = sec; + } +} + +/* True when the line src[i, le) is the `=cut` that ends a POD block. */ +static bool pl_is_cut_line(const char *src, uint32_t i, uint32_t le) { + static const char cut[] = "=cut"; + uint32_t n = (uint32_t)(sizeof(cut) - SKIP_ONE); + return le - i >= n && memcmp(src + i, cut, n) == 0 && (le - i == n || !pl_word(src[i + n])); +} + +/* Every POD section of the file. A block runs from a line that starts with + * `=` and a letter through the next line that starts with `=cut`, or to the + * end of the file: what perl skips and perldoc reads. */ +static void pl_scan_file(pl_scan_t *s) { + bool in_pod = false; + uint32_t pb = 0; + uint32_t i = 0; + while (i < s->len && !s->failed) { + const char *nl = (const char *)memchr(s->src + i, '\n', s->len - i); + uint32_t le = nl ? (uint32_t)(nl - s->src) : s->len; + if (!in_pod && pl_is_command(s->src, i, le)) { + in_pod = true; + pb = i; + } + if (in_pod && pl_is_cut_line(s->src, i, le)) { + pl_block_sections(s, pb, le); + in_pod = false; + } + i = le < s->len ? le + SKIP_ONE : s->len; + } + if (in_pod && !s->failed) { + pl_block_sections(s, pb, s->len); + } +} + +static void pl_scan_init(pl_scan_t *s, CBMExtractCtx *ctx) { + memset(s, 0, sizeof(*s)); + s->a = pl_scratch(ctx); + s->src = ctx->source; + s->len = (ctx->source && ctx->source_len > 0) ? (uint32_t)ctx->source_len : 0; + s->line = SKIP_ONE; +} + +/* ── Docs -> tokens ──────────────────────────────────────────────── */ + +typedef struct { + const char *name; + uint32_t len; + int sec; +} pl_named_t; + +static int pl_name_cmp(const char *a, size_t al, const char *b, size_t bl) { + int c = memcmp(a, b, al < bl ? al : bl); + if (c != 0) { + return c; + } + return (al > bl) - (al < bl); +} + +static int pl_named_cmp(const void *a, const void *b) { + const pl_named_t *x = (const pl_named_t *)a; + const pl_named_t *y = (const pl_named_t *)b; + int c = pl_name_cmp(x->name, x->len, y->name, y->len); + return c != 0 ? c : (x->sec > y->sec) - (x->sec < y->sec); +} + +/* The sections that name something, sorted by name and then by position. */ +static pl_named_t *pl_named_sections(const pl_scan_t *s, int *count) { + *count = 0; + pl_named_t *named = + (pl_named_t *)cbm_arena_alloc(s->a, (size_t)(s->nsecs + SKIP_ONE) * sizeof(*named)); + if (!named) { + return NULL; + } + int n = 0; + for (int i = 0; i < s->nsecs; i++) { + if (s->secs[i].name_len > 0) { + named[n++] = (pl_named_t){s->secs[i].name, s->secs[i].name_len, i}; + } + } + qsort(named, (size_t)n, sizeof(*named), pl_named_cmp); + *count = n; + return named; +} + +/* Index of the first section named `name`, or `count`. */ +static int pl_named_first(const pl_named_t *named, int count, const char *name) { + size_t nl = strlen(name); + int lo = 0; + int hi = count; + while (lo < hi) { + int mid = lo + ((hi - lo) / PAIR_LEN); + if (pl_name_cmp(named[mid].name, named[mid].len, name, nl) < 0) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + return lo; +} + +/* True when doc[o, dl) starts with the section `sec`, alone or followed by the + * blank line that joins it to the next one. */ +static bool pl_piece_is(const pl_scan_t *s, const pl_sec_t *sec, const char *doc, size_t o, + size_t dl) { + size_t n = sec->eb - sec->sb; + if (n == 0 || n > dl - o || memcmp(doc + o, s->src + sec->sb, n) != 0) { + return false; + } + return o + n == dl || + (dl - o - n >= PAIR_LEN && doc[o + n] == '\n' && doc[o + n + SKIP_ONE] == '\n'); +} + +/* One definition's doc -> tokens: every piece of it that is a section of the + * file is read where it stands, so its references carry their own lines. */ +static void pl_harvest_def(CBMExtractCtx *ctx, const pl_scan_t *s, const pl_named_t *named, + int count, const CBMDefinition *def, uint32_t fallback_line) { + const char *doc = def->docstring; + size_t dl = strlen(doc); + if (!pl_is_command(doc, 0, dl)) { + return; /* a `#` comment: no formatting codes */ + } + size_t nl = strlen(def->name); + size_t o = 0; + int cand = named ? pl_named_first(named, count, def->name) : count; + for (; o < dl && cand < count && + pl_name_cmp(named[cand].name, named[cand].len, def->name, nl) == 0; + cand++) { + const pl_sec_t *sec = &s->secs[named[cand].sec]; + if (!pl_piece_is(s, sec, doc, o, dl)) { + continue; /* a section the doc does not hold */ + } + pl_emit_t em = { + .ctx = ctx, .def = def, .text = s->src, .line_pos = sec->sb, .line = sec->line}; + pl_harvest_pod(&em, sec->sb, sec->eb, false); + o += sec->eb - sec->sb; + o += o < dl ? PAIR_LEN : 0; + } + if (o < dl) { /* text whose place in the file is not known */ + pl_emit_t em = { + .ctx = ctx, .def = def, .text = doc, .line = fallback_line, .fixed_line = true}; + pl_harvest_pod(&em, o, dl, false); + } +} + +/* The core's filter: a definition whose doc is harvested. */ +static bool pl_documented(const CBMDefinition *d) { + return d->docstring && d->docstring[0] && d->qualified_name && d->name; +} + +typedef struct { + const char *qn; + int idx; +} pl_docdef_t; + +static int pl_docdef_cmp(const void *a, const void *b) { + const pl_docdef_t *x = (const pl_docdef_t *)a; + const pl_docdef_t *y = (const pl_docdef_t *)b; + int c = strcmp(x->qn, y->qn); + return c != 0 ? c : (x->idx > y->idx) - (x->idx < y->idx); +} + +/* twin[i]: definition i repeats the qualified name AND the doc of an earlier + * one (the same sub name in two packages of a file is one node with one doc): + * its references are the earlier one's. NULL when there is nothing to mark. */ +static bool *pl_twins(CBMExtractCtx *ctx) { + const CBMDefArray *defs = &ctx->result->defs; + CBMArena *a = pl_scratch(ctx); + pl_docdef_t *docs = (pl_docdef_t *)cbm_arena_alloc(a, (size_t)defs->count * sizeof(*docs)); + bool *twin = (bool *)cbm_arena_calloc(a, (size_t)defs->count * sizeof(bool)); + if (!docs || !twin) { + return NULL; + } + int n = 0; + for (int i = 0; i < defs->count; i++) { + if (pl_documented(&defs->items[i])) { + docs[n++] = (pl_docdef_t){defs->items[i].qualified_name, i}; + } + } + qsort(docs, (size_t)n, sizeof(*docs), pl_docdef_cmp); + int first = 0; + for (int i = SKIP_ONE; i < n; i++) { + if (strcmp(docs[i].qn, docs[first].qn) != 0) { + first = i; + } else { + twin[docs[i].idx] = strcmp(defs->items[docs[i].idx].docstring, + defs->items[docs[first].idx].docstring) == 0; + } + } + return twin; +} + +/* Every documented definition of the file -> tokens, in definition order. */ +static void pl_harvest_file(CBMExtractCtx *ctx) { + pl_scan_t s; + pl_scan_init(&s, ctx); + pl_scan_file(&s); + int count = 0; + pl_named_t *named = s.failed ? NULL : pl_named_sections(&s, &count); + const bool *twin = pl_twins(ctx); + const CBMDefArray *defs = &ctx->result->defs; + for (int i = 0; i < defs->count; i++) { + const CBMDefinition *d = &defs->items[i]; + if (pl_documented(d) && !(twin && twin[i])) { + pl_harvest_def(ctx, &s, named, named ? count : 0, d, d->start_line); + } + } +} + +/* First position >= from of `needle` (n bytes) in src[0, len); SIZE_MAX: none. */ +static size_t pl_find(const char *src, size_t from, size_t len, const char *needle, size_t n) { + if (n == 0 || n > len) { + return SIZE_MAX; + } + for (size_t i = from; i + n <= len; i++) { + const char *hit = (const char *)memchr(src + i, needle[0], len - n + SKIP_ONE - i); + if (!hit) { + return SIZE_MAX; + } + i = (size_t)(hit - src); + if (memcmp(src + i, needle, n) == 0) { + return i; + } + } + return SIZE_MAX; +} + +/* The module doc (the core's File stand-in): runs of the file's POD joined by + * a blank line (extract_defs.c, doc_perl_module), each found again in the + * source after the one before, so that a reference gets the line it is written + * on. Text that is not found keeps the doc's first line. */ +static void pl_harvest_module(CBMExtractCtx *ctx, const CBMDefinition *def, const char *doc, + uint32_t doc_line) { + const char *src = ctx->source; + size_t sl = ctx->source_len > 0 ? (size_t)ctx->source_len : 0; + size_t dl = strlen(doc); + size_t o = 0; + size_t at = 0; + size_t line_pos = 0; + uint32_t line = SKIP_ONE; + while (src && o < dl) { + size_t le = o; + while (le < dl && doc[le] != '\n') { + le++; + } + size_t p = pl_find(src, at, sl, doc + o, le - o); + if (p == SIZE_MAX) { + break; + } + size_t k = 0; + while (o + k < dl && p + k < sl && doc[o + k] == src[p + k]) { + k++; + } + for (; line_pos < p; line_pos++) { + line += src[line_pos] == '\n' ? SKIP_ONE : 0; + } + pl_emit_t em = {.ctx = ctx, .def = def, .text = src, .line_pos = p, .line = line}; + pl_harvest_pod(&em, p, p + k, true); + o += k; + at = p + k; + while (o < dl && doc[o] == '\n') { + o++; /* the blank line that joins two runs */ + } + } + if (o < dl) { + pl_emit_t em = {.ctx = ctx, .def = def, .text = doc, .line = doc_line, .fixed_line = true}; + pl_harvest_pod(&em, o, dl, true); + } +} + +/* Index of `def` in the file's definitions; -1 when it is not one of them. */ +static int pl_def_index(const CBMDefArray *defs, const CBMDefinition *def) { + uintptr_t base = (uintptr_t)defs->items; + uintptr_t p = (uintptr_t)def; + if (!defs->items || defs->count <= 0 || p < base) { + return CBM_NOT_FOUND; + } + uintptr_t off = p - base; + if (off % sizeof(CBMDefinition) != 0 || off / sizeof(CBMDefinition) >= (uintptr_t)defs->count) { + return CBM_NOT_FOUND; + } + return (int)(off / sizeof(CBMDefinition)); +} + +void cbm_doclink_perl_parse_doc(CBMExtractCtx *ctx, const CBMDefinition *def, const char *doc, + uint32_t doc_line) { + if (!ctx || !ctx->result || !def || !doc || !def->qualified_name) { + return; + } + if (def->label && strcmp(def->label, "File") == 0) { + pl_harvest_module(ctx, def, doc, doc_line); /* the file's module doc */ + return; + } + const CBMDefArray *defs = &ctx->result->defs; + int idx = pl_def_index(defs, def); + if (idx < 0 || doc != def->docstring) { + /* a doc that is not a listed definition's own: nothing says where in + * the file it stands, so its references get the line the core passed */ + if (pl_is_command(doc, 0, strlen(doc))) { + pl_emit_t em = { + .ctx = ctx, .def = def, .text = doc, .line = doc_line, .fixed_line = true}; + pl_harvest_pod(&em, 0, strlen(doc), false); + } + return; + } + for (int i = idx - SKIP_ONE; i >= 0; i--) { + if (pl_documented(&defs->items[i])) { + return; /* harvested with the file's first documented definition */ + } + } + pl_harvest_file(ctx); +} + +/* ── Scope blob ──────────────────────────────────────────────────── */ + +typedef struct { + CBMArena *a; + char *buf; + size_t len; + size_t cap; + bool failed; +} pl_sb_t; + +static void pl_sb_put(pl_sb_t *sb, const char *s, size_t n) { + if (sb->failed) { + return; + } + if (sb->len + n + SKIP_ONE > sb->cap) { + size_t ncap = sb->cap ? sb->cap : PL_SB_INIT; + while (ncap < sb->len + n + SKIP_ONE) { + ncap *= PAIR_LEN; + } + char *grown = (char *)cbm_arena_alloc(sb->a, ncap); + if (!grown) { + sb->failed = true; + return; + } + if (sb->len > 0) { + memcpy(grown, sb->buf, sb->len); + } + sb->buf = grown; + sb->cap = ncap; + } + memcpy(sb->buf + sb->len, s, n); + sb->len += n; + sb->buf[sb->len] = '\0'; +} + +static void pl_sb_puts(pl_sb_t *sb, const char *s) { + pl_sb_put(sb, s, strlen(s)); +} + +enum { PL_PKG_MAIN = -1, PL_PKG_UNKNOWN = -2 }; + +typedef struct { + const char *name; /* in the source */ + uint32_t len; + int order; /* declaration order in the file */ + int index; /* of its P record; set by pl_number_packages */ +} pl_pkg_t; + +typedef struct { + const char *name; /* in the source */ + uint32_t len; + uint32_t line; + int pkg; /* declaration (pl_pkg_t.order) it stands in, or PL_PKG_* */ +} pl_subdecl_t; + +typedef struct { + pl_scan_t *s; /* arena and growth */ + pl_pkg_t *pkgs; + int npkgs; + int cap_pkgs; + pl_subdecl_t *subs; + int nsubs; + int cap_subs; + int *saved; /* the package in force outside each open block */ + int nsaved; + int cap_saved; + int current; /* declaration in force, or PL_PKG_* */ + int pending_block; /* declaration whose block comes next, or PL_PKG_UNKNOWN */ + bool pending; +} pl_walk_t; + +static bool pl_kind(TSNode n, const char *kind) { + return strcmp(ts_node_type(n), kind) == 0; +} + +/* A package or sub name as a blob field: word characters, `:` and `'`. */ +static bool pl_name_ok(const char *p, size_t n) { + if (n == 0 || n > PL_NAME_MAX) { + return false; + } + for (size_t i = 0; i < n; i++) { + if (!pl_word(p[i]) && p[i] != ':' && p[i] != '\'') { + return false; + } + } + return true; +} + +/* The source text of a declaration's `name` field. */ +static bool pl_decl_name(const pl_walk_t *w, TSNode decl, const char **name, uint32_t *len) { + static const char field[] = "name"; + TSNode n = ts_node_child_by_field_name(decl, field, (uint32_t)(sizeof(field) - SKIP_ONE)); + if (ts_node_is_null(n)) { + return false; + } + uint32_t b = ts_node_start_byte(n); + uint32_t e = ts_node_end_byte(n); + if (e <= b || e > w->s->len) { + return false; + } + *name = w->s->src + b; + *len = e - b; + return pl_name_ok(*name, *len); +} + +static bool pl_has_block(TSNode decl) { + uint32_t n = ts_node_child_count(decl); + for (uint32_t i = 0; i < n; i++) { + if (pl_kind(ts_node_child(decl, i), "block")) { + return true; + } + } + return false; +} + +/* `package NAME;` sets the package to the end of the enclosing block; + * `package NAME { .. }` sets it inside its own block only. */ +static void pl_walk_package(pl_walk_t *w, TSNode decl) { + const char *name = NULL; + uint32_t len = 0; + int id = PL_PKG_UNKNOWN; + if (pl_decl_name(w, decl, &name, &len)) { + pl_pkg_t *pkgs = + (pl_pkg_t *)pl_grow(w->s, w->pkgs, w->npkgs, &w->cap_pkgs, sizeof(pl_pkg_t)); + if (!pkgs) { + return; + } + w->pkgs = pkgs; + id = w->npkgs; + w->pkgs[w->npkgs++] = (pl_pkg_t){.name = name, .len = len, .order = id}; + } + if (pl_has_block(decl)) { + w->pending = true; + w->pending_block = id; + } else { + w->current = id; + } +} + +static void pl_walk_sub(pl_walk_t *w, TSNode decl) { + const char *name = NULL; + uint32_t len = 0; + if (!pl_decl_name(w, decl, &name, &len)) { + return; + } + pl_subdecl_t *subs = + (pl_subdecl_t *)pl_grow(w->s, w->subs, w->nsubs, &w->cap_subs, sizeof(pl_subdecl_t)); + if (!subs) { + return; + } + w->subs = subs; + w->subs[w->nsubs++] = (pl_subdecl_t){.name = name, + .len = len, + .line = ts_node_start_point(decl).row + TS_LINE_OFFSET, + .pkg = w->current}; +} + +static bool pl_is_scope(TSNode n) { + return pl_kind(n, "block") || pl_kind(n, "block_statement"); +} + +/* Enter a node; false when its children are not walked (a sub's body and an + * expression hold no declaration this blob lists). */ +static bool pl_walk_enter(pl_walk_t *w, TSNode n) { + if (pl_kind(n, "subroutine_declaration_statement")) { + pl_walk_sub(w, n); + return false; + } + if (pl_kind(n, "expression_statement") || pl_kind(n, "pod") || + pl_kind(n, "method_declaration_statement")) { + return false; + } + if (pl_kind(n, "package_statement") || pl_kind(n, "class_statement") || + pl_kind(n, "role_statement")) { + pl_walk_package(w, n); + return true; + } + if (pl_is_scope(n)) { + int *saved = (int *)pl_grow(w->s, w->saved, w->nsaved, &w->cap_saved, sizeof(int)); + if (!saved) { + return false; + } + w->saved = saved; + w->saved[w->nsaved++] = w->current; + if (w->pending) { + w->current = w->pending_block; + w->pending = false; + } + } + return true; +} + +static void pl_walk_leave(pl_walk_t *w, TSNode n) { + if (pl_is_scope(n) && w->nsaved > 0) { + w->current = w->saved[--w->nsaved]; + } +} + +/* Packages and subs of the tree, with the package each sub stands in. An + * iterative walk: the nesting depth is the input's. */ +static void pl_walk_tree(pl_walk_t *w, TSNode root) { + TSTreeCursor cur = ts_tree_cursor_new(root); + bool more = true; + while (more && !w->s->failed) { + TSNode n = ts_tree_cursor_current_node(&cur); + bool entered = pl_walk_enter(w, n); + if (entered && ts_tree_cursor_goto_first_child(&cur)) { + continue; + } + if (entered) { + pl_walk_leave(w, n); + } + while (more && !ts_tree_cursor_goto_next_sibling(&cur)) { + more = ts_tree_cursor_goto_parent(&cur); + if (more) { + pl_walk_leave(w, ts_tree_cursor_current_node(&cur)); + } + } + } + ts_tree_cursor_delete(&cur); +} + +static int pl_pkg_name_cmp(const void *a, const void *b) { + const pl_pkg_t *x = (const pl_pkg_t *)a; + const pl_pkg_t *y = (const pl_pkg_t *)b; + int c = pl_name_cmp(x->name, x->len, y->name, y->len); + return c != 0 ? c : (x->order > y->order) - (x->order < y->order); +} + +static int pl_pkg_order_cmp(const void *a, const void *b) { + const pl_pkg_t *x = (const pl_pkg_t *)a; + const pl_pkg_t *y = (const pl_pkg_t *)b; + return (x->order > y->order) - (x->order < y->order); +} + +/* Give every declaration the index of its name's P record (names in order of + * their first declaration) and write the P records. */ +static void pl_number_packages(pl_walk_t *w, pl_sb_t *sb) { + qsort(w->pkgs, (size_t)w->npkgs, sizeof(pl_pkg_t), pl_pkg_name_cmp); + for (int i = 0; i < w->npkgs; i++) { /* index = order of the name's first declaration */ + bool repeat = + i > 0 && pl_name_cmp(w->pkgs[i].name, w->pkgs[i].len, w->pkgs[i - SKIP_ONE].name, + w->pkgs[i - SKIP_ONE].len) == 0; + w->pkgs[i].index = repeat ? w->pkgs[i - SKIP_ONE].index : w->pkgs[i].order; + } + qsort(w->pkgs, (size_t)w->npkgs, sizeof(pl_pkg_t), pl_pkg_order_cmp); + int records = 0; + for (int i = 0; i < w->npkgs; i++) { + pl_pkg_t *p = &w->pkgs[i]; + if (p->index != p->order) { + p->index = w->pkgs[p->index].index; /* its first declaration's record */ + continue; + } + p->index = records++; + pl_sb_puts(sb, "P\t"); + pl_sb_put(sb, p->name, p->len); + pl_sb_puts(sb, "\n"); + } +} + +static int pl_subdecl_cmp(const void *a, const void *b) { + const pl_subdecl_t *x = (const pl_subdecl_t *)a; + const pl_subdecl_t *y = (const pl_subdecl_t *)b; + if (x->line != y->line) { + return x->line < y->line ? -1 : 1; + } + return pl_name_cmp(x->name, x->len, y->name, y->len); +} + +/* One S record per sub definition of the file, with the package the tree + * places it in. */ +static void pl_put_subs(CBMExtractCtx *ctx, pl_walk_t *w, pl_sb_t *sb) { + qsort(w->subs, (size_t)w->nsubs, sizeof(pl_subdecl_t), pl_subdecl_cmp); + const CBMDefArray *defs = &ctx->result->defs; + for (int i = 0; i < defs->count; i++) { + const CBMDefinition *d = &defs->items[i]; + if (!d->name || !d->label || strcmp(d->label, "Function") != 0 || + !pl_name_ok(d->name, strlen(d->name))) { + continue; + } + pl_subdecl_t key = { + .name = d->name, .len = (uint32_t)strlen(d->name), .line = d->start_line}; + const pl_subdecl_t *hit = + w->nsubs > 0 ? (const pl_subdecl_t *)bsearch(&key, w->subs, (size_t)w->nsubs, + sizeof(pl_subdecl_t), pl_subdecl_cmp) + : NULL; + int pkg = hit ? hit->pkg : PL_PKG_UNKNOWN; + char field[CBM_SZ_32]; + if (pkg >= 0) { + snprintf(field, sizeof(field), "S\t%d\t", w->pkgs[pkg].index); + } else { + snprintf(field, sizeof(field), "S\t%s\t", pkg == PL_PKG_MAIN ? "-" : "?"); + } + pl_sb_puts(sb, field); + pl_sb_puts(sb, d->name); + pl_sb_puts(sb, "\n"); + } +} + +static void pl_put_heading(pl_sb_t *sb, const pl_sec_t *sec, const char *key) { + if (!key[0]) { + return; + } + pl_sb_puts(sb, "H\t"); + if (sec->name_len > 0) { + pl_sb_put(sb, sec->name, sec->name_len); + } else { + pl_sb_puts(sb, "-"); + } + pl_sb_puts(sb, "\t"); + pl_sb_puts(sb, key); + pl_sb_puts(sb, "\n"); +} + +/* The text after an =item's bullet (`*`) or number (`1.`), or NULL when the + * item has neither. */ +static const char *pl_after_bullet(const char *key) { + const char *p = key; + if (*p == '*') { + p++; + } else { + while (isdigit((unsigned char)*p)) { + p++; + } + if (p == key) { + return NULL; + } + p += *p == '.'; + } + return (*p == ' ' || *p == '\0') ? pl_skip_spaces(p) : NULL; +} + +/* One H record per heading. Pod::Simple reads `=item * text` and `=item 1. + * text` as a bullet and its text when the list is a bullet or numbered list, + * and as the whole text otherwise; which kind a list is depends on its first + * item, so a numbered text is recorded both ways. */ +static void pl_put_headings(const pl_scan_t *s, pl_sb_t *sb) { + for (int i = 0; i < s->nsecs; i++) { + const pl_sec_t *sec = &s->secs[i]; + char key[CBM_DOCLINK_PERL_KEY_MAX]; + cbm_doclink_perl_plain_text(s->src + sec->hb, sec->he - sec->hb, key, sizeof(key)); + const char *rest = sec->item ? pl_after_bullet(key) : NULL; + if (!rest) { + pl_put_heading(sb, sec, key); + } else if (key[0] == '*') { + pl_put_heading(sb, sec, rest); + } else if (rest[0]) { + pl_put_heading(sb, sec, key); + pl_put_heading(sb, sec, rest); + } + } +} + +const char *cbm_doclink_perl_scan_scope(CBMExtractCtx *ctx) { + if (!ctx || !ctx->result || ts_node_is_null(ctx->root)) { + return NULL; + } + pl_scan_t s; + pl_scan_init(&s, ctx); + pl_sb_t sb = {.a = ctx->arena}; + pl_sb_puts(&sb, CBM_DOCLINK_PERL_SCOPE_TAG "\n"); + if (ts_node_has_error(ctx->root)) { + pl_sb_puts(&sb, "E\n"); + } + pl_walk_t w = {.s = &s, .current = PL_PKG_MAIN, .pending_block = PL_PKG_UNKNOWN}; + pl_walk_tree(&w, ctx->root); + pl_number_packages(&w, &sb); + pl_put_subs(ctx, &w, &sb); + pl_scan_file(&s); + pl_put_headings(&s, &sb); + if (s.failed || sb.failed || !sb.buf) { + return NULL; + } + return sb.buf; +} diff --git a/src/pipeline/doc_links.c b/src/pipeline/doc_links.c index a365ddf41..c912ac9e0 100644 --- a/src/pipeline/doc_links.c +++ b/src/pipeline/doc_links.c @@ -47,7 +47,7 @@ const char *cbm_doclink_reason_name(int reason) { static const cbm_doclink_resolver_t *const DOCLINK_RESOLVERS[] = { &cbm_doclink_cs_resolver, &cbm_doclink_md_resolver, &cbm_doclink_pdf_resolver, &cbm_doclink_rst_resolver, &cbm_doclink_adoc_resolver, &cbm_doclink_go_resolver, - &cbm_doclink_jvm_resolver, &cbm_doclink_rs_resolver, + &cbm_doclink_jvm_resolver, &cbm_doclink_rs_resolver, &cbm_doclink_perl_resolver, }; enum { DOCLINK_RESOLVER_COUNT = sizeof(DOCLINK_RESOLVERS) / sizeof(DOCLINK_RESOLVERS[0]) }; diff --git a/src/pipeline/doc_links.h b/src/pipeline/doc_links.h index a85e3d512..e0ca5820f 100644 --- a/src/pipeline/doc_links.h +++ b/src/pipeline/doc_links.h @@ -298,6 +298,7 @@ bool cbm_doclinks_binds_lines(CBMLanguage lang); extern const cbm_doclink_resolver_t cbm_doclink_go_resolver; extern const cbm_doclink_resolver_t cbm_doclink_jvm_resolver; extern const cbm_doclink_resolver_t cbm_doclink_rs_resolver; +extern const cbm_doclink_resolver_t cbm_doclink_perl_resolver; /* True when `rel_path` is a scope input of some language (scope_input). */ bool cbm_doclinks_is_scope_input(const char *rel_path); diff --git a/src/pipeline/doc_links_perl.c b/src/pipeline/doc_links_perl.c new file mode 100644 index 000000000..8736ce506 --- /dev/null +++ b/src/pipeline/doc_links_perl.c @@ -0,0 +1,489 @@ +/* + * doc_links_perl.c — resolving Perl POD L<> references (doclink_perl.c). + * + * A page is a module. L binds the module whose file declares + * `package Mojo::Base;` (the scope blob's P records). When several files + * declare it, the one the library layout names (…/Mojo/Base.pm) is the + * module; more than one such file, or none among several, is ambiguous. A page + * the repository does not declare (CPAN, the Perl core, a .pod page: + * discovery keeps no .pod file, so nothing can say it is in the repository) + * is external, and so is a man page (L). + * + * The target of a module is its File node: a POD page is the file it is + * written in (the field test's audited rule). Its Module node would do for + * most, but not where a same-named Folder holds the module's qualified name + * (lib/Mojo.pm next to lib/Mojo/). + * + * A section, L: when a heading of the module's file reads + * `section` (POD text as it renders, cbm_doclink_perl_plain_text) and names a + * sub that has a definition in that file, the target is the sub; headings + * that name two subs are ambiguous. A section that names no sub is a place in + * the module's documentation: the target is the module. L is the + * same page: a sub its heading names binds; a heading that names no sub is a + * place in the source's own page, no reference worth an edge or a row + * (local); a section the page has no heading for is missing (doc rot). + * + * Test code (t/, xt/, *.t, and cbm_is_test_file) is bound only from test code. + * Every edge is exact: a module name and a heading are spelled out in full. + * + * The scope blob carries no lines (doclink_perl.c), and without a scope_delta + * hook every change of a blob is a full re-resolution: a change that adds or + * drops a package, a sub or a heading; an edit inside a sub changes no blob. + */ +#include "pipeline/doc_links.h" +#include "pipeline/pipeline_internal.h" /* cbm_pipeline_file_node */ +#include "cbm.h" +#include "doclink.h" +#include "helpers.h" /* cbm_fqn_module_source_lang, cbm_is_test_file */ +#include "foundation/arena.h" +#include "foundation/constants.h" +#include "foundation/hash_table.h" +#include "foundation/log.h" +#include "foundation/mem_core.h" +#include "graph_buffer/graph_buffer.h" + +#include +#include +#include +#include +#include +#include + +/* ── The index ───────────────────────────────────────────────────── */ + +typedef struct { + const char *sub; /* the sub the heading names; NULL: none */ + const char *key; /* the heading as it renders */ +} pr_head_t; + +typedef struct { + const char *path; /* rel_path */ + const char *module_qn; /* the file's module qualified name */ + bool test; + int nsubs; + const char **subs; /* subs with a definition in the file */ + int nheads; + pr_head_t *heads; +} pr_file_t; + +typedef struct { + const char *name; + int file; +} pr_decl_t; + +typedef struct { + int first; /* into decls */ + int count; +} pr_range_t; + +typedef struct { + CBMArena arena; + const char *project; + pr_file_t *files; + int nfiles; + int *run_to_file; /* run_file -> files index, -1: none */ + int run_count; + pr_decl_t *decls; /* every package declaration, by name, then path */ + int ndecls; + CBMHashTable *pkg; /* package name -> pr_range_t */ + bool oom; +} pr_index_t; + +static bool pr_test_path(const char *path) { + if (cbm_is_test_file(path, CBM_LANG_PERL)) { + return true; + } + size_t n = strlen(path); + if (n >= PAIR_LEN && strcmp(path + n - PAIR_LEN, ".t") == 0) { + return true; + } + return strncmp(path, "t/", PAIR_LEN) == 0 || strncmp(path, "xt/", CBM_SZ_3) == 0 || + strstr(path, "/t/") != NULL || strstr(path, "/xt/") != NULL; +} + +/* One record line of a blob: its fields, split at tabs (at most `cap`). */ +static int pr_fields(const char *b, const char *e, const char **f, size_t *fl, int cap) { + int n = 0; + const char *s = b; + for (const char *p = b; p <= e && n < cap; p++) { + if (p == e || *p == '\t') { + f[n] = s; + fl[n] = (size_t)(p - s); + n++; + s = p + SKIP_ONE; + } + } + return n; +} + +enum { PR_MAX_FIELDS = 4, PR_F_TAG = 0, PR_F_A = 1, PR_F_B = 2 }; + +static const char *pr_copy(pr_index_t *ix, const char *s, size_t n) { + char *c = cbm_arena_strndup(&ix->arena, s, n); + ix->oom = ix->oom || !c; + return c; +} + +/* Count, then fill: the records of one file's blob. */ +static void pr_read_blob(pr_index_t *ix, int fi, const char *blob, bool fill) { + pr_file_t *f = &ix->files[fi]; + int np = 0; + int ns = 0; + int nh = 0; + const char *line = strchr(blob, '\n'); /* the tag line */ + while (line && !ix->oom) { + const char *b = line + SKIP_ONE; + const char *e = strchr(b, '\n'); + if (!e) { + e = b + strlen(b); + } + const char *fld[PR_MAX_FIELDS]; + size_t fln[PR_MAX_FIELDS]; + int n = pr_fields(b, e, fld, fln, PR_MAX_FIELDS); + if (n >= PAIR_LEN && fln[PR_F_TAG] == SKIP_ONE) { + char tag = fld[PR_F_TAG][0]; + if (tag == 'P') { + if (fill) { + ix->decls[ix->ndecls++] = + (pr_decl_t){pr_copy(ix, fld[PR_F_A], fln[PR_F_A]), fi}; + } + np++; + } else if (tag == 'S' && n >= CBM_SZ_3) { + if (fill) { + f->subs[f->nsubs++] = pr_copy(ix, fld[PR_F_B], fln[PR_F_B]); + } + ns++; + } else if (tag == 'H' && n >= CBM_SZ_3) { + if (fill) { + bool none = fln[PR_F_A] == SKIP_ONE && fld[PR_F_A][0] == '-'; + f->heads[f->nheads++] = (pr_head_t){ + none ? NULL : pr_copy(ix, fld[PR_F_A], fln[PR_F_A]), + pr_copy(ix, fld[PR_F_B], fln[PR_F_B]), + }; + } + nh++; + } + } + line = *e ? e : NULL; + } + if (!fill) { + ix->ndecls += np; + f->subs = + ns ? (const char **)cbm_arena_alloc(&ix->arena, (size_t)ns * sizeof(char *)) : NULL; + f->heads = + nh ? (pr_head_t *)cbm_arena_alloc(&ix->arena, (size_t)nh * sizeof(pr_head_t)) : NULL; + ix->oom = ix->oom || (ns && !f->subs) || (nh && !f->heads); + } +} + +static int pr_decl_cmp(const void *a, const void *b) { + const pr_decl_t *x = (const pr_decl_t *)a; + const pr_decl_t *y = (const pr_decl_t *)b; + int c = strcmp(x->name, y->name); + return c != 0 ? c : (x->file > y->file) - (x->file < y->file); +} + +static void pr_destroy(void *index) { + pr_index_t *ix = (pr_index_t *)index; + if (!ix) { + return; + } + cbm_ht_free(ix->pkg); + cbm_free(CBM_MEM_CLASS_OTHER, ix->run_to_file); + cbm_arena_destroy(&ix->arena); + cbm_free(CBM_MEM_CLASS_OTHER, ix); +} + +static bool pr_index_files(pr_index_t *ix, const cbm_doclink_build_in_t *in) { + ix->nfiles = in->file_count; + ix->files = + in->file_count + ? (pr_file_t *)cbm_arena_calloc(&ix->arena, (size_t)in->file_count * sizeof(pr_file_t)) + : NULL; + ix->run_count = in->run_file_count; + ix->run_to_file = (int *)cbm_calloc(CBM_MEM_CLASS_OTHER, + (size_t)(in->run_file_count + SKIP_ONE) * sizeof(int)); + if ((in->file_count && !ix->files) || !ix->run_to_file) { + return false; + } + for (int r = 0; r < in->run_file_count; r++) { + ix->run_to_file[r] = CBM_NOT_FOUND; + } + for (int i = 0; i < in->file_count && !ix->oom; i++) { + const cbm_doclink_file_t *src = &in->files[i]; + pr_file_t *f = &ix->files[i]; + f->path = src->rel_path; /* valid until destroy */ + f->module_qn = + cbm_fqn_module_source_lang(&ix->arena, ix->project, src->rel_path, CBM_LANG_PERL); + f->test = pr_test_path(src->rel_path); + ix->oom = ix->oom || !f->module_qn; + if (src->run_file >= 0 && src->run_file < in->run_file_count) { + ix->run_to_file[src->run_file] = i; + } + if (src->scope) { + pr_read_blob(ix, i, src->scope, false); + } + } + return !ix->oom; +} + +static bool pr_index_packages(pr_index_t *ix, const cbm_doclink_build_in_t *in) { + int total = ix->ndecls; + ix->ndecls = 0; + ix->decls = + total ? (pr_decl_t *)cbm_arena_alloc(&ix->arena, (size_t)total * sizeof(pr_decl_t)) : NULL; + if (total && !ix->decls) { + return false; + } + for (int i = 0; i < in->file_count && !ix->oom; i++) { + if (in->files[i].scope) { + pr_read_blob(ix, i, in->files[i].scope, true); + } + } + if (ix->oom) { + return false; + } + qsort(ix->decls, (size_t)ix->ndecls, sizeof(pr_decl_t), pr_decl_cmp); + ix->pkg = cbm_ht_create_in(CBM_MEM_CLASS_OTHER, (uint32_t)(ix->ndecls + SKIP_ONE)); + if (!ix->pkg) { + return false; + } + for (int i = 0; i < ix->ndecls;) { + int j = i; + while (j < ix->ndecls && strcmp(ix->decls[j].name, ix->decls[i].name) == 0) { + j++; + } + pr_range_t *r = (pr_range_t *)cbm_arena_alloc(&ix->arena, sizeof(pr_range_t)); + if (!r) { + return false; + } + *r = (pr_range_t){i, j - i}; + cbm_ht_set(ix->pkg, ix->decls[i].name, r); + i = j; + } + return true; +} + +static void *pr_build(const cbm_doclink_build_in_t *in) { + pr_index_t *ix = (pr_index_t *)cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*ix)); + if (!ix) { + return NULL; + } + cbm_arena_init(&ix->arena); + ix->project = in->ctx->project_name; + if (!pr_index_files(ix, in) || !pr_index_packages(ix, in)) { + cbm_log_error("doc_links.perl.error", "step", "index", "reason", "alloc"); + pr_destroy(ix); + return NULL; + } + char b[PAIR_LEN][CBM_SZ_32]; + snprintf(b[0], sizeof(b[0]), "%d", ix->nfiles); + snprintf(b[SKIP_ONE], sizeof(b[SKIP_ONE]), "%u", cbm_ht_count(ix->pkg)); + cbm_log_info("doc_links.perl.index", "files", b[0], "packages", b[SKIP_ONE]); + return ix; +} + +/* ── Resolution ──────────────────────────────────────────────────── */ + +typedef struct { + const pr_index_t *ix; + const cbm_gbuf_t *g; + const pr_file_t *src; + cbm_doclink_outcome_t *out; +} pr_ctx_t; + +static void pr_row(const pr_ctx_t *c, int reason) { + c->out->kind = CBM_DOCLINK_UNRESOLVED; + c->out->reason = reason; +} + +static void pr_edge(const pr_ctx_t *c, const pr_file_t *f, const cbm_gbuf_node_t *node) { + if (!node) { + pr_row(c, CBM_DOCLINK_REASON_GRAPH_GAP); + return; + } + if (f->test && !c->src->test) { + pr_row(c, CBM_DOCLINK_REASON_TEST_ONLY); + return; + } + c->out->kind = CBM_DOCLINK_EDGE; + c->out->target = node; + c->out->exact = true; +} + +static bool pr_node_is(const cbm_gbuf_node_t *n, const pr_file_t *f, const char *label) { + return n && n->label && n->file_path && strcmp(n->label, label) == 0 && + strcmp(n->file_path, f->path) == 0; +} + +/* The module's node: its File node. */ +static const cbm_gbuf_node_t *pr_module_node(const pr_ctx_t *c, const pr_file_t *f) { + return cbm_pipeline_file_node(c->g, c->ix->project, f->path); +} + +/* The node of sub `name` of file f; NULL when it has none of its own. */ +static const cbm_gbuf_node_t *pr_sub_node(const pr_ctx_t *c, const pr_file_t *f, const char *name) { + size_t ql = strlen(f->module_qn); + size_t nl = strlen(name); + char stack[CBM_SZ_512]; + char *qn = ql + nl + PAIR_LEN <= sizeof(stack) + ? stack + : (char *)cbm_alloc(CBM_MEM_CLASS_OTHER, ql + nl + PAIR_LEN); + if (!qn) { + return NULL; + } + memcpy(qn, f->module_qn, ql); + qn[ql] = '.'; + memcpy(qn + ql + SKIP_ONE, name, nl + SKIP_ONE); + const cbm_gbuf_node_t *n = cbm_gbuf_find_by_qn(c->g, qn); + if (qn != stack) { + cbm_free(CBM_MEM_CLASS_OTHER, qn); + } + return (pr_node_is(n, f, "Function") || pr_node_is(n, f, "Method")) ? n : NULL; +} + +static bool pr_has_sub(const pr_file_t *f, const char *name) { + for (int i = 0; i < f->nsubs; i++) { + if (strcmp(f->subs[i], name) == 0) { + return true; + } + } + return false; +} + +/* The one sub a heading `key` of file f names (it has a definition there): + * 1 found, 0 none, 2 several. */ +static int pr_heading_sub(const pr_file_t *f, const char *key, const char **sub) { + *sub = NULL; + for (int i = 0; i < f->nheads; i++) { + const pr_head_t *h = &f->heads[i]; + if (!h->sub || strcmp(h->key, key) != 0 || !pr_has_sub(f, h->sub)) { + continue; + } + if (*sub && strcmp(*sub, h->sub) != 0) { + return PAIR_LEN; + } + *sub = h->sub; + } + return *sub ? SKIP_ONE : 0; +} + +static bool pr_has_heading(const pr_file_t *f, const char *key) { + for (int i = 0; i < f->nheads; i++) { + if (strcmp(f->heads[i].key, key) == 0) { + return true; + } + } + return false; +} + +static void pr_section(const pr_ctx_t *c, const pr_file_t *f, const char *section, bool same_page) { + char key[CBM_DOCLINK_PERL_KEY_MAX]; + cbm_doclink_perl_plain_text(section, strlen(section), key, sizeof(key)); + const char *sub = NULL; + int found = pr_heading_sub(f, key, &sub); + if (found > SKIP_ONE) { + pr_row(c, CBM_DOCLINK_REASON_AMBIGUOUS); + } else if (found == SKIP_ONE) { + pr_edge(c, f, pr_sub_node(c, f, sub)); + } else if (same_page && pr_has_heading(f, key)) { + c->out->kind = CBM_DOCLINK_LOCAL; /* a place in the source's own page */ + } else if (same_page) { + pr_row(c, CBM_DOCLINK_REASON_MISSING); /* a section the page does not have */ + } else { + pr_edge(c, f, pr_module_node(c, f)); + } +} + +/* True when `path` is the file the library layout gives module `page`. */ +static bool pr_layout_names(const char *path, const char *page) { + char want[CBM_DOCLINK_PERL_KEY_MAX + CBM_SZ_8]; + size_t w = 0; + for (const char *p = page; *p && w + CBM_SZ_4 < sizeof(want); p++) { + if (p[0] == ':' && p[SKIP_ONE] == ':') { + want[w++] = '/'; + p++; + } else { + want[w++] = *p; + } + } + memcpy(want + w, ".pm", CBM_SZ_4); + w += CBM_SZ_3; + size_t n = strlen(path); + return n >= w && strcmp(path + n - w, want) == 0 && (n == w || path[n - w - SKIP_ONE] == '/'); +} + +/* The file of module `page`: NULL with *ambiguous false when the repository + * declares no such package. */ +static const pr_file_t *pr_page_file(const pr_index_t *ix, const char *page, bool *ambiguous) { + *ambiguous = false; + const pr_range_t *r = (const pr_range_t *)cbm_ht_get(ix->pkg, page); + if (!r || r->count == 0) { + return NULL; + } + if (r->count == SKIP_ONE) { + return &ix->files[ix->decls[r->first].file]; + } + const pr_file_t *hit = NULL; + for (int i = r->first; i < r->first + r->count; i++) { + const pr_file_t *f = &ix->files[ix->decls[i].file]; + if (f == hit || !pr_layout_names(f->path, page)) { + continue; + } + if (hit) { + *ambiguous = true; + return NULL; + } + hit = f; + } + *ambiguous = hit == NULL; + return hit; +} + +static void pr_resolve(const void *index, void *state, int run_file, const CBMDocLink *link, + const cbm_gbuf_t *graph, cbm_doclink_outcome_t *out) { + (void)state; /* Perl keeps no per-file working state */ + const pr_index_t *ix = (const pr_index_t *)index; + out->kind = CBM_DOCLINK_UNRESOLVED; + out->reason = CBM_DOCLINK_REASON_MISSING; + out->target = NULL; + out->exact = false; + if (!ix || !link->raw || run_file < 0 || run_file >= ix->run_count || + ix->run_to_file[run_file] < 0) { + return; + } + pr_ctx_t c = {.ix = ix, .g = graph, .src = &ix->files[ix->run_to_file[run_file]], .out = out}; + CBMPerlLink pl; + if (!cbm_doclink_perl_parse_link(link->raw, &pl)) { + pr_row(&c, CBM_DOCLINK_REASON_UNPARSEABLE); + return; + } + if (pl.kind == CBM_PERL_LINK_MAN) { + pr_row(&c, CBM_DOCLINK_REASON_EXTERNAL); + return; + } + if (pl.kind == CBM_PERL_LINK_SECTION) { + pr_section(&c, c.src, pl.section, true); + return; + } + bool ambiguous = false; + const pr_file_t *f = pr_page_file(ix, pl.page, &ambiguous); + if (!f) { + pr_row(&c, ambiguous ? CBM_DOCLINK_REASON_AMBIGUOUS : CBM_DOCLINK_REASON_EXTERNAL); + return; + } + if (pl.section[0]) { + pr_section(&c, f, pl.section, false); + return; + } + pr_edge(&c, f, pr_module_node(&c, f)); +} + +const cbm_doclink_resolver_t cbm_doclink_perl_resolver = { + .langs = {CBM_LANG_PERL}, + .lang_count = 1, + .scope_tag = CBM_DOCLINK_PERL_SCOPE_TAG, + .build = pr_build, + .destroy = pr_destroy, + .resolve = pr_resolve, +}; diff --git a/tests/test_doc_mentions_helpers.h b/tests/test_doc_mentions_helpers.h index e37ee9368..f0e2312e1 100644 --- a/tests/test_doc_mentions_helpers.h +++ b/tests/test_doc_mentions_helpers.h @@ -439,7 +439,7 @@ static const int DM_HELD_FAMILIES[] = { CBM_DOCLINK_JAVA_VALUE, CBM_DOCLINK_KT_KDOC_LINK, CBM_DOCLINK_KT_SEE, CBM_DOCLINK_KT_THROWS, CBM_DOCLINK_KT_SAMPLE, CBM_DOCLINK_RS_INTRA_DOC, - CBM_DOCLINK_RS_LINK_DEF, + CBM_DOCLINK_RS_LINK_DEF, CBM_DOCLINK_PERL_POD_L, }; static inline void dm_ship_held_families(void) { diff --git a/tests/test_doc_mentions_perl.c b/tests/test_doc_mentions_perl.c new file mode 100644 index 000000000..177d4a9d5 --- /dev/null +++ b/tests/test_doc_mentions_perl.c @@ -0,0 +1,292 @@ +/* + * test_doc_mentions_perl.c -- Perl POD L<> codes as doc-comment references. + * + * A sub's doc is the POD section that names it; the file's module doc (its + * File node's docstring) is the POD no sub holds. The L<> codes of both are + * the references: module-level ones come from the File node. + */ +#include "../src/foundation/compat.h" +#include "test_framework.h" +#include "test_helpers.h" +#include "test_doc_mentions_helpers.h" + +#include "cbm.h" +#include "doclink.h" + +#include +#include + +/* ── extraction ──────────────────────────────────────────────────── */ + +static const char *const PL_KNOB = + "package Knob;\n" /* 1 */ + "\n" /* 2 */ + "=head1 NAME\n" /* 3 */ + "\n" /* 4 */ + "Knob - see L and L.\n" /* 5 */ + "\n" /* 6 */ + "=cut\n" /* 7 */ + "\n" /* 8 */ + "sub turn { return 1; }\n" /* 9 */ + "1;\n" /* 10 */ + "__END__\n" /* 11 */ + "\n" /* 12 */ + "=head1 METHODS\n" /* 13 */ + "\n" /* 14 */ + "Methods, see L and L.\n" /* 15 */ + "\n" /* 16 */ + "=head2 turn\n" /* 17 */ + "\n" /* 18 */ + "Turns, like L.\n" /* 19 */ + "\n" /* 20 */ + "=head2 spin\n" /* 21 */ + "\n" /* 22 */ + "About L and L.\n" /* 23 */ + "\n" /* 24 */ + "=cut\n"; /* 25 */ + +/* The one token `raw`: its line, whether it is the file's, its family. */ +static bool pl_token_is(const CBMFileResult *r, const char *raw, uint32_t line, bool file, + int syntax) { + if (dm_count_tokens(r, raw) != 1) { + printf(" token %s: %d found\n", raw, dm_count_tokens(r, raw)); + return false; + } + const CBMDocLink *t = dm_find_token(r, raw); + bool is_file = (t->flags & CBM_DOCLINK_FLAG_FILE) != 0; + if (t->line != line || is_file != file || t->syntax != syntax) { + printf(" token %s: line %u file %d syntax %d\n", raw, t->line, is_file, t->syntax); + return false; + } + return true; +} + +TEST(doc_mentions_perl_tokens) { + CBMFileResult *r = dm_extract(PL_KNOB, CBM_LANG_PERL, "lib/Knob.pm"); + ASSERT_NOT_NULL(r); + /* the module doc's references come from the file, each on its own line */ + ASSERT_TRUE(pl_token_is(r, "Dial", 5, true, CBM_DOCLINK_PERL_POD_L)); + ASSERT_TRUE(pl_token_is(r, "the knob|Knob::Turn/turning", 5, true, CBM_DOCLINK_PERL_POD_L)); + ASSERT_TRUE(pl_token_is(r, "/turn", 15, true, CBM_DOCLINK_PERL_POD_L)); + ASSERT_TRUE(pl_token_is(r, "https://example.org/k", 15, true, CBM_DOCLINK_HREF)); + ASSERT_TRUE(pl_token_is(r, "perlfunc/open", 23, true, CBM_DOCLINK_PERL_POD_L)); + ASSERT_TRUE(pl_token_is(r, "crontab(5)", 23, true, CBM_DOCLINK_PERL_POD_L)); + /* a sub's section is the sub's, and only the sub's */ + ASSERT_TRUE(pl_token_is(r, "Knob::Spin", 19, false, CBM_DOCLINK_PERL_POD_L)); + const CBMDocLink *t = dm_find_token(r, "Knob::Spin"); + ASSERT_NOT_NULL(strstr(t->source_qn, "turn")); + ASSERT_EQ(r->doc_links.count, 7); + cbm_free_result(r); + PASS(); +} + +/* ── resolution ──────────────────────────────────────────────────── */ + +static void pl_write_repo(const char *repo) { + th_write_file(TH_PATH(repo, "lib/Knob.pm"), + "package Knob;\n" + "\n" + "=head1 NAME\n" + "\n" + "Knob - L, L, L.\n" + "\n" + "=head1 SEE ALSO\n" + "\n" + "L, L, L, L, L, L,\n" + "L.\n" + "\n" + "=head1 NOTES\n" + "\n" + "Nothing.\n" + "\n" + "=cut\n" + "\n" + "sub turn { return 1; }\n" + "\n" + "=head2 turn\n" + "\n" + "Turns, see L.\n" + "\n" + "=cut\n" + "\n" + "1;\n"); + th_write_file(TH_PATH(repo, "lib/Knob/Dial.pm"), "package Knob::Dial;\n" + "\n" + "=head1 OVERVIEW\n" + "\n" + "The dial of L.\n" + "\n" + "=cut\n" + "\n" + "sub spin { return 2; }\n" + "\n" + "=head2 spin\n" + "\n" + "Spins.\n" + "\n" + "=cut\n" + "\n" + "1;\n"); + /* a second declaration of Knob::Dial, not where the library layout puts it */ + th_write_file(TH_PATH(repo, "lib/Extra.pm"), "package Knob::Dial;\nsub extra { 3 }\n1;\n"); + /* Dup twice, neither at its layout path */ + th_write_file(TH_PATH(repo, "lib/A.pm"), "package Dup;\nsub a { 1 }\n1;\n"); + th_write_file(TH_PATH(repo, "lib/B.pm"), "package Dup;\nsub b { 1 }\n1;\n"); + th_write_file(TH_PATH(repo, "t/lib/TestHelper.pm"), + "package TestHelper;\nsub help { 1 }\n1;\n"); +} + +static bool pl_edge(const char *db, const char *src, const char *tgt) { + char props[512]; + int n = 0; + dm_edge(db, src, tgt, props, sizeof(props), &n); + if (n != 1 || !strstr(props, "\"tier\":\"exact\"")) { + printf(" edge %s -> %s: %d found %s\n", src, tgt, n, props); + return false; + } + return true; +} + +static bool pl_row(const char *db, const char *rel, const char *raw, const char *reason) { + char got[64]; + dm_row(db, rel, raw, got, sizeof(got), NULL, 0); + if (strcmp(got, reason) != 0) { + printf(" row %s [%s]: \"%s\", expected \"%s\"\n", rel, raw, got, reason); + return false; + } + return true; +} + +TEST(doc_mentions_perl_resolution) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlperl_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[512]; + char db[512]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + snprintf(db, sizeof(db), "%s/g.db", tmp); + pl_write_repo(repo); + ASSERT_EQ(dm_index(repo, db, NULL), 0); + const char *knob = "lib/Knob.pm"; + /* a module the library layout names, though another file declares it too */ + ASSERT_TRUE(pl_edge(db, "lib.Knob.pm.__file__", "lib.Knob.Dial.pm.__file__")); + /* a section whose heading names a sub of the module: the sub */ + ASSERT_TRUE(pl_edge(db, "lib.Knob.pm.__file__", "lib.Knob.Dial.spin")); + ASSERT_TRUE(pl_edge(db, "lib.Knob.turn", "lib.Knob.Dial.spin")); + /* the same page: a sub binds, a section that names none is local */ + ASSERT_TRUE(pl_edge(db, "lib.Knob.pm.__file__", "lib.Knob.turn")); + ASSERT_TRUE(pl_row(db, knob, "/NOTES", "")); + /* ... and one the page has no heading for is doc rot */ + ASSERT_TRUE(pl_row(db, knob, "/GONE", "missing")); + /* outside the repository, a man page, test code, two declarations */ + ASSERT_TRUE(pl_row(db, knob, "Mojo::Base", "external")); + ASSERT_TRUE(pl_row(db, knob, "crontab(5)", "external")); + ASSERT_TRUE(pl_row(db, knob, "TestHelper", "test_only_target")); + ASSERT_TRUE(pl_row(db, knob, "Dup", "ambiguous")); + /* the page is the file, also where a Folder holds lib/Knob.pm's module name */ + ASSERT_TRUE(pl_edge(db, "lib.Knob.Dial.pm.__file__", "lib.Knob.pm.__file__")); + th_cleanup(tmp); + PASS(); +} + +/* ── incremental == full; one worker == several ──────────────────── */ + +#define PL_INC_DIAL \ + "package Knob::Dial;\n\n=head1 OVERVIEW\n\nThe dial.\n\n=cut\n\nsub spin { 2 }\n\n" \ + "=head2 spin\n\nSpins.\n\n=cut\n\n1;\n" + +TEST(doc_mentions_perl_incremental_equals_full) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlperl_inc_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[512]; + char inc_db[512]; + char full_db[512]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + snprintf(inc_db, sizeof(inc_db), "%s/inc.db", tmp); + snprintf(full_db, sizeof(full_db), "%s/full.db", tmp); + th_write_file(TH_PATH(repo, "lib/Knob.pm"), + "package Knob;\n\n=head1 NAME\n\nKnob - see L.\n\n=cut\n\n" + "sub turn { 1 }\n1;\n"); + th_write_file(TH_PATH(repo, "lib/Knob/Dial.pm"), PL_INC_DIAL); + ASSERT_EQ(dm_index(repo, inc_db, NULL), 0); + ASSERT_TRUE(pl_edge(inc_db, "lib.Knob.pm.__file__", "lib.Knob.Dial.spin")); + + /* the module doc's text changes: the file is repaired alone */ + th_write_file( + TH_PATH(repo, "lib/Knob.pm"), + "package Knob;\n\n=head1 NAME\n\nKnob, a knob - see L.\n\n=cut\n\n" + "sub turn { 1 }\n1;\n"); + ASSERT_EQ( + dm_step(repo, inc_db, full_db, "module doc edit", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0); + ASSERT_TRUE(pl_edge(inc_db, "lib.Knob.pm.__file__", "lib.Knob.Dial.spin")); + + /* a reference added to the module doc: an edge from the File node */ + th_write_file(TH_PATH(repo, "lib/Knob.pm"), + "package Knob;\n\n=head1 NAME\n\nKnob, a knob - see L and\n" + "L.\n\n=cut\n\nsub turn { 1 }\n1;\n"); + ASSERT_EQ( + dm_step(repo, inc_db, full_db, "module doc link", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0); + ASSERT_TRUE(pl_edge(inc_db, "lib.Knob.pm.__file__", "lib.Knob.Dial.pm.__file__")); + + /* a new heading changes the file's scope blob: without a scope_delta hook + * that is a full run */ + th_write_file(TH_PATH(repo, "lib/Knob.pm"), + "package Knob;\n\n=head1 NAME\n\nKnob, a knob - see L and\n" + "L.\n\n=head1 SEE ALSO\n\nL\n\n=cut\n\n" + "sub turn { 1 }\n1;\n"); + ASSERT_EQ(dm_step(repo, inc_db, full_db, "heading added", CBM_INCREMENTAL_ROUTE_FORCED_FULL), + 0); + + /* the heading that named the sub goes: another file's section reference + * now names a place in the module's documentation, the module */ + th_write_file( + TH_PATH(repo, "lib/Knob/Dial.pm"), + "package Knob::Dial;\n\n=head1 OVERVIEW\n\nThe dial.\n\n=cut\n\nsub spin { 2 }\n1;\n"); + ASSERT_EQ(dm_step(repo, inc_db, full_db, "heading removed", CBM_INCREMENTAL_ROUTE_FORCED_FULL), + 0); + char props[512]; + int n = 0; + dm_edge(inc_db, "lib.Knob.pm.__file__", "lib.Knob.Dial.spin", props, sizeof(props), &n); + ASSERT_EQ(n, 0); + th_cleanup(tmp); + PASS(); +} + +TEST(doc_mentions_perl_workers_agree) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlperl_par_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[512]; + char par_db[512]; + char seq_db[512]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + snprintf(par_db, sizeof(par_db), "%s/par.db", tmp); + snprintf(seq_db, sizeof(seq_db), "%s/seq.db", tmp); + enum { MODULES = 60 }; /* past the parallel threshold */ + for (int i = 0; i < MODULES; i++) { + char rel[64]; + char src[1024]; + int next = (i + 1) % MODULES; + snprintf(rel, sizeof(rel), "lib/Par/M%02d.pm", i); + snprintf(src, sizeof(src), + "package Par::M%02d;\n\n=head1 NAME\n\nPar::M%02d - see L,\n" + "L, L, L and L.\n\n=cut\n\n" + "sub run%02d { %d }\n\n=head2 run%02d\n\nRuns; also L.\n\n" + "=cut\n\n1;\n", + i, i, next, next, next, i, i, i, i, next); + th_write_file(TH_PATH(repo, rel), src); + } + ASSERT_EQ(dm_workers_agree(repo, par_db, seq_db), 0); + ASSERT_TRUE(pl_edge(par_db, "lib.Par.M00.pm.__file__", "lib.Par.M01.run01")); + th_cleanup(tmp); + PASS(); +} + +SUITE(doc_mentions_perl) { + dm_ship_held_families(); + RUN_TEST(doc_mentions_perl_tokens); + RUN_TEST(doc_mentions_perl_resolution); + RUN_TEST(doc_mentions_perl_incremental_equals_full); + RUN_TEST(doc_mentions_perl_workers_agree); +} diff --git a/tests/test_main.c b/tests/test_main.c index ae29520be..8c2c814d1 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -1828,6 +1828,7 @@ extern void suite_doc_links_adoc(void); extern void suite_doc_mentions_go(void); extern void suite_doc_mentions_jvm(void); extern void suite_doc_mentions_rs(void); +extern void suite_doc_mentions_perl(void); extern void suite_infrascan(void); extern void suite_cli(void); extern void suite_agent_clients(void); @@ -2302,6 +2303,7 @@ int main(int argc, char **argv) { RUN_SELECTED_SUITE(doc_mentions_go); RUN_SELECTED_SUITE(doc_mentions_jvm); RUN_SELECTED_SUITE(doc_mentions_rs); + RUN_SELECTED_SUITE(doc_mentions_perl); /* Infrastructure scanning */ RUN_SELECTED_SUITE(infrascan); From 196ca71cda6000d2597107636d9bb7b5aeb250e3 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 10 Oct 2026 15:43:40 +0200 Subject: [PATCH 06/17] doc-links(ts): JSDoc and TSDoc links become MENTIONS edges, held until their audit A TypeScript, TSX or JavaScript doc comment (a block comment opening with a slash and two stars) names code with {@link Ref}, {@linkcode Ref} (each with an optional `|` label) and `@see Ref` at the start of a tag line that holds no inline tag. doclink_ts.c harvests them (code spans and fenced blocks hold none, {@linkplain} is not harvested, a hyphenated word after @see is prose, a reference naming the declaration's own parameter is dropped, a URL is the core's href family) and writes each file's scope: imports, exports, export * barrels, top-level declarations, namespace bodies and functions or classes declared inside a function body. doc_links_ts.c resolves a reference as the TypeScript checker sees it from the documented declaration, in one index over TS, TSX and JS files: 1. the namespaces around the reference, innermost first; 2. a function or class declared in the enclosing function body (the graph gives it a file-level name: two of one name in one file share one node, a graph gap); 3. the file's own top-level declarations; 4. its imports, followed through export { a as b }, export ... from and export * barrels (cycle guard, depth bound; two barrels that answer differently are ambiguous); a relative specifier resolves with the usual extension and index rules, a bare one is external; 5. the standard library's global names: external; 6. the top-level declarations of script files (no import, no export): one binds, several are ambiguous; 7. the members of the documented class or interface, `this.` included. The other segments walk members or a module's exports. A member declared without a node of its own is a graph gap; test code binds only from test code. There is no scope_delta hook yet: an export change is a full run. The families are held (ships=false) until their held-out audit passes; the suite ships them through the test seam. Proof: the TypeScript compiler (perf-bench/typescript b465fdbfe175), ship build, against the field test's prototype records (cascade results left out): 69 edges; 57 of the prototype's 61 pairs bound alike. The 4 others are doc blocks without a definition of their own (on an `if` statement, a local `var`, an enum member, a free-standing @typedef block), which the core does not harvest. Of the 12 edges the prototype has no pair for, 4 are the same targets the prototype resolved without recording a source node, and 8 bind PropertyDescriptor to the repository's own src/lib/es5.d.ts, which the prototype called an external library global (kept: the repository declares it). Unresolved rows agree with the prototype's reasons apart from the documented reason split (not_in_scope -> missing/ambiguous). Known limit: package.json and tsconfig.json are not indexed (discovery ignores them), so workspace packages and tsconfig `paths` stay external. Tests (doc_mentions_ts): tokens with their lines; the scope blob and its persisted form; every resolution rule, including a function-body declaration that binds and two of one name that do not; incremental equals full (a doc edit and a body edit repair one file, an export change is a full run); 60 modules give the same edges and rows with one worker and several. Each rule red with it reverted (the body declaration's visibility and its line check, barrel ambiguity, the parameter rule, code spans, the library globals). Signed-off-by: Martin Vogel --- Makefile.cbm | 4 +- internal/cbm/doclink.c | 17 + internal/cbm/doclink.h | 18 + internal/cbm/doclink_ts.c | 905 +++++++++++++++++++++++++ src/pipeline/doc_links.c | 1 + src/pipeline/doc_links.h | 1 + src/pipeline/doc_links_ts.c | 1021 +++++++++++++++++++++++++++++ tests/test_doc_mentions_helpers.h | 2 + tests/test_doc_mentions_ts.c | 343 ++++++++++ tests/test_main.c | 2 + 10 files changed, 2313 insertions(+), 1 deletion(-) create mode 100644 internal/cbm/doclink_ts.c create mode 100644 src/pipeline/doc_links_ts.c create mode 100644 tests/test_doc_mentions_ts.c diff --git a/Makefile.cbm b/Makefile.cbm index b8aa4f71e..b6341756c 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -390,6 +390,7 @@ EXTRACTION_SRCS = \ $(CBM_DIR)/doclink_jvm.c \ $(CBM_DIR)/doclink_rs.c \ $(CBM_DIR)/doclink_perl.c \ + $(CBM_DIR)/doclink_ts.c \ $(CBM_DIR)/sql_values.c \ $(CBM_DIR)/helpers.c \ $(CBM_DIR)/cpp_branch_views.c \ @@ -520,6 +521,7 @@ PIPELINE_SRCS = \ src/pipeline/doc_links_jvm.c \ src/pipeline/doc_links_rs.c \ src/pipeline/doc_links_perl.c \ + src/pipeline/doc_links_ts.c \ src/pipeline/doc_links_msbuild.c \ src/pipeline/pass_route_nodes.c \ src/pipeline/pass_enrichment.c \ @@ -782,7 +784,7 @@ TEST_DISCOVER_SRCS = \ TEST_GRAPH_BUFFER_SRCS = tests/test_graph_buffer.c -TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_pipeline_conventions.c tests/test_pipeline_frozen.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_doclinks.c tests/test_doc_mentions.c tests/test_doc_links_md.c tests/test_doc_links_pdf.c tests/test_doc_links_rst.c tests/test_doc_links_adoc.c tests/test_doc_mentions_go.c tests/test_doc_mentions_jvm.c tests/test_doc_mentions_rs.c tests/test_doc_mentions_perl.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c +TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_pipeline_conventions.c tests/test_pipeline_frozen.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_doclinks.c tests/test_doc_mentions.c tests/test_doc_links_md.c tests/test_doc_links_pdf.c tests/test_doc_links_rst.c tests/test_doc_links_adoc.c tests/test_doc_mentions_go.c tests/test_doc_mentions_jvm.c tests/test_doc_mentions_rs.c tests/test_doc_mentions_perl.c tests/test_doc_mentions_ts.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c TEST_WATCHER_SRCS = tests/test_watcher.c diff --git a/internal/cbm/doclink.c b/internal/cbm/doclink.c index 4957e5489..7231a2308 100644 --- a/internal/cbm/doclink.c +++ b/internal/cbm/doclink.c @@ -217,6 +217,23 @@ static const doclink_lang_t DOCLINK_LANGS[] = { .parse_doc = cbm_doclink_perl_parse_doc, .scan_scope = cbm_doclink_perl_scan_scope, .scope_tag = CBM_DOCLINK_PERL_SCOPE_TAG}, + /* TypeScript, TSX and JavaScript: one scope, since their files import + * each other */ + {.lang = CBM_LANG_TYPESCRIPT, + .parse_doc = cbm_doclink_ts_parse_doc, + .scan_scope = cbm_doclink_ts_scan_scope, + .scope_tag = CBM_DOCLINK_TS_SCOPE_TAG, + .portable_scope = cbm_doclink_ts_portable_scope}, + {.lang = CBM_LANG_TSX, + .parse_doc = cbm_doclink_ts_parse_doc, + .scan_scope = cbm_doclink_ts_scan_scope, + .scope_tag = CBM_DOCLINK_TS_SCOPE_TAG, + .portable_scope = cbm_doclink_ts_portable_scope}, + {.lang = CBM_LANG_JAVASCRIPT, + .parse_doc = cbm_doclink_ts_parse_doc, + .scan_scope = cbm_doclink_ts_scan_scope, + .scope_tag = CBM_DOCLINK_TS_SCOPE_TAG, + .portable_scope = cbm_doclink_ts_portable_scope}, }; static const doclink_lang_t *doclink_lang(CBMLanguage lang) { diff --git a/internal/cbm/doclink.h b/internal/cbm/doclink.h index 479d104ff..c1d868da5 100644 --- a/internal/cbm/doclink.h +++ b/internal/cbm/doclink.h @@ -100,6 +100,10 @@ X(CBM_DOCLINK_RS_LINK_DEF, CBM_LANG_RUST, "link_def", false, false, "MENTIONS") \ /* perl (doclink_perl.c): POD L<> codes; held until their audit passes */ \ X(CBM_DOCLINK_PERL_POD_L, CBM_LANG_PERL, "pod_L", false, false, "MENTIONS") \ + /* typescript and javascript (doclink_ts.c): JSDoc/TSDoc; held until their audit */ \ + X(CBM_DOCLINK_TS_LINK, CBM_LANG_TYPESCRIPT, "link", false, false, "MENTIONS") \ + X(CBM_DOCLINK_TS_LINKCODE, CBM_LANG_TYPESCRIPT, "linkcode", false, false, "MENTIONS") \ + X(CBM_DOCLINK_TS_SEE, CBM_LANG_TYPESCRIPT, "see", false, false, "MENTIONS") \ /* markdown (doclink_md.c): what a document's text names explicitly */ \ X(CBM_DOCLINK_MD_LINK, CBM_LANG_MARKDOWN, "link", false, true, "MENTIONS") \ X(CBM_DOCLINK_MD_PATH, CBM_LANG_MARKDOWN, "path", false, true, "MENTIONS") \ @@ -537,6 +541,20 @@ typedef enum { * are read. */ CBMDocLinkRsPathStatus cbm_doclink_rs_parse_path(const char *raw, size_t len, bool lenient, CBMDocLinkRsPath *out); +/* ── TypeScript and JavaScript (doclink_ts.c) ──────────────────────── */ + +void cbm_doclink_ts_parse_doc(CBMExtractCtx *ctx, const CBMDefinition *def, const char *doc, + uint32_t doc_line); +const char *cbm_doclink_ts_scan_scope(CBMExtractCtx *ctx); +char *cbm_doclink_ts_portable_scope(const char *scope); + +/* TS, TSX and JS files share one scope tag: one resolver indexes them all. */ +#define CBM_DOCLINK_TS_SCOPE_TAG "ts1" + +/* The longest reference a TS/JS token keeps; a longer one is cut just past + * this length and the resolver reports it as unparseable. */ +enum { CBM_DOCLINK_TS_REF_MAX = 512 }; + /* ── Perl POD (doclink_perl.c) ─────────────────────────────────────── */ void cbm_doclink_perl_parse_doc(CBMExtractCtx *ctx, const CBMDefinition *def, const char *doc, diff --git a/internal/cbm/doclink_ts.c b/internal/cbm/doclink_ts.c new file mode 100644 index 000000000..9fbcd5eb7 --- /dev/null +++ b/internal/cbm/doclink_ts.c @@ -0,0 +1,905 @@ +/* + * doclink_ts.c — TypeScript and JavaScript doc-comment references (JSDoc, + * TSDoc) and the TS/JS doc-link scope. + * + * Tokens. Only a block comment that opens with a slash and two stars is a + * doc. Its lines lose their leading white space, one `*` and one space. A + * reference is + * - {@link Ref}, {@link Ref label}, {@link Ref | label}: family `link`; + * - {@linkcode Ref ...} the same way: family `linkcode`; + * - `@see Ref` (or `@see {Ref}`) at the start of a tag line that holds no + * inline link tag: family `see`. + * The reference is the text up to the first `|` or white space; {@linkplain} + * is not harvested. A URL is the core's href family. Text in a code span + * (backticks on one line) or in a fenced code block holds no reference, and a + * hyphenated word after @see (aria-colcount) is prose. A reference that names + * a parameter of the documented declaration is the declaration's own: no + * token. + * + * Scope blob, one record per line after the tag line "ts1" (tab-separated): + * M the file has module syntax (an import or an export) + * I local kind spec orig an import binding: kind d (default), n (named, + * `orig` is the name the module exports), s (a + * namespace: `* as local`, `local = require(spec)`) + * A spec export * from spec + * S name spec export * as name from spec + * F name spec orig export { orig as name } from spec + * L name local export { local as name } (and `export default x;`: + * name "default") + * D name flags a top-level declaration: e exported, d the default + * export (flags "-" for none) + * N path first last a namespace body, path dotted from the file's top + * level, with its lines; the persisted blob keeps + * "N path" (cbm_doclink_ts_portable_scope) + * V name line first last a function or class declared inside a function + * body: its line and the body's lines (where it is + * visible); only the file's own references read it, so + * the persisted blob drops it + */ +#include "doclink.h" + +#include "arena.h" +#include "helpers.h" +#include "foundation/constants.h" +#include "foundation/mem_core.h" +#include "tree_sitter/api.h" + +#include +#include +#include +#include + +enum { TS_PREFIX_LEN = 3, TS_FENCE_LEN = 3, TS_WALK_MAX_DEPTH = 64 }; + +/* ── Tokens ──────────────────────────────────────────────────────── */ + +typedef struct { + CBMExtractCtx *ctx; + const CBMDefinition *def; +} ts_emit_t; + +static bool ts_ident_char(char c) { + return isalnum((unsigned char)c) || c == '_' || c == '$' || (unsigned char)c >= 0x80; +} + +/* A parameter named in the signature text "(a: T, ...rest, public b = 1)": + * after a top-level `(` or `,`, past modifiers and `...`, the identifier. */ +static bool ts_param_in_signature(const char *sig, const char *ref, size_t n) { + static const char *const mods[] = {"public ", "private ", "protected ", + "readonly ", "override ", NULL}; + int depth = 0; + for (const char *p = sig; *p; p++) { + if (*p == '(' || *p == '[' || *p == '{' || *p == '<') { + depth++; + } else if (*p == ')' || *p == ']' || *p == '}' || *p == '>') { + depth--; + } + bool start = (*p == '(' && depth == SKIP_ONE) || (*p == ',' && depth == SKIP_ONE); + if (!start) { + continue; + } + const char *q = p + SKIP_ONE; + for (bool again = true; again;) { + while (*q == ' ' || *q == '\t' || *q == '\n') { + q++; + } + again = false; + for (int m = 0; mods[m]; m++) { + size_t ml = strlen(mods[m]); + if (strncmp(q, mods[m], ml) == 0) { + q += ml; + again = true; + } + } + } + while (*q == '.') { + q++; + } + size_t k = 0; + while (q[k] && ts_ident_char(q[k])) { + k++; + } + if (k == n && memcmp(q, ref, n) == 0) { + return true; + } + } + return false; +} + +/* A parameter of the documented declaration. */ +static bool ts_is_param(const CBMDefinition *def, const char *ref, size_t n) { + if (!def->param_names) { + return def->signature && ts_param_in_signature(def->signature, ref, n); + } + for (int i = 0; def->param_names[i]; i++) { + const char *p = def->param_names[i]; + while (*p == '.') { + p++; /* a rest parameter */ + } + if (strlen(p) == n && memcmp(p, ref, n) == 0) { + return true; + } + } + return false; +} + +static bool ts_is_url(const char *t, size_t n) { + size_t i = 0; + while (i < n && (isalnum((unsigned char)t[i]) || t[i] == '+' || t[i] == '.' || t[i] == '-')) { + i++; + } + if (i > 0 && i + CBM_SZ_3 <= n && memcmp(t + i, "://", CBM_SZ_3) == 0) { + return true; + } + return (n > CBM_SZ_4 && strncmp(t, "www.", CBM_SZ_4) == 0) || + (n > CBM_SZ_7 && strncmp(t, "mailto:", CBM_SZ_7) == 0); +} + +static void ts_push(const ts_emit_t *em, const char *ref, size_t n, int syntax, uint32_t line) { + if (n == 0) { + return; + } + if (n > CBM_DOCLINK_TS_REF_MAX) { + n = CBM_DOCLINK_TS_REF_MAX + SKIP_ONE; /* clipped: the resolver calls it unparseable */ + } + if (ts_is_url(ref, n)) { + syntax = CBM_DOCLINK_HREF; + } else { + size_t k = 0; + while (k < n && ts_ident_char(ref[k])) { + k++; + } + if (k == n && ts_is_param(em->def, ref, n)) { + return; /* the declaration's own parameter */ + } + } + char *copy = cbm_arena_strndup(em->ctx->arena, ref, n); + if (!copy) { + return; + } + CBMDocLink link = { + .source_qn = em->def->qualified_name, + .raw = copy, + .line = line, + .def_line = em->def->start_line, + .syntax = (uint16_t)syntax, + }; + cbm_doclinks_push(&em->ctx->result->doc_links, em->ctx->arena, link); +} + +/* One doc line: its text after the leading `*`, and its source line. */ +typedef struct { + const char *p; + size_t n; + uint32_t line; +} ts_line_t; + +/* The doc's lines, its opening and closing marks cut off. */ +static int ts_doc_lines(CBMArena *a, const char *doc, uint32_t doc_line, ts_line_t **out) { + size_t dl = strlen(doc); + if (dl < TS_PREFIX_LEN || strncmp(doc, "/**", TS_PREFIX_LEN) != 0) { + return 0; + } + const char *b = doc + TS_PREFIX_LEN; + const char *e = doc + dl; + if (e - b >= PAIR_LEN && e[-SKIP_ONE] == '/' && e[-PAIR_LEN] == '*') { + e -= PAIR_LEN; + } + int cap = SKIP_ONE; + for (const char *p = b; p < e; p++) { + cap += *p == '\n'; + } + ts_line_t *lines = (ts_line_t *)cbm_arena_alloc(a, (size_t)cap * sizeof(ts_line_t)); + if (!lines) { + return 0; + } + int n = 0; + uint32_t line = doc_line; + for (const char *p = b; p <= e && n < cap;) { + const char *le = p; + while (le < e && *le != '\n') { + le++; + } + const char *s = p; + while (s < le && (*s == ' ' || *s == '\t')) { + s++; + } + if (s < le && *s == '*') { + s++; + if (s < le && *s == ' ') { + s++; + } + } + const char *t = le; + while (t > s && (t[-SKIP_ONE] == ' ' || t[-SKIP_ONE] == '\t' || t[-SKIP_ONE] == '\r')) { + t--; + } + lines[n++] = (ts_line_t){s, (size_t)(t - s), line}; + line++; + if (le >= e) { + break; + } + p = le + SKIP_ONE; + } + *out = lines; + return n; +} + +/* Mark the bytes of a line that sit in a code span (backticks paired on the + * line). */ +static void ts_mask_spans(const ts_line_t *ln, bool *mask) { + size_t i = 0; + while (i < ln->n) { + if (ln->p[i] != '`') { + i++; + continue; + } + size_t run = 0; + while (i + run < ln->n && ln->p[i + run] == '`') { + run++; + } + size_t j = i + run; + size_t close = SIZE_MAX; + while (j < ln->n) { + size_t r = 0; + while (j + r < ln->n && ln->p[j + r] == '`') { + r++; + } + if (r == run) { + close = j; + break; + } + j += r ? r : SKIP_ONE; + } + if (close == SIZE_MAX) { + i += run; + continue; + } + for (size_t k = i; k < close + run; k++) { + mask[k] = true; + } + i = close + run; + } +} + +/* The first position >= j of p[0, n) that is no white space. */ +static size_t ts_skip_space(const char *p, size_t j, size_t n) { + while (j < n && isspace((unsigned char)p[j])) { + j++; + } + return j; +} + +/* The reference of an inline tag body: up to `|`, white space or `}`. */ +static size_t ts_ref_len(const char *p, size_t n) { + size_t k = 0; + while (k < n && p[k] != '|' && p[k] != '}' && !isspace((unsigned char)p[k])) { + k++; + } + return k; +} + +/* {@link ...} and {@linkcode ...} of one line; true when the line holds one. */ +static bool ts_line_inline(const ts_emit_t *em, const ts_line_t *ln, const bool *mask) { + bool any = false; + static const char open[] = "{@link"; + size_t ol = sizeof(open) - SKIP_ONE; + for (size_t i = 0; i + ol <= ln->n; i++) { + if (mask[i] || memcmp(ln->p + i, open, ol) != 0) { + continue; + } + size_t j = i + ol; + int syntax = CBM_DOCLINK_TS_LINK; + if (j + CBM_SZ_4 <= ln->n && memcmp(ln->p + j, "code", CBM_SZ_4) == 0) { + syntax = CBM_DOCLINK_TS_LINKCODE; + j += CBM_SZ_4; + } + if (j < ln->n && ln->p[j] != '}' && !isspace((unsigned char)ln->p[j])) { + continue; /* {@linkplain} or another word */ + } + any = true; + j = ts_skip_space(ln->p, j, ln->n); + ts_push(em, ln->p + j, ts_ref_len(ln->p + j, ln->n - j), syntax, ln->line); + i = j; + } + return any; +} + +/* `@see Ref` at the start of a tag line. */ +static void ts_line_see(const ts_emit_t *em, const ts_line_t *ln) { + static const char tag[] = "@see"; + size_t tl = sizeof(tag) - SKIP_ONE; + if (ln->n < tl || memcmp(ln->p, tag, tl) != 0 || + (ln->n > tl && !isspace((unsigned char)ln->p[tl]))) { + return; + } + size_t j = ts_skip_space(ln->p, tl, ln->n); + const char *r = ln->p + j; + size_t n = ln->n - j; + if (n > 0 && r[0] == '{') { + r++; + n--; + size_t k = 0; + while (k < n && r[k] != '}') { + k++; + } + n = k; + } + size_t k = 0; + while (k < n && !isspace((unsigned char)r[k])) { + k++; + } + while (k > 0 && (r[k - SKIP_ONE] == '.' || r[k - SKIP_ONE] == ',' || r[k - SKIP_ONE] == ';' || + r[k - SKIP_ONE] == ':')) { + k--; /* sentence punctuation */ + } + size_t w = 0; + while (w < k && ts_ident_char(r[w])) { + w++; + } + if (w > 0 && w < k && r[w] == '-') { + return; /* a hyphenated word: prose */ + } + ts_push(em, r, k, CBM_DOCLINK_TS_SEE, ln->line); +} + +void cbm_doclink_ts_parse_doc(CBMExtractCtx *ctx, const CBMDefinition *def, const char *doc, + uint32_t doc_line) { + if (!ctx || !ctx->result || !def || !doc || !def->qualified_name) { + return; + } + CBMArena *a = ctx->scratch ? ctx->scratch : ctx->arena; + ts_line_t *lines = NULL; + int n = ts_doc_lines(a, doc, doc_line, &lines); + ts_emit_t em = {ctx, def}; + bool fence = false; + for (int i = 0; i < n; i++) { + const ts_line_t *ln = &lines[i]; + if (ln->n >= TS_FENCE_LEN && memcmp(ln->p, "```", TS_FENCE_LEN) == 0) { + fence = !fence; + continue; + } + if (fence || ln->n == 0) { + continue; + } + bool *mask = (bool *)cbm_arena_calloc(a, ln->n + SKIP_ONE); + if (!mask) { + return; + } + ts_mask_spans(ln, mask); + if (!ts_line_inline(&em, ln, mask)) { + ts_line_see(&em, ln); + } + } +} + +/* ── Scope blob ──────────────────────────────────────────────────── */ + +typedef struct { + CBMArena *a; + char *buf; + size_t len; + size_t cap; + bool oom; +} ts_sb_t; + +static void ts_sb_put(ts_sb_t *sb, const char *s, size_t n) { + if (sb->oom) { + return; + } + if (sb->len + n + SKIP_ONE > sb->cap) { + size_t ncap = sb->cap ? sb->cap * PAIR_LEN : CBM_SZ_256; + while (ncap < sb->len + n + SKIP_ONE) { + ncap *= PAIR_LEN; + } + char *grown = (char *)cbm_arena_alloc(sb->a, ncap); + if (!grown) { + sb->oom = true; + return; + } + if (sb->len) { + memcpy(grown, sb->buf, sb->len); + } + sb->buf = grown; + sb->cap = ncap; + } + memcpy(sb->buf + sb->len, s, n); + sb->len += n; + sb->buf[sb->len] = '\0'; +} + +static void ts_sb_puts(ts_sb_t *sb, const char *s) { + ts_sb_put(sb, s, strlen(s)); +} + +/* A field's text with tabs and newlines as spaces, never empty. */ +static void ts_sb_field(ts_sb_t *sb, const char *s, size_t n) { + ts_sb_put(sb, "\t", SKIP_ONE); + if (n == 0) { + ts_sb_put(sb, "-", SKIP_ONE); + return; + } + for (size_t i = 0; i < n; i++) { + char c = (s[i] == '\t' || s[i] == '\n' || s[i] == '\r') ? ' ' : s[i]; + ts_sb_put(sb, &c, SKIP_ONE); + } +} + +typedef struct { + CBMExtractCtx *ctx; + ts_sb_t *sb; + bool module_syntax; +} ts_walk_t; + +static bool ts_kind(TSNode n, const char *k) { + return !ts_node_is_null(n) && strcmp(ts_node_type(n), k) == 0; +} + +static void ts_node_field(ts_walk_t *w, TSNode n) { + uint32_t b = ts_node_start_byte(n); + uint32_t e = ts_node_end_byte(n); + ts_sb_field(w->sb, w->ctx->source + b, e - b); +} + +/* A string literal's content (quotes dropped). */ +static void ts_string_field(ts_walk_t *w, TSNode s) { + uint32_t b = ts_node_start_byte(s); + uint32_t e = ts_node_end_byte(s); + const char *src = w->ctx->source; + if (e - b >= PAIR_LEN && (src[b] == '"' || src[b] == '\'' || src[b] == '`')) { + b++; + e--; + } + ts_sb_field(w->sb, src + b, e - b); +} + +static TSNode ts_source_of(TSNode stmt) { + TSNode s = ts_node_child_by_field_name(stmt, TS_FIELD("source")); + if (!ts_node_is_null(s)) { + return s; + } + uint32_t nc = ts_node_child_count(stmt); + for (uint32_t i = 0; i < nc; i++) { + TSNode c = ts_node_child(stmt, i); + if (ts_kind(c, "string")) { + return c; + } + } + return s; +} + +static void ts_import_rec(ts_walk_t *w, TSNode local, char kind, TSNode src, TSNode orig) { + ts_sb_puts(w->sb, "\nI"); + ts_node_field(w, local); + char k = kind; + ts_sb_field(w->sb, &k, SKIP_ONE); + ts_string_field(w, src); + if (ts_node_is_null(orig)) { + ts_sb_field(w->sb, "", 0); + } else { + ts_node_field(w, orig); + } +} + +static void ts_walk_import(ts_walk_t *w, TSNode stmt) { + w->module_syntax = true; + TSNode src = ts_source_of(stmt); + uint32_t nc = ts_node_child_count(stmt); + for (uint32_t i = 0; i < nc; i++) { + TSNode c = ts_node_child(stmt, i); + if (ts_kind(c, "import_require_clause")) { + TSNode id = ts_node_named_child(c, 0); + TSNode s = ts_node_child_by_field_name(c, TS_FIELD("source")); + if (ts_kind(id, "identifier") && !ts_node_is_null(s)) { + TSNode none = {0}; + ts_import_rec(w, id, 's', s, none); + } + continue; + } + if (!ts_kind(c, "import_clause") || ts_node_is_null(src)) { + continue; + } + uint32_t cc = ts_node_child_count(c); + for (uint32_t k = 0; k < cc; k++) { + TSNode sub = ts_node_child(c, k); + TSNode none = {0}; + if (ts_kind(sub, "identifier")) { + ts_import_rec(w, sub, 'd', src, none); + } else if (ts_kind(sub, "namespace_import")) { + TSNode id = ts_node_named_child(sub, ts_node_named_child_count(sub) - SKIP_ONE); + if (ts_kind(id, "identifier")) { + ts_import_rec(w, id, 's', src, none); + } + } else if (ts_kind(sub, "named_imports")) { + TSTreeCursor cur = ts_tree_cursor_new(sub); + for (bool more = ts_tree_cursor_goto_first_child(&cur); more; + more = ts_tree_cursor_goto_next_sibling(&cur)) { + TSNode spec = ts_tree_cursor_current_node(&cur); + if (!ts_kind(spec, "import_specifier")) { + continue; + } + TSNode name = ts_node_child_by_field_name(spec, TS_FIELD("name")); + TSNode alias = ts_node_child_by_field_name(spec, TS_FIELD("alias")); + if (!ts_node_is_null(name)) { + ts_import_rec(w, ts_node_is_null(alias) ? name : alias, 'n', src, name); + } + } + ts_tree_cursor_delete(&cur); + } + } + } +} + +/* The name a declaration node declares; null when it has none. */ +static TSNode ts_decl_name(TSNode d) { + return ts_node_child_by_field_name(d, TS_FIELD("name")); +} + +static bool ts_is_decl(TSNode d) { + static const char *const kinds[] = {"class_declaration", + "abstract_class_declaration", + "function_declaration", + "generator_function_declaration", + "function_signature", + "interface_declaration", + "type_alias_declaration", + "enum_declaration", + "internal_module", + "module", + "class", + NULL}; + for (int i = 0; kinds[i]; i++) { + if (ts_kind(d, kinds[i])) { + return true; + } + } + return false; +} + +static void ts_decl_rec(ts_walk_t *w, TSNode name, bool exported, bool dflt) { + ts_sb_puts(w->sb, "\nD"); + ts_node_field(w, name); + char flags[CBM_SZ_4]; + size_t f = 0; + if (exported) { + flags[f++] = 'e'; + } + if (dflt) { + flags[f++] = 'd'; + } + ts_sb_field(w->sb, flags, f); +} + +static void ts_walk_namespaces(ts_walk_t *w, TSNode body, const char *prefix, int depth); + +/* One top-level (or namespace-level, when prefix) declaration. */ +static void ts_walk_decl(ts_walk_t *w, TSNode d, bool exported, bool dflt, const char *prefix, + int depth) { + if (ts_kind(d, "ambient_declaration")) { + uint32_t nc = ts_node_named_child_count(d); + for (uint32_t i = 0; i < nc; i++) { + ts_walk_decl(w, ts_node_named_child(d, i), exported, dflt, prefix, depth); + } + return; + } + if (ts_kind(d, "lexical_declaration") || ts_kind(d, "variable_declaration")) { + if (prefix) { + return; + } + TSTreeCursor cur = ts_tree_cursor_new(d); + for (bool more = ts_tree_cursor_goto_first_child(&cur); more; + more = ts_tree_cursor_goto_next_sibling(&cur)) { + TSNode v = ts_tree_cursor_current_node(&cur); + TSNode name = ts_kind(v, "variable_declarator") ? ts_decl_name(v) : v; + if (ts_kind(name, "identifier")) { + ts_decl_rec(w, name, exported, false); + } + } + ts_tree_cursor_delete(&cur); + return; + } + if (!ts_is_decl(d)) { + return; + } + TSNode name = ts_decl_name(d); + if (ts_node_is_null(name)) { + return; + } + if (!prefix) { + ts_decl_rec(w, name, exported, dflt); + } + if ((ts_kind(d, "internal_module") || ts_kind(d, "module")) && depth < TS_WALK_MAX_DEPTH) { + TSNode body = ts_node_child_by_field_name(d, TS_FIELD("body")); + if (ts_node_is_null(body)) { + return; + } + uint32_t b = ts_node_start_byte(name); + uint32_t e = ts_node_end_byte(name); + size_t pl = prefix ? strlen(prefix) : 0; + char *path = (char *)cbm_arena_alloc(w->sb->a, pl + (e - b) + PAIR_LEN); + if (!path) { + w->sb->oom = true; + return; + } + size_t k = 0; + if (pl) { + memcpy(path, prefix, pl); + path[pl] = '.'; + k = pl + SKIP_ONE; + } + memcpy(path + k, w->ctx->source + b, e - b); + path[k + (e - b)] = '\0'; + ts_sb_puts(w->sb, "\nN"); + ts_sb_field(w->sb, path, strlen(path)); + char num[CBM_SZ_32]; + snprintf(num, sizeof(num), "%u", ts_node_start_point(body).row + SKIP_ONE); + ts_sb_field(w->sb, num, strlen(num)); + snprintf(num, sizeof(num), "%u", ts_node_end_point(body).row + SKIP_ONE); + ts_sb_field(w->sb, num, strlen(num)); + ts_walk_namespaces(w, body, path, depth + SKIP_ONE); + } +} + +/* The namespaces declared in a namespace body, exported or not. */ +static void ts_walk_namespaces(ts_walk_t *w, TSNode body, const char *prefix, int depth) { + TSTreeCursor cur = ts_tree_cursor_new(body); + for (bool more = ts_tree_cursor_goto_first_child(&cur); more; + more = ts_tree_cursor_goto_next_sibling(&cur)) { + TSNode c = ts_tree_cursor_current_node(&cur); + if (!ts_node_is_named(c)) { + continue; + } + if (ts_kind(c, "export_statement")) { + TSNode d = ts_node_child_by_field_name(c, TS_FIELD("declaration")); + if (!ts_node_is_null(d)) { + ts_walk_decl(w, d, true, false, prefix, depth); + } + } else if (ts_kind(c, "expression_statement")) { + uint32_t ec = ts_node_named_child_count(c); + for (uint32_t k = 0; k < ec; k++) { + ts_walk_decl(w, ts_node_named_child(c, k), false, false, prefix, depth); + } + } else { + ts_walk_decl(w, c, false, false, prefix, depth); + } + } + ts_tree_cursor_delete(&cur); +} + +static bool ts_has_default(TSNode stmt) { + uint32_t nc = ts_node_child_count(stmt); + for (uint32_t i = 0; i < nc; i++) { + if (ts_kind(ts_node_child(stmt, i), "default")) { + return true; + } + } + return false; +} + +static void ts_walk_export_clause(ts_walk_t *w, TSNode clause, TSNode src) { + TSTreeCursor cur = ts_tree_cursor_new(clause); + for (bool more = ts_tree_cursor_goto_first_child(&cur); more; + more = ts_tree_cursor_goto_next_sibling(&cur)) { + TSNode spec = ts_tree_cursor_current_node(&cur); + if (!ts_kind(spec, "export_specifier")) { + continue; + } + TSNode name = ts_node_child_by_field_name(spec, TS_FIELD("name")); + TSNode alias = ts_node_child_by_field_name(spec, TS_FIELD("alias")); + if (ts_node_is_null(name)) { + continue; + } + TSNode as = ts_node_is_null(alias) ? name : alias; + if (ts_node_is_null(src)) { + ts_sb_puts(w->sb, "\nL"); + ts_node_field(w, as); + ts_node_field(w, name); + } else { + ts_sb_puts(w->sb, "\nF"); + ts_node_field(w, as); + ts_string_field(w, src); + ts_node_field(w, name); + } + } + ts_tree_cursor_delete(&cur); +} + +static void ts_walk_export(ts_walk_t *w, TSNode stmt) { + w->module_syntax = true; + TSNode src = ts_source_of(stmt); + TSNode decl = ts_node_child_by_field_name(stmt, TS_FIELD("declaration")); + bool dflt = ts_has_default(stmt); + if (!ts_node_is_null(decl)) { + ts_walk_decl(w, decl, true, dflt, NULL, 0); + return; + } + uint32_t nc = ts_node_child_count(stmt); + bool star = false; + for (uint32_t i = 0; i < nc; i++) { + TSNode c = ts_node_child(stmt, i); + if (ts_kind(c, "export_clause")) { + ts_walk_export_clause(w, c, src); + return; + } + if (ts_kind(c, "namespace_export") && !ts_node_is_null(src)) { + TSNode id = ts_node_named_child(c, 0); + if (!ts_node_is_null(id)) { + ts_sb_puts(w->sb, "\nS"); + ts_node_field(w, id); + ts_string_field(w, src); + } + return; + } + star = star || ts_kind(c, "*"); + } + if (star && !ts_node_is_null(src)) { + ts_sb_puts(w->sb, "\nA"); + ts_string_field(w, src); + return; + } + TSNode value = ts_node_child_by_field_name(stmt, TS_FIELD("value")); + if (dflt && ts_kind(value, "identifier")) { + ts_sb_puts(w->sb, "\nL\tdefault"); + ts_node_field(w, value); + } +} + +static bool ts_is_function_like(TSNode n) { + static const char *const kinds[] = {"function_declaration", "generator_function_declaration", + "function_expression", "function", + "generator_function", "arrow_function", + "method_definition", NULL}; + for (int i = 0; kinds[i]; i++) { + if (ts_kind(n, kinds[i])) { + return true; + } + } + return false; +} + +typedef struct { + uint32_t first; /* a function body's lines */ + uint32_t last; + int depth; /* the function's depth in the tree */ +} ts_scope_t; + +static void ts_put_nested(ts_walk_t *w, TSNode decl, const ts_scope_t *in) { + TSNode name = ts_decl_name(decl); + if (!ts_kind(name, "identifier") && !ts_kind(name, "type_identifier")) { + return; + } + char num[CBM_SZ_32]; + ts_sb_puts(w->sb, "\nV"); + ts_node_field(w, name); + snprintf(num, sizeof(num), "%u", ts_node_start_point(decl).row + SKIP_ONE); + ts_sb_field(w->sb, num, strlen(num)); + snprintf(num, sizeof(num), "%u", in->first); + ts_sb_field(w->sb, num, strlen(num)); + snprintf(num, sizeof(num), "%u", in->last); + ts_sb_field(w->sb, num, strlen(num)); +} + +/* V records: every function or class declared inside a function body, with + * its line and the body's lines (the scope it is visible in). The graph gives + * such a declaration a file-level qualified name. One cursor walk over the + * tree; the stack holds the function bodies around the current node. */ +static void ts_walk_nested(ts_walk_t *w, TSNode root) { + int cap = CBM_SZ_64; + int ns = 0; + ts_scope_t *scopes = (ts_scope_t *)cbm_arena_alloc(w->sb->a, (size_t)cap * sizeof(ts_scope_t)); + if (!scopes) { + w->sb->oom = true; + return; + } + TSTreeCursor cur = ts_tree_cursor_new(root); + int depth = 0; + for (bool more = true; more && !w->sb->oom;) { + TSNode n = ts_tree_cursor_current_node(&cur); + while (ns > 0 && scopes[ns - SKIP_ONE].depth >= depth) { + ns--; /* left that function */ + } + if (ts_node_is_named(n)) { + if (ns > 0 && + (ts_kind(n, "function_declaration") || + ts_kind(n, "generator_function_declaration") || ts_kind(n, "class_declaration"))) { + ts_put_nested(w, n, &scopes[ns - SKIP_ONE]); + } + TSNode body = ts_is_function_like(n) ? ts_node_child_by_field_name(n, TS_FIELD("body")) + : (TSNode){0}; + if (!ts_node_is_null(body)) { + if (ns == cap) { + int ncap = cap * PAIR_LEN; + ts_scope_t *g = + (ts_scope_t *)cbm_arena_alloc(w->sb->a, (size_t)ncap * sizeof(ts_scope_t)); + if (!g) { + w->sb->oom = true; + break; + } + memcpy(g, scopes, (size_t)ns * sizeof(ts_scope_t)); + scopes = g; + cap = ncap; + } + scopes[ns++] = (ts_scope_t){ts_node_start_point(body).row + SKIP_ONE, + ts_node_end_point(body).row + SKIP_ONE, depth}; + } + } + if (ts_tree_cursor_goto_first_child(&cur)) { + depth++; + continue; + } + while (!ts_tree_cursor_goto_next_sibling(&cur)) { + if (!ts_tree_cursor_goto_parent(&cur)) { + more = false; + break; + } + depth--; + } + } + ts_tree_cursor_delete(&cur); +} + +const char *cbm_doclink_ts_scan_scope(CBMExtractCtx *ctx) { + if (!ctx || ts_node_is_null(ctx->root)) { + return NULL; + } + ts_sb_t sb = {.a = ctx->arena}; + ts_sb_puts(&sb, CBM_DOCLINK_TS_SCOPE_TAG); + ts_walk_t w = {.ctx = ctx, .sb = &sb}; + TSTreeCursor cur = ts_tree_cursor_new(ctx->root); + for (bool more = ts_tree_cursor_goto_first_child(&cur); more; + more = ts_tree_cursor_goto_next_sibling(&cur)) { + TSNode c = ts_tree_cursor_current_node(&cur); + if (!ts_node_is_named(c)) { + continue; + } + if (ts_kind(c, "import_statement")) { + ts_walk_import(&w, c); + } else if (ts_kind(c, "export_statement")) { + ts_walk_export(&w, c); + } else if (ts_kind(c, "expression_statement")) { + uint32_t ec = ts_node_named_child_count(c); + for (uint32_t k = 0; k < ec; k++) { + ts_walk_decl(&w, ts_node_named_child(c, k), false, false, NULL, 0); + } + } else { + ts_walk_decl(&w, c, false, false, NULL, 0); + } + } + ts_tree_cursor_delete(&cur); + ts_walk_nested(&w, ctx->root); + if (w.module_syntax) { + ts_sb_puts(&sb, "\nM"); + } + return sb.oom ? NULL : sb.buf; +} + +/* The blob without lines: "N path first last" -> "N path", V records dropped. */ +char *cbm_doclink_ts_portable_scope(const char *scope) { + size_t n = strlen(scope); + char *out = (char *)cbm_alloc(CBM_MEM_CLASS_OTHER, n + SKIP_ONE); + if (!out) { + return NULL; + } + size_t w = 0; + for (const char *p = scope; *p;) { + const char *nl = strchr(p, '\n'); + size_t len = nl ? (size_t)(nl - p) : strlen(p); + size_t keep = len; + if (len >= PAIR_LEN && p[0] == 'V' && p[SKIP_ONE] == '\t') { + p = nl ? nl + SKIP_ONE : p + len; + continue; + } + if (len >= PAIR_LEN && p[0] == 'N' && p[SKIP_ONE] == '\t') { + const char *path_end = memchr(p + PAIR_LEN, '\t', len - PAIR_LEN); + keep = path_end ? (size_t)(path_end - p) : len; + } + memcpy(out + w, p, keep); + w += keep; + if (nl) { + out[w++] = '\n'; + } + p = nl ? nl + SKIP_ONE : p + len; + } + out[w] = '\0'; + return out; +} diff --git a/src/pipeline/doc_links.c b/src/pipeline/doc_links.c index c912ac9e0..f01fff7c7 100644 --- a/src/pipeline/doc_links.c +++ b/src/pipeline/doc_links.c @@ -48,6 +48,7 @@ static const cbm_doclink_resolver_t *const DOCLINK_RESOLVERS[] = { &cbm_doclink_cs_resolver, &cbm_doclink_md_resolver, &cbm_doclink_pdf_resolver, &cbm_doclink_rst_resolver, &cbm_doclink_adoc_resolver, &cbm_doclink_go_resolver, &cbm_doclink_jvm_resolver, &cbm_doclink_rs_resolver, &cbm_doclink_perl_resolver, + &cbm_doclink_ts_resolver, }; enum { DOCLINK_RESOLVER_COUNT = sizeof(DOCLINK_RESOLVERS) / sizeof(DOCLINK_RESOLVERS[0]) }; diff --git a/src/pipeline/doc_links.h b/src/pipeline/doc_links.h index e0ca5820f..f21049792 100644 --- a/src/pipeline/doc_links.h +++ b/src/pipeline/doc_links.h @@ -299,6 +299,7 @@ extern const cbm_doclink_resolver_t cbm_doclink_go_resolver; extern const cbm_doclink_resolver_t cbm_doclink_jvm_resolver; extern const cbm_doclink_resolver_t cbm_doclink_rs_resolver; extern const cbm_doclink_resolver_t cbm_doclink_perl_resolver; +extern const cbm_doclink_resolver_t cbm_doclink_ts_resolver; /* True when `rel_path` is a scope input of some language (scope_input). */ bool cbm_doclinks_is_scope_input(const char *rel_path); diff --git a/src/pipeline/doc_links_ts.c b/src/pipeline/doc_links_ts.c new file mode 100644 index 000000000..db007f37e --- /dev/null +++ b/src/pipeline/doc_links_ts.c @@ -0,0 +1,1021 @@ +/* + * doc_links_ts.c — resolving TypeScript and JavaScript doc references + * (doclink_ts.c), one index over TS, TSX and JS files, which import each other. + * + * A reference is a path: segments joined by `.`, `#` (an instance member) or + * `~` (a module's member), with an optional `()` or meaning suffix + * (`:class`, `:0`) that is dropped (one node holds every overload). Its first + * segment is looked up as the TypeScript checker would see it from the + * documented declaration, and the field test's prototype does: + * 1. the declaration's own parameters (dropped at extraction); + * 2. the namespaces around the reference's line, innermost first; + * 3. the file's own top-level declarations; + * 4. the file's imports: named, default and namespace imports, followed + * through `export { a as b }`, `export ... from` and `export *` barrels + * (a cycle guard and a depth bound; two barrels that answer differently + * are ambiguous); a relative specifier resolves with the usual extension + * and index rules, every bare one is external (package.json and + * tsconfig.json are not indexed); + * 5. the standard library's global names: external; + * 6. the top-level declarations of the repository's script files (files + * without an import or an export are global): one binds, several are + * ambiguous; + * 7. the members of the documented declaration's class or interface (of the + * declaration itself for a class or interface doc), `this.` included. + * Nothing found is missing. The other segments walk members (a node's + * qualified name + `.` + segment) or a module's exports. A member a class, + * interface or enum declares without a node of its own is a graph gap. Test + * code binds only from test code. Edges are exact for a name the file + * declares or imports, unique otherwise. + */ +#include "pipeline/doc_links.h" +#include "pipeline/pipeline_internal.h" /* cbm_pipeline_file_node */ +#include "cbm.h" +#include "doclink.h" +#include "helpers.h" /* cbm_fqn_module_source_lang, cbm_is_test_file */ +#include "foundation/arena.h" +#include "foundation/constants.h" +#include "foundation/hash_table.h" +#include "foundation/log.h" +#include "foundation/mem_core.h" +#include "graph_buffer/graph_buffer.h" + +#include +#include +#include +#include +#include +#include +#include + +enum { + TR_MAX_DEPTH = 16, + TR_MAX_SEGS = 16, + TR_QN_MAX = 2048, + TR_PATH_MAX = 1024, + TR_FIELDS = 5, + TR_HTML_MIN = 11, /* HTML + Element */ + TR_SVG_MIN = 10, /* SVG + Element */ +}; + +/* ── The index ───────────────────────────────────────────────────── */ + +typedef struct { + const char *a; /* local name / exported name */ + const char *spec; + const char *b; /* orig / local */ + char kind; /* imports: d n s */ +} tr_rec_t; + +typedef struct { + const char *path; + uint32_t first; + uint32_t last; +} tr_ns_t; + +typedef struct { + const char *name; + uint32_t line; /* the declaration's */ + uint32_t first; /* the function body it is visible in */ + uint32_t last; +} tr_nest_t; + +typedef struct { + const char *path; + const char *module_qn; + bool test; + bool module_syntax; + tr_rec_t *imps; /* I */ + int nimps; + tr_rec_t *stars; /* A: spec */ + int nstars; + tr_rec_t *star_as; /* S: a = name, spec */ + int nstar_as; + tr_rec_t *froms; /* F: a = name, spec, b = orig */ + int nfroms; + tr_rec_t *locals; /* L: a = name, b = local */ + int nlocals; + tr_rec_t *decls; /* D: a = name, kind e/d flags: 'e', 'd', 'b' (both), '-' */ + int ndecls; + tr_ns_t *nss; + int nnss; + tr_nest_t *nested; /* V: only in a fresh blob, the run's own files */ + int nnested; +} tr_file_t; + +typedef struct { + CBMArena arena; + const char *project; + tr_file_t *files; + int nfiles; + int *run_to_file; + int run_count; + CBMHashTable *by_path; /* rel_path -> (file index + 1) as a pointer */ + CBMHashTable *globals; /* name -> tr_global_t */ + bool oom; +} tr_index_t; + +typedef struct { + int file; + int count; /* script files declaring the name */ +} tr_global_t; + +static const char *tr_copy(tr_index_t *ix, const char *s, size_t n) { + char *c = cbm_arena_strndup(&ix->arena, s, n); + ix->oom = ix->oom || !c; + return c; +} + +static bool tr_dash(const char *s, size_t n) { + return n == SKIP_ONE && s[0] == '-'; +} + +/* Append one record to a growing arena array. */ +static tr_rec_t *tr_push(tr_index_t *ix, tr_rec_t **items, int *count) { + if (*count == 0 || (*count >= CBM_SZ_4 && (*count & (*count - SKIP_ONE)) == 0)) { + int ncap = *count ? *count * PAIR_LEN : CBM_SZ_4; /* 4, 8, 16, ... */ + tr_rec_t *g = (tr_rec_t *)cbm_arena_alloc(&ix->arena, (size_t)ncap * sizeof(tr_rec_t)); + if (!g) { + ix->oom = true; + return NULL; + } + if (*count) { + memcpy(g, *items, (size_t)*count * sizeof(tr_rec_t)); + } + *items = g; + } + tr_rec_t *r = &(*items)[(*count)++]; + memset(r, 0, sizeof(*r)); + return r; +} + +static void tr_push_ns(tr_index_t *ix, tr_file_t *f, const char *path, uint32_t first, + uint32_t last) { + if (f->nnss == 0 || (f->nnss >= CBM_SZ_4 && (f->nnss & (f->nnss - SKIP_ONE)) == 0)) { + int ncap = f->nnss ? f->nnss * PAIR_LEN : CBM_SZ_4; + tr_ns_t *g = (tr_ns_t *)cbm_arena_alloc(&ix->arena, (size_t)ncap * sizeof(tr_ns_t)); + if (!g) { + ix->oom = true; + return; + } + if (f->nnss) { + memcpy(g, f->nss, (size_t)f->nnss * sizeof(tr_ns_t)); + } + f->nss = g; + } + f->nss[f->nnss++] = (tr_ns_t){path, first, last}; +} + +static void tr_push_nested(tr_index_t *ix, tr_file_t *f, const char *name, uint32_t line, + uint32_t first, uint32_t last) { + if (f->nnested == 0 || + (f->nnested >= CBM_SZ_4 && (f->nnested & (f->nnested - SKIP_ONE)) == 0)) { + int ncap = f->nnested ? f->nnested * PAIR_LEN : CBM_SZ_4; + tr_nest_t *g = (tr_nest_t *)cbm_arena_alloc(&ix->arena, (size_t)ncap * sizeof(tr_nest_t)); + if (!g) { + ix->oom = true; + return; + } + if (f->nnested) { + memcpy(g, f->nested, (size_t)f->nnested * sizeof(tr_nest_t)); + } + f->nested = g; + } + f->nested[f->nnested++] = (tr_nest_t){name, line, first, last}; +} + +static void tr_read_record(tr_index_t *ix, tr_file_t *f, const char **fl, const size_t *ln, int n) { + char tag = fl[0][0]; + tr_rec_t *r = NULL; + if (tag == 'M') { + f->module_syntax = true; + } else if (tag == 'I' && n >= TR_FIELDS && (r = tr_push(ix, &f->imps, &f->nimps))) { + r->a = tr_copy(ix, fl[1], ln[1]); + r->kind = fl[PAIR_LEN][0]; + r->spec = tr_copy(ix, fl[CBM_SZ_3], ln[CBM_SZ_3]); + r->b = tr_dash(fl[CBM_SZ_4], ln[CBM_SZ_4]) ? NULL : tr_copy(ix, fl[CBM_SZ_4], ln[CBM_SZ_4]); + } else if (tag == 'A' && n >= PAIR_LEN && (r = tr_push(ix, &f->stars, &f->nstars))) { + r->spec = tr_copy(ix, fl[1], ln[1]); + } else if (tag == 'S' && n >= CBM_SZ_3 && (r = tr_push(ix, &f->star_as, &f->nstar_as))) { + r->a = tr_copy(ix, fl[1], ln[1]); + r->spec = tr_copy(ix, fl[PAIR_LEN], ln[PAIR_LEN]); + } else if (tag == 'F' && n >= CBM_SZ_4 && (r = tr_push(ix, &f->froms, &f->nfroms))) { + r->a = tr_copy(ix, fl[1], ln[1]); + r->spec = tr_copy(ix, fl[PAIR_LEN], ln[PAIR_LEN]); + r->b = tr_copy(ix, fl[CBM_SZ_3], ln[CBM_SZ_3]); + } else if (tag == 'L' && n >= CBM_SZ_3 && (r = tr_push(ix, &f->locals, &f->nlocals))) { + r->a = tr_copy(ix, fl[1], ln[1]); + r->b = tr_copy(ix, fl[PAIR_LEN], ln[PAIR_LEN]); + } else if (tag == 'D' && n >= CBM_SZ_3 && (r = tr_push(ix, &f->decls, &f->ndecls))) { + r->a = tr_copy(ix, fl[1], ln[1]); + bool e = memchr(fl[PAIR_LEN], 'e', ln[PAIR_LEN]) != NULL; + bool d = memchr(fl[PAIR_LEN], 'd', ln[PAIR_LEN]) != NULL; + r->kind = (e && d) ? 'b' : e ? 'e' : d ? 'd' : '-'; + } else if (tag == 'V' && n >= TR_FIELDS) { + tr_push_nested(ix, f, tr_copy(ix, fl[1], ln[1]), + (uint32_t)strtoul(fl[PAIR_LEN], NULL, CBM_DECIMAL_BASE), + (uint32_t)strtoul(fl[CBM_SZ_3], NULL, CBM_DECIMAL_BASE), + (uint32_t)strtoul(fl[CBM_SZ_4], NULL, CBM_DECIMAL_BASE)); + } else if (tag == 'N' && n >= PAIR_LEN) { + uint32_t first = + n >= CBM_SZ_4 ? (uint32_t)strtoul(fl[PAIR_LEN], NULL, CBM_DECIMAL_BASE) : 0; + uint32_t last = n >= CBM_SZ_4 ? (uint32_t)strtoul(fl[CBM_SZ_3], NULL, CBM_DECIMAL_BASE) : 0; + tr_push_ns(ix, f, tr_copy(ix, fl[1], ln[1]), first, last); + } +} + +static void tr_read_blob(tr_index_t *ix, tr_file_t *f, const char *blob) { + const char *line = strchr(blob, '\n'); /* the tag line */ + while (line && !ix->oom) { + const char *b = line + SKIP_ONE; + const char *e = strchr(b, '\n'); + if (!e) { + e = b + strlen(b); + } + const char *fl[TR_FIELDS]; + size_t ln[TR_FIELDS]; + int n = 0; + const char *s = b; + for (const char *p = b; p <= e && n < TR_FIELDS; p++) { + if (p == e || *p == '\t') { + fl[n] = s; + ln[n] = (size_t)(p - s); + n++; + s = p + SKIP_ONE; + } + } + if (n > 0 && ln[0] == SKIP_ONE) { + tr_read_record(ix, f, fl, ln, n); + } + line = *e ? e : NULL; + } +} + +static CBMLanguage tr_lang(const char *path) { + const char *dot = strrchr(path, '.'); + if (dot && (strcmp(dot, ".ts") == 0 || strcmp(dot, ".mts") == 0 || strcmp(dot, ".cts") == 0)) { + return CBM_LANG_TYPESCRIPT; + } + return dot && strcmp(dot, ".tsx") == 0 ? CBM_LANG_TSX : CBM_LANG_JAVASCRIPT; +} + +static void tr_destroy(void *index) { + tr_index_t *ix = (tr_index_t *)index; + if (!ix) { + return; + } + cbm_ht_free(ix->by_path); + cbm_ht_free(ix->globals); + cbm_free(CBM_MEM_CLASS_OTHER, ix->run_to_file); + cbm_arena_destroy(&ix->arena); + cbm_free(CBM_MEM_CLASS_OTHER, ix); +} + +static bool tr_add_globals(tr_index_t *ix) { + ix->globals = cbm_ht_create_in(CBM_MEM_CLASS_OTHER, CBM_SZ_256); + if (!ix->globals) { + return false; + } + for (int i = 0; i < ix->nfiles; i++) { + const tr_file_t *f = &ix->files[i]; + if (f->module_syntax) { + continue; /* a module's names are its own */ + } + for (int k = 0; k < f->ndecls; k++) { + tr_global_t *g = (tr_global_t *)cbm_ht_get(ix->globals, f->decls[k].a); + if (g) { + g->count += g->file == i ? 0 : SKIP_ONE; + continue; + } + g = (tr_global_t *)cbm_arena_alloc(&ix->arena, sizeof(tr_global_t)); + if (!g) { + return false; + } + *g = (tr_global_t){i, SKIP_ONE}; + cbm_ht_set(ix->globals, f->decls[k].a, g); + } + } + return true; +} + +static void *tr_build(const cbm_doclink_build_in_t *in) { + tr_index_t *ix = (tr_index_t *)cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*ix)); + if (!ix) { + return NULL; + } + cbm_arena_init(&ix->arena); + ix->project = in->ctx->project_name; + ix->nfiles = in->file_count; + ix->run_count = in->run_file_count; + ix->files = + in->file_count + ? (tr_file_t *)cbm_arena_calloc(&ix->arena, (size_t)in->file_count * sizeof(tr_file_t)) + : NULL; + ix->run_to_file = (int *)cbm_calloc(CBM_MEM_CLASS_OTHER, + (size_t)(in->run_file_count + SKIP_ONE) * sizeof(int)); + ix->by_path = cbm_ht_create_in(CBM_MEM_CLASS_OTHER, (uint32_t)(in->file_count + SKIP_ONE)); + bool ok = (!in->file_count || ix->files) && ix->run_to_file && ix->by_path; + for (int r = 0; ok && r < in->run_file_count; r++) { + ix->run_to_file[r] = CBM_NOT_FOUND; + } + for (int i = 0; ok && i < in->file_count && !ix->oom; i++) { + const cbm_doclink_file_t *src = &in->files[i]; + tr_file_t *f = &ix->files[i]; + f->path = src->rel_path; + f->module_qn = + cbm_fqn_module_source_lang(&ix->arena, ix->project, src->rel_path, CBM_LANG_TYPESCRIPT); + f->test = cbm_is_test_file(src->rel_path, tr_lang(src->rel_path)); + ix->oom = ix->oom || !f->module_qn; + cbm_ht_set(ix->by_path, src->rel_path, (void *)(uintptr_t)(i + SKIP_ONE)); + if (src->run_file >= 0 && src->run_file < in->run_file_count) { + ix->run_to_file[src->run_file] = i; + } + if (src->scope) { + tr_read_blob(ix, f, src->scope); + } + } + ok = ok && !ix->oom && tr_add_globals(ix); + if (!ok) { + cbm_log_error("doc_links.ts.error", "step", "index", "reason", "alloc"); + tr_destroy(ix); + return NULL; + } + char b[PAIR_LEN][CBM_SZ_32]; + snprintf(b[0], sizeof(b[0]), "%d", ix->nfiles); + snprintf(b[SKIP_ONE], sizeof(b[SKIP_ONE]), "%u", cbm_ht_count(ix->globals)); + cbm_log_info("doc_links.ts.index", "files", b[0], "global_names", b[SKIP_ONE]); + return ix; +} + +/* ── Resolution ──────────────────────────────────────────────────── */ + +typedef enum { + TR_NONE = 0, /* nothing found */ + TR_NODE, + TR_MODULE, /* a file's exports (a namespace import) */ + TR_EXTERNAL, + TR_AMBIGUOUS, + TR_GAP, /* declared, without a node */ + TR_MISSING, + TR_NOT_INDEXED, /* a relative import of a file the index does not hold */ +} tr_kind_t; + +typedef struct { + tr_kind_t kind; + const cbm_gbuf_node_t *node; + int file; + bool exact; +} tr_res_t; + +typedef struct { + const tr_index_t *ix; + const cbm_gbuf_t *g; + int src; + cbm_doclink_outcome_t *out; + int visited[TR_MAX_DEPTH * PAIR_LEN]; + const char *visited_name[TR_MAX_DEPTH * PAIR_LEN]; + int nvisited; +} tr_ctx_t; + +static tr_res_t tr_mk(tr_kind_t k) { + return (tr_res_t){k, NULL, CBM_NOT_FOUND, false}; +} + +/* The node at `qn`, when it belongs to file f (any file when f is NULL). */ +static const cbm_gbuf_node_t *tr_node(const tr_ctx_t *c, const char *qn, const tr_file_t *f) { + const cbm_gbuf_node_t *n = cbm_gbuf_find_by_qn(c->g, qn); + if (!n || !n->label) { + return NULL; + } + if (f && (!n->file_path || strcmp(n->file_path, f->path) != 0)) { + return NULL; + } + return n; +} + +static bool tr_join(char *out, size_t cap, const char *a, const char *b, const char *c) { + int n = c ? snprintf(out, cap, "%s.%s.%s", a, b, c) : snprintf(out, cap, "%s.%s", a, b); + return n > 0 && (size_t)n < cap; +} + +static const tr_rec_t *tr_decl(const tr_file_t *f, const char *name) { + for (int i = 0; i < f->ndecls; i++) { + if (strcmp(f->decls[i].a, name) == 0) { + return &f->decls[i]; + } + } + return NULL; +} + +/* A top-level declaration of file f: its node, or a graph gap. */ +static tr_res_t tr_decl_res(const tr_ctx_t *c, const tr_file_t *f, const char *name) { + char qn[TR_QN_MAX]; + if (!tr_join(qn, sizeof(qn), f->module_qn, name, NULL)) { + return tr_mk(TR_GAP); + } + const cbm_gbuf_node_t *n = tr_node(c, qn, f); + tr_res_t r = tr_mk(n ? TR_NODE : TR_GAP); + r.node = n; + return r; +} + +static int tr_file_at(const tr_index_t *ix, const char *path) { + void *v = cbm_ht_get(ix->by_path, path); + return v ? (int)((uintptr_t)v - SKIP_ONE) : CBM_NOT_FOUND; +} + +/* Normalize dir + "/" + rel into out ("." and ".." resolved); false when it + * leaves the repository. */ +static bool tr_norm(const char *dir, size_t dl, const char *rel, char *out, size_t cap) { + char buf[TR_PATH_MAX]; + int n = dl ? snprintf(buf, sizeof(buf), "%.*s/%s", (int)dl, dir, rel) + : snprintf(buf, sizeof(buf), "%s", rel); + if (n <= 0 || (size_t)n >= sizeof(buf)) { + return false; + } + size_t w = 0; + for (char *seg = buf; *seg;) { + char *slash = strchr(seg, '/'); + size_t sl = slash ? (size_t)(slash - seg) : strlen(seg); + if (sl == PAIR_LEN && seg[0] == '.' && seg[SKIP_ONE] == '.') { + if (w == 0) { + return false; + } + while (w > 0 && out[w - SKIP_ONE] != '/') { + w--; + } + w = w > 0 ? w - SKIP_ONE : 0; + } else if (!(sl == SKIP_ONE && seg[0] == '.') && sl > 0) { + if (w > 0) { + out[w++] = '/'; + } + if (w + sl + SKIP_ONE >= cap) { + return false; + } + memcpy(out + w, seg, sl); + w += sl; + } + seg = slash ? slash + SKIP_ONE : seg + sl; + } + out[w] = '\0'; + return w > 0; +} + +/* The file a module specifier written in file `from` names: -1 with + * *external for a bare specifier (a package). */ +static int tr_spec_file(const tr_index_t *ix, int from, const char *spec, bool *external) { + *external = false; + if (!(spec[0] == '.' && + (spec[SKIP_ONE] == '/' || spec[SKIP_ONE] == '\0' || + (spec[SKIP_ONE] == '.' && (spec[PAIR_LEN] == '/' || spec[PAIR_LEN] == '\0'))))) { + *external = true; + return CBM_NOT_FOUND; + } + const char *path = ix->files[from].path; + const char *slash = strrchr(path, '/'); + char base[TR_PATH_MAX]; + if (!tr_norm(path, slash ? (size_t)(slash - path) : 0, spec, base, sizeof(base))) { + return CBM_NOT_FOUND; + } + static const char *const strip[] = {".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", NULL}; + static const char *const exts[] = {".ts", ".tsx", ".d.ts", ".mts", ".cts", + ".js", ".jsx", ".mjs", ".cjs", NULL}; + static const char *const index[] = {"/index.ts", "/index.tsx", "/index.d.ts", "/index.js", + NULL}; + int hit = tr_file_at(ix, base); + if (hit >= 0) { + return hit; + } + char stem[TR_PATH_MAX]; + snprintf(stem, sizeof(stem), "%s", base); + size_t bl = strlen(stem); + for (int i = 0; strip[i]; i++) { + size_t el = strlen(strip[i]); + if (bl > el && strcmp(stem + bl - el, strip[i]) == 0) { + stem[bl - el] = '\0'; + break; + } + } + char cand[TR_PATH_MAX + CBM_SZ_16]; + for (int i = 0; exts[i] && hit < 0; i++) { + snprintf(cand, sizeof(cand), "%s%s", stem, exts[i]); + hit = tr_file_at(ix, cand); + } + for (int i = 0; index[i] && hit < 0; i++) { + snprintf(cand, sizeof(cand), "%s%s", base, index[i]); + hit = tr_file_at(ix, cand); + } + return hit; +} + +static tr_res_t tr_export_of(tr_ctx_t *c, int file, const char *name, int depth); + +/* What a specifier's file exports as `name`. */ +static tr_res_t tr_from(tr_ctx_t *c, int file, const char *spec, const char *name, int depth) { + bool external = false; + int f = tr_spec_file(c->ix, file, spec, &external); + if (f < 0) { + return tr_mk(external ? TR_EXTERNAL : TR_NOT_INDEXED); + } + if (!name) { + tr_res_t r = tr_mk(TR_MODULE); + r.file = f; + return r; + } + return tr_export_of(c, f, name, depth + SKIP_ONE); +} + +/* A name bound at the top level of file `file`: its declaration or import. */ +static tr_res_t tr_top_binding(tr_ctx_t *c, int file, const char *name, int depth) { + const tr_file_t *f = &c->ix->files[file]; + if (tr_decl(f, name)) { + tr_res_t r = tr_decl_res(c, f, name); + r.exact = true; + return r; + } + for (int i = 0; i < f->nimps; i++) { + const tr_rec_t *im = &f->imps[i]; + if (strcmp(im->a, name) != 0) { + continue; + } + tr_res_t r = im->kind == 's' ? tr_from(c, file, im->spec, NULL, depth) + : im->kind == 'd' ? tr_from(c, file, im->spec, "default", depth) + : tr_from(c, file, im->spec, im->b ? im->b : name, depth); + r.exact = r.kind == TR_NODE; + return r; + } + return tr_mk(TR_NONE); +} + +static bool tr_seen(tr_ctx_t *c, int file, const char *name) { + for (int i = 0; i < c->nvisited; i++) { + if (c->visited[i] == file && strcmp(c->visited_name[i], name) == 0) { + return true; + } + } + if (c->nvisited < (int)(sizeof(c->visited) / sizeof(c->visited[0]))) { + c->visited[c->nvisited] = file; + c->visited_name[c->nvisited] = name; + c->nvisited++; + } + return false; +} + +static bool tr_same(const tr_res_t *a, const tr_res_t *b) { + return a->kind == b->kind && a->node == b->node && a->file == b->file; +} + +/* What file `file` exports as `name`; TR_NONE when it exports no such name. */ +static tr_res_t tr_export_of(tr_ctx_t *c, int file, const char *name, int depth) { + if (depth > TR_MAX_DEPTH || tr_seen(c, file, name)) { + return tr_mk(TR_NONE); + } + const tr_file_t *f = &c->ix->files[file]; + bool dflt = strcmp(name, "default") == 0; + for (int i = 0; i < f->ndecls; i++) { + const tr_rec_t *d = &f->decls[i]; + bool hit = dflt ? (d->kind == 'd' || d->kind == 'b') + : (strcmp(d->a, name) == 0 && + (d->kind == 'e' || d->kind == 'b' || !f->module_syntax)); + if (hit) { + return tr_decl_res(c, f, d->a); + } + } + for (int i = 0; i < f->nlocals; i++) { + if (strcmp(f->locals[i].a, name) == 0) { + return tr_top_binding(c, file, f->locals[i].b, depth + SKIP_ONE); + } + } + for (int i = 0; i < f->nfroms; i++) { + if (strcmp(f->froms[i].a, name) == 0) { + return tr_from(c, file, f->froms[i].spec, f->froms[i].b, depth); + } + } + for (int i = 0; i < f->nstar_as; i++) { + if (strcmp(f->star_as[i].a, name) == 0) { + return tr_from(c, file, f->star_as[i].spec, NULL, depth); + } + } + tr_res_t found = tr_mk(TR_NONE); + for (int i = 0; i < f->nstars && !dflt; i++) { + tr_res_t r = tr_from(c, file, f->stars[i].spec, name, depth); + if (r.kind == TR_NONE || r.kind == TR_NOT_INDEXED) { + continue; + } + if (found.kind == TR_NONE) { + found = r; + } else if (!tr_same(&found, &r)) { + return tr_mk(TR_AMBIGUOUS); + } + } + return found; +} + +/* ECMAScript, DOM and Node.js globals: external. Sorted. */ +static const char *const TR_LIB[] = {"AbortController", + "AbortSignal", + "AggregateError", + "Array", + "ArrayBuffer", + "ArrayLike", + "AsyncGenerator", + "AsyncIterable", + "AsyncIterableIterator", + "AsyncIterator", + "Atomics", + "Awaited", + "BigInt", + "BigInt64Array", + "BigUint64Array", + "Blob", + "Boolean", + "Buffer", + "CSSStyleDeclaration", + "Capitalize", + "ConstructorParameters", + "CustomEvent", + "DOMException", + "DataView", + "Date", + "Document", + "DocumentFragment", + "Element", + "Error", + "ErrorEvent", + "Event", + "EventListener", + "EventTarget", + "Exclude", + "Extract", + "File", + "FileList", + "Float32Array", + "Float64Array", + "FormData", + "Function", + "Generator", + "HTMLElement", + "Headers", + "Infinity", + "InstanceType", + "Int16Array", + "Int32Array", + "Int8Array", + "Intl", + "Iterable", + "IterableIterator", + "Iterator", + "JSON", + "KeyboardEvent", + "Location", + "Lowercase", + "Map", + "Math", + "MessageEvent", + "MouseEvent", + "MutationObserver", + "NaN", + "Navigator", + "Node", + "NodeJS", + "NodeList", + "NonNullable", + "Number", + "Object", + "Omit", + "OmitThisParameter", + "Parameters", + "Partial", + "Pick", + "Promise", + "PromiseLike", + "PropertyKey", + "Proxy", + "RangeError", + "Readonly", + "ReadonlyArray", + "ReadonlyMap", + "ReadonlySet", + "Record", + "ReferenceError", + "Reflect", + "RegExp", + "Request", + "Required", + "Response", + "ReturnType", + "Set", + "SharedArrayBuffer", + "String", + "Symbol", + "SyntaxError", + "TemplateStringsArray", + "TextDecoder", + "TextEncoder", + "ThisParameterType", + "ThisType", + "TypeError", + "URIError", + "URL", + "URLSearchParams", + "Uint16Array", + "Uint32Array", + "Uint8Array", + "Uint8ClampedArray", + "Uncapitalize", + "Uppercase", + "WeakMap", + "WeakRef", + "WeakSet", + "WebSocket", + "Window", + "Worker", + "XMLHttpRequest", + "__dirname", + "__filename", + "clearInterval", + "clearTimeout", + "console", + "decodeURIComponent", + "document", + "encodeURIComponent", + "exports", + "fetch", + "globalThis", + "isFinite", + "isNaN", + "module", + "parseFloat", + "parseInt", + "process", + "queueMicrotask", + "require", + "setImmediate", + "setInterval", + "setTimeout", + "structuredClone", + "undefined", + "window"}; + +static int tr_lib_cmp(const void *k, const void *e) { + return strcmp((const char *)k, *(const char *const *)e); +} + +static bool tr_lib(const char *name) { + size_t n = strlen(name); + if ((n > TR_HTML_MIN && strncmp(name, "HTML", CBM_SZ_4) == 0 && + strcmp(name + n - CBM_SZ_7, "Element") == 0) || + (n > TR_SVG_MIN && strncmp(name, "SVG", CBM_SZ_3) == 0 && + strcmp(name + n - CBM_SZ_7, "Element") == 0)) { + return true; + } + return bsearch(name, TR_LIB, sizeof(TR_LIB) / sizeof(TR_LIB[0]), sizeof(TR_LIB[0]), + tr_lib_cmp) != NULL; +} + +static bool tr_label_container(const char *l) { + return l && (strcmp(l, "Class") == 0 || strcmp(l, "Interface") == 0 || strcmp(l, "Enum") == 0); +} + +/* The class, interface or enum whose members a bare name may be: the + * documented declaration itself when it is one, else the one it is a member + * of. NULL: none. */ +static const cbm_gbuf_node_t *tr_container(const tr_ctx_t *c, const char *source_qn) { + const cbm_gbuf_node_t *s = cbm_gbuf_find_by_qn(c->g, source_qn); + if (s && tr_label_container(s->label)) { + return s; + } + const char *dot = strrchr(source_qn, '.'); + if (!dot || (size_t)(dot - source_qn) >= TR_QN_MAX) { + return NULL; + } + char parent[TR_QN_MAX]; + memcpy(parent, source_qn, (size_t)(dot - source_qn)); + parent[dot - source_qn] = '\0'; + const cbm_gbuf_node_t *p = cbm_gbuf_find_by_qn(c->g, parent); + return p && tr_label_container(p->label) ? p : NULL; +} + +static tr_res_t tr_member(const tr_ctx_t *c, const cbm_gbuf_node_t *of, const char *name) { + char qn[TR_QN_MAX]; + if (!tr_join(qn, sizeof(qn), of->qualified_name, name, NULL)) { + return tr_mk(TR_MISSING); + } + const cbm_gbuf_node_t *n = cbm_gbuf_find_by_qn(c->g, qn); + if (n) { + tr_res_t r = tr_mk(TR_NODE); + r.node = n; + r.exact = true; + return r; + } + /* a class, interface or enum member often has no node of its own */ + return tr_mk(tr_label_container(of->label) ? TR_GAP : TR_MISSING); +} + +/* The first segment, seen from the documented declaration. */ +static tr_res_t tr_head(tr_ctx_t *c, const CBMDocLink *link, const char *name) { + const tr_file_t *src = &c->ix->files[c->src]; + if (strcmp(name, "this") == 0) { + const cbm_gbuf_node_t *k = tr_container(c, link->source_qn); + tr_res_t r = tr_mk(k ? TR_NODE : TR_MISSING); + r.node = k; + r.exact = true; + return r; + } + /* the namespaces around the reference, innermost first */ + const tr_ns_t *best[TR_MAX_SEGS]; + int nb = 0; + for (int i = 0; i < src->nnss && nb < TR_MAX_SEGS; i++) { + const tr_ns_t *ns = &src->nss[i]; + if (ns->first <= link->line && link->line <= ns->last) { + best[nb++] = ns; + } + } + for (int k = nb - SKIP_ONE; k >= 0; k--) { /* nested ones come later in the blob */ + char qn[TR_QN_MAX]; + const cbm_gbuf_node_t *n = tr_join(qn, sizeof(qn), src->module_qn, best[k]->path, name) + ? tr_node(c, qn, src) + : NULL; + if (n) { + tr_res_t r = tr_mk(TR_NODE); + r.node = n; + r.exact = true; + return r; + } + } + /* a function or class declared in a function body around the reference, + * innermost first; the graph keeps it under a file-level name, which is + * its own only when the node starts on its line */ + const tr_nest_t *vis = NULL; + for (int i = 0; i < src->nnested; i++) { + const tr_nest_t *v = &src->nested[i]; + if (strcmp(v->name, name) == 0 && v->first <= link->line && link->line <= v->last && + (!vis || v->last - v->first < vis->last - vis->first)) { + vis = v; + } + } + if (vis) { + tr_res_t r = tr_decl_res(c, src, name); + if (r.kind == TR_NODE && r.node->start_line != (int)vis->line) { + r = tr_mk(TR_GAP); /* the node is another declaration's of that name */ + } + r.exact = true; + return r; + } + tr_res_t r = tr_top_binding(c, c->src, name, 0); + if (r.kind != TR_NONE) { + return r; + } + if (tr_lib(name)) { + return tr_mk(TR_EXTERNAL); + } + const tr_global_t *g = (const tr_global_t *)cbm_ht_get(c->ix->globals, name); + if (g) { + if (g->count > SKIP_ONE) { + return tr_mk(TR_AMBIGUOUS); + } + r = tr_decl_res(c, &c->ix->files[g->file], name); + return r; + } + const cbm_gbuf_node_t *k = tr_container(c, link->source_qn); + if (k) { + r = tr_member(c, k, name); + if (r.kind == TR_NODE) { + return r; + } + } + return tr_mk(TR_MISSING); +} + +static bool tr_ident(const char *s, size_t n) { + if (n == 0 || isdigit((unsigned char)s[0])) { + return false; + } + for (size_t i = 0; i < n; i++) { + unsigned char ch = (unsigned char)s[i]; + if (!(isalnum(ch) || ch == '_' || ch == '$' || ch >= 0x80)) { + return false; + } + } + return true; +} + +/* Split a reference into its segments; false when it is no name path. */ +static int tr_segments(char *buf, char **segs) { + size_t n = strlen(buf); + char *paren = strchr(buf, '('); + if (paren) { + if (buf[n - SKIP_ONE] != ')') { + return 0; + } + *paren = '\0'; /* (), (args): one node holds every overload */ + } + char *colon = strchr(buf, ':'); + if (colon) { + *colon = '\0'; /* a meaning: :class, :0 */ + } + int ns = 0; + for (char *p = buf; ns < TR_MAX_SEGS;) { + size_t k = strcspn(p, ".#~"); + if (!tr_ident(p, k)) { + return 0; + } + segs[ns++] = p; + if (p[k] == '\0') { + break; + } + p[k] = '\0'; + p += k + SKIP_ONE; + } + return ns; +} + +static void tr_out(tr_ctx_t *c, tr_res_t r) { + switch (r.kind) { + case TR_NODE: { + const tr_file_t *src = &c->ix->files[c->src]; + const char *tp = r.node->file_path; + int tf = tp ? tr_file_at(c->ix, tp) : CBM_NOT_FOUND; + if (tf >= 0 && c->ix->files[tf].test && !src->test) { + c->out->kind = CBM_DOCLINK_UNRESOLVED; + c->out->reason = CBM_DOCLINK_REASON_TEST_ONLY; + return; + } + c->out->kind = CBM_DOCLINK_EDGE; + c->out->target = r.node; + c->out->exact = r.exact; + return; + } + case TR_EXTERNAL: + c->out->reason = CBM_DOCLINK_REASON_EXTERNAL; + break; + case TR_AMBIGUOUS: + c->out->reason = CBM_DOCLINK_REASON_AMBIGUOUS; + break; + case TR_GAP: + c->out->reason = CBM_DOCLINK_REASON_GRAPH_GAP; + break; + case TR_NOT_INDEXED: + c->out->reason = CBM_DOCLINK_REASON_NOT_INDEXED; + break; + default: + c->out->reason = CBM_DOCLINK_REASON_MISSING; + break; + } + c->out->kind = CBM_DOCLINK_UNRESOLVED; +} + +static void tr_resolve(const void *index, void *state, int run_file, const CBMDocLink *link, + const cbm_gbuf_t *graph, cbm_doclink_outcome_t *out) { + (void)state; /* TS/JS keeps no per-file working state */ + const tr_index_t *ix = (const tr_index_t *)index; + out->kind = CBM_DOCLINK_UNRESOLVED; + out->reason = CBM_DOCLINK_REASON_MISSING; + out->target = NULL; + out->exact = false; + if (!ix || !link->raw || run_file < 0 || run_file >= ix->run_count || + ix->run_to_file[run_file] < 0) { + return; + } + tr_ctx_t c = {.ix = ix, .g = graph, .src = ix->run_to_file[run_file], .out = out}; + char buf[CBM_DOCLINK_TS_REF_MAX + PAIR_LEN]; + size_t rl = strlen(link->raw); + char *segs[TR_MAX_SEGS]; + int ns = 0; + if (rl <= CBM_DOCLINK_TS_REF_MAX && strncmp(link->raw, "module:", CBM_SZ_7) != 0) { + memcpy(buf, link->raw, rl + SKIP_ONE); + ns = tr_segments(buf, segs); + } + if (ns == 0) { + out->reason = CBM_DOCLINK_REASON_UNPARSEABLE; + return; + } + tr_res_t r = tr_head(&c, link, segs[0]); + for (int i = SKIP_ONE; i < ns && (r.kind == TR_NODE || r.kind == TR_MODULE); i++) { + if (r.kind == TR_MODULE) { + c.nvisited = 0; + r = tr_export_of(&c, r.file, segs[i], 0); + r.kind = r.kind == TR_NONE ? TR_MISSING : r.kind; + } else { + r = tr_member(&c, r.node, segs[i]); + } + } + if (r.kind == TR_MODULE) { + /* a module named by itself: its file */ + const cbm_gbuf_node_t *fn = + cbm_pipeline_file_node(graph, ix->project, ix->files[r.file].path); + r = tr_mk(fn ? TR_NODE : TR_GAP); + r.node = fn; + r.exact = true; + } + tr_out(&c, r); +} + +const cbm_doclink_resolver_t cbm_doclink_ts_resolver = { + .langs = {CBM_LANG_TYPESCRIPT, CBM_LANG_TSX, CBM_LANG_JAVASCRIPT}, + .lang_count = CBM_SZ_3, + .scope_tag = CBM_DOCLINK_TS_SCOPE_TAG, + .build = tr_build, + .destroy = tr_destroy, + .resolve = tr_resolve, +}; diff --git a/tests/test_doc_mentions_helpers.h b/tests/test_doc_mentions_helpers.h index f0e2312e1..b6d502ddb 100644 --- a/tests/test_doc_mentions_helpers.h +++ b/tests/test_doc_mentions_helpers.h @@ -440,6 +440,8 @@ static const int DM_HELD_FAMILIES[] = { CBM_DOCLINK_KT_SEE, CBM_DOCLINK_KT_THROWS, CBM_DOCLINK_KT_SAMPLE, CBM_DOCLINK_RS_INTRA_DOC, CBM_DOCLINK_RS_LINK_DEF, CBM_DOCLINK_PERL_POD_L, + CBM_DOCLINK_TS_LINK, CBM_DOCLINK_TS_LINKCODE, + CBM_DOCLINK_TS_SEE, }; static inline void dm_ship_held_families(void) { diff --git a/tests/test_doc_mentions_ts.c b/tests/test_doc_mentions_ts.c new file mode 100644 index 000000000..32581e10d --- /dev/null +++ b/tests/test_doc_mentions_ts.c @@ -0,0 +1,343 @@ +/* + * test_doc_mentions_ts.c -- TypeScript and JavaScript doc-comment references + * (JSDoc and TSDoc {@link}, {@linkcode}, @see). + */ +#include "../src/foundation/compat.h" +#include "test_framework.h" +#include "test_helpers.h" +#include "test_doc_mentions_helpers.h" + +#include "cbm.h" +#include "doclink.h" + +#include +#include +#include + +/* ── extraction ──────────────────────────────────────────────────── */ + +static const char *const TS_WIDGET = + "import { Helper, gadget as g } from \"./helper\";\n" /* 1 */ + "import * as ns from \"./helper\";\n" /* 2 */ + "import Def from \"./def\";\n" /* 3 */ + "\n" /* 4 */ + "/**\n" /* 5 */ + " * The widget. See {@link Helper} and {@link Widget#spin | spin}, also\n" /* 6 */ + " * {@linkcode ns.Helper}. Not `{@link Hidden}` nor {@linkplain Plain}.\n" /* 7 */ + " * ```ts\n" /* 8 */ + " * {@link Fenced}\n" /* 9 */ + " * ```\n" /* 10 */ + " * @see Gadget.\n" /* 11 */ + " * @see {@link Other}\n" /* 12 */ + " * @see https://example.org/w\n" /* 13 */ + " * @see aria-colcount for details\n" /* 14 */ + " */\n" /* 15 */ + "export class Widget {\n" /* 16 */ + " /**\n" /* 17 */ + " * Spins n times, see {@link n} and {@link Widget.size}.\n" /* 18 */ + " */\n" /* 19 */ + " spin(n: number): void {}\n" /* 20 */ + "}\n" /* 21 */ + "\n" /* 22 */ + "export namespace Space {\n" /* 23 */ + " export namespace Inner {\n" /* 24 */ + " export function deep() {}\n" /* 25 */ + " }\n" /* 26 */ + "}\n" /* 27 */ + "function hidden() {}\n" /* 28 */ + "export * from \"./a\";\n" /* 29 */ + "export * as all from \"./b\";\n" /* 30 */ + "export { x as y } from \"./c\";\n" /* 31 */ + "export { hidden as shown };\n" /* 32 */ + "export default Widget;\n"; /* 33 */ + +static bool ts_token_is(const CBMFileResult *r, const char *raw, uint32_t line, int syntax) { + if (dm_count_tokens(r, raw) != 1) { + printf(" token %s: %d found\n", raw, dm_count_tokens(r, raw)); + return false; + } + const CBMDocLink *t = dm_find_token(r, raw); + if (t->line != line || t->syntax != syntax) { + printf(" token %s: line %u syntax %d\n", raw, t->line, t->syntax); + return false; + } + return true; +} + +TEST(doc_mentions_ts_tokens) { + CBMFileResult *r = dm_extract(TS_WIDGET, CBM_LANG_TYPESCRIPT, "src/widget.ts"); + ASSERT_NOT_NULL(r); + ASSERT_TRUE(ts_token_is(r, "Helper", 6, CBM_DOCLINK_TS_LINK)); + ASSERT_TRUE(ts_token_is(r, "Widget#spin", 6, CBM_DOCLINK_TS_LINK)); + ASSERT_TRUE(ts_token_is(r, "ns.Helper", 7, CBM_DOCLINK_TS_LINKCODE)); + ASSERT_TRUE(ts_token_is(r, "Gadget", 11, CBM_DOCLINK_TS_SEE)); + /* an inline tag in an @see line is the tag's, once */ + ASSERT_TRUE(ts_token_is(r, "Other", 12, CBM_DOCLINK_TS_LINK)); + ASSERT_TRUE(ts_token_is(r, "https://example.org/w", 13, CBM_DOCLINK_HREF)); + ASSERT_TRUE(ts_token_is(r, "Widget.size", 18, CBM_DOCLINK_TS_LINK)); + /* a code span, {@linkplain}, a fenced block, prose, the method's own + * parameter: no token */ + ASSERT_EQ(dm_count_tokens(r, "Hidden"), 0); + ASSERT_EQ(dm_count_tokens(r, "Plain"), 0); + ASSERT_EQ(dm_count_tokens(r, "Fenced"), 0); + ASSERT_EQ(dm_count_tokens(r, "aria-colcount"), 0); + ASSERT_EQ(dm_count_tokens(r, "n"), 0); + ASSERT_EQ(r->doc_links.count, 7); + cbm_free_result(r); + PASS(); +} + +static bool ts_has_line(const char *blob, const char *line) { + size_t n = strlen(line); + for (const char *p = blob; p && *p;) { + const char *nl = strchr(p, '\n'); + size_t len = nl ? (size_t)(nl - p) : strlen(p); + if (len == n && memcmp(p, line, n) == 0) { + return true; + } + p = nl ? nl + 1 : NULL; + } + printf(" no line: %s\n", line); + return false; +} + +TEST(doc_mentions_ts_scope_blob) { + CBMFileResult *r = dm_extract(TS_WIDGET, CBM_LANG_TYPESCRIPT, "src/widget.ts"); + ASSERT_NOT_NULL(r); + const char *s = r->doc_scope; + ASSERT_NOT_NULL(s); + ASSERT(strncmp(s, "ts1\n", 4) == 0); + ASSERT(ts_has_line(s, "I\tHelper\tn\t./helper\tHelper")); + ASSERT(ts_has_line(s, "I\tg\tn\t./helper\tgadget")); + ASSERT(ts_has_line(s, "I\tns\ts\t./helper\t-")); + ASSERT(ts_has_line(s, "I\tDef\td\t./def\t-")); + ASSERT(ts_has_line(s, "D\tWidget\te")); + ASSERT(ts_has_line(s, "D\tSpace\te")); + ASSERT(ts_has_line(s, "D\thidden\t-")); + ASSERT(ts_has_line(s, "N\tSpace\t23\t27")); + ASSERT(ts_has_line(s, "N\tSpace.Inner\t24\t26")); + ASSERT(ts_has_line(s, "A\t./a")); + ASSERT(ts_has_line(s, "S\tall\t./b")); + ASSERT(ts_has_line(s, "F\ty\t./c\tx")); + ASSERT(ts_has_line(s, "L\tshown\thidden")); + ASSERT(ts_has_line(s, "L\tdefault\tWidget")); + ASSERT(ts_has_line(s, "M")); + /* the persisted blob keeps no line */ + char *p = cbm_doclink_ts_portable_scope(s); + ASSERT_NOT_NULL(p); + ASSERT(ts_has_line(p, "N\tSpace")); + ASSERT(ts_has_line(p, "N\tSpace.Inner")); + ASSERT(ts_has_line(p, "D\tWidget\te")); + cbm_free(CBM_MEM_CLASS_OTHER, p); + cbm_free_result(r); + PASS(); +} + +/* ── resolution ──────────────────────────────────────────────────── */ + +static void ts_write_repo(const char *repo) { + th_write_file(TH_PATH(repo, "src/helper.ts"), "/** A helper. */\n" + "export class Helper {\n" + " run(): void {}\n" + "}\n" + "export function gadget() {}\n" + "function internal() {}\n"); + th_write_file(TH_PATH(repo, "src/def.ts"), "export default class Def {}\n"); + th_write_file(TH_PATH(repo, "src/barrel/index.ts"), + "export * from \"../helper\";\nexport * from \"./more\";\n"); + th_write_file(TH_PATH(repo, "src/barrel/more.ts"), "export function extra() {}\n"); + th_write_file(TH_PATH(repo, "src/dup1.ts"), "export function twin() {}\n"); + th_write_file(TH_PATH(repo, "src/dup2.ts"), "export function twin() {}\n"); + th_write_file(TH_PATH(repo, "src/twins.ts"), + "export * from \"./dup1\";\nexport * from \"./dup2\";\n"); + /* functions declared in a function body: visible there, under file-level + * names in the graph */ + th_write_file(TH_PATH(repo, "src/closure.ts"), + "export function outer() {\n" + " /** Calls {@link helperInside} and {@link uniqueInside}. */\n" + " function inside() {}\n" + " function helperInside() {}\n" + " function uniqueInside() {}\n" + "}\n" + "export function other() {\n" + " function helperInside() {}\n" + "}\n"); + /* a script: no import, no export, so its names are global */ + th_write_file(TH_PATH(repo, "src/global.js"), "function legacyGlobal() {}\n"); + th_write_file(TH_PATH(repo, "src/widget.ts"), + "import { Helper, gadget as g } from \"./helper\";\n" + "import * as ns from \"./helper\";\n" + "import Def from \"./def\";\n" + "import { extra } from \"./barrel\";\n" + "import { twin } from \"./twins\";\n" + "import { Thing } from \"some-package\";\n" + "import { Gone } from \"./nowhere\";\n" + "\n" + "/**\n" + " * Links: {@link Helper}, {@link Helper.run}, {@link g}, {@link ns.gadget},\n" + " * {@link Def}, {@link extra}, {@link twin}, {@link Thing}, {@link Gone},\n" + " * {@link Promise}, {@link legacyGlobal}, {@link Nope}, {@link spin},\n" + " * {@link Space.Inner.deep}, {@link Helper#missingMember}.\n" + " */\n" + "export class Widget {\n" + " spin(): void {}\n" + " /** Turns, see {@link this.spin}. */\n" + " turn(): void {}\n" + "}\n" + "\n" + "export namespace Space {\n" + " export namespace Inner {\n" + " /** Deep, see {@link deep2}. */\n" + " export function deep() {}\n" + " export function deep2() {}\n" + " }\n" + "}\n"); +} + +static bool ts_edge(const char *db, const char *src, const char *tgt) { + char props[512]; + int n = 0; + dm_edge(db, src, tgt, props, sizeof(props), &n); + if (n != 1) { + printf(" edge %s -> %s: %d found\n", src, tgt, n); + return false; + } + return true; +} + +static bool ts_row(const char *db, const char *raw, const char *reason) { + char got[64]; + dm_row(db, "src/widget.ts", raw, got, sizeof(got), NULL, 0); + if (strcmp(got, reason) != 0) { + printf(" row [%s]: \"%s\", expected \"%s\"\n", raw, got, reason); + return false; + } + return true; +} + +TEST(doc_mentions_ts_resolution) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlts_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[512]; + char db[512]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + snprintf(db, sizeof(db), "%s/g.db", tmp); + ts_write_repo(repo); + ASSERT_EQ(dm_index(repo, db, NULL), 0); + const char *w = "src.widget.Widget"; + /* imports: named, aliased, a member of one, a namespace, a default */ + ASSERT_TRUE(ts_edge(db, w, "src.helper.Helper")); + ASSERT_TRUE(ts_edge(db, w, "src.helper.Helper.run")); + ASSERT_TRUE(ts_edge(db, w, "src.helper.gadget")); + ASSERT_TRUE(ts_edge(db, w, "src.def.Def")); + /* through an export * barrel; two barrels that answer differently */ + ASSERT_TRUE(ts_edge(db, w, "src.barrel.more.extra")); + ASSERT_TRUE(ts_row(db, "twin", "ambiguous")); + /* a package, a relative import the index does not hold, the standard + * library */ + ASSERT_TRUE(ts_row(db, "Thing", "external")); + ASSERT_TRUE(ts_row(db, "Gone", "not_indexed")); + ASSERT_TRUE(ts_row(db, "Promise", "external")); + /* a script's global; nothing anywhere */ + ASSERT_TRUE(ts_edge(db, w, "src.global.legacyGlobal")); + ASSERT_TRUE(ts_row(db, "Nope", "missing")); + /* the class's own member; a path through namespaces */ + ASSERT_TRUE(ts_edge(db, w, "src.widget.Widget.spin")); + ASSERT_TRUE(ts_edge(db, w, "src.widget.Space.Inner.deep")); + /* a class member without a node of its own */ + ASSERT_TRUE(ts_row(db, "Helper#missingMember", "graph_gap")); + /* this. is the container's; a namespace's own names first */ + ASSERT_TRUE(ts_edge(db, "src.widget.Widget.turn", "src.widget.Widget.spin")); + ASSERT_TRUE(ts_edge(db, "src.widget.Space.Inner.deep", "src.widget.Space.Inner.deep2")); + /* a function body's own declaration binds; two of one name share one node */ + ASSERT_TRUE(ts_edge(db, "src.closure.inside", "src.closure.uniqueInside")); + char reason[64]; + dm_row(db, "src/closure.ts", "helperInside", reason, sizeof(reason), NULL, 0); + ASSERT_STR_EQ(reason, "graph_gap"); + th_cleanup(tmp); + PASS(); +} + +/* ── incremental == full; one worker == several ──────────────────── */ + +TEST(doc_mentions_ts_incremental_equals_full) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlts_inc_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[512]; + char inc_db[512]; + char full_db[512]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + snprintf(inc_db, sizeof(inc_db), "%s/inc.db", tmp); + snprintf(full_db, sizeof(full_db), "%s/full.db", tmp); + th_write_file(TH_PATH(repo, "src/lib.ts"), "export function a() {}\nexport function b() {}\n"); + th_write_file(TH_PATH(repo, "src/use.ts"), "import { a } from \"./lib\";\n" + "/** Uses {@link a}. */\n" + "export function use() {}\n"); + ASSERT_EQ(dm_index(repo, inc_db, NULL), 0); + ASSERT_TRUE(ts_edge(inc_db, "src.use.use", "src.lib.a")); + + /* a doc edit: the file is repaired alone */ + th_write_file(TH_PATH(repo, "src/use.ts"), "import { a } from \"./lib\";\n" + "/** Uses {@link a}, and {@link Nope}. */\n" + "export function use() {}\n"); + ASSERT_EQ(dm_step(repo, inc_db, full_db, "doc edit", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0); + + /* a body edit in the target file: its scope is unchanged */ + th_write_file(TH_PATH(repo, "src/lib.ts"), + "export function a() { return 1; }\nexport function b() {}\n"); + ASSERT_EQ(dm_step(repo, inc_db, full_db, "body edit", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0); + ASSERT_TRUE(ts_edge(inc_db, "src.use.use", "src.lib.a")); + + /* an export goes: without a scope_delta hook that is a full run */ + th_write_file(TH_PATH(repo, "src/lib.ts"), "function a() {}\nexport function b() {}\n"); + ASSERT_EQ(dm_step(repo, inc_db, full_db, "export removed", CBM_INCREMENTAL_ROUTE_FORCED_FULL), + 0); + char props[512]; + int n = 0; + dm_edge(inc_db, "src.use.use", "src.lib.a", props, sizeof(props), &n); + ASSERT_EQ(n, 0); + th_cleanup(tmp); + PASS(); +} + +TEST(doc_mentions_ts_workers_agree) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlts_par_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[512]; + char par_db[512]; + char seq_db[512]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + snprintf(par_db, sizeof(par_db), "%s/par.db", tmp); + snprintf(seq_db, sizeof(seq_db), "%s/seq.db", tmp); + enum { MODULES = 60 }; /* past the parallel threshold */ + for (int i = 0; i < MODULES; i++) { + char rel[64]; + char src[1024]; + int next = (i + 1) % MODULES; + snprintf(rel, sizeof(rel), "src/m%02d.ts", i); + snprintf(src, sizeof(src), + "import { F%02d, K%02d } from \"./m%02d\";\n" + "/** F%02d; see {@link F%02d}, {@link K%02d.run}, {@link K%02d#gone},\n" + " * {@link Promise}, {@link Nope} and {@link F%02d}. */\n" + "export function F%02d() {}\n" + "export class K%02d {\n run() {}\n}\n", + next, next, next, i, next, next, next, i, i, i); + th_write_file(TH_PATH(repo, rel), src); + } + ASSERT_EQ(dm_workers_agree(repo, par_db, seq_db), 0); + ASSERT_TRUE(ts_edge(par_db, "src.m00.F00", "src.m01.K01.run")); + th_cleanup(tmp); + PASS(); +} + +SUITE(doc_mentions_ts) { + dm_ship_held_families(); + RUN_TEST(doc_mentions_ts_tokens); + RUN_TEST(doc_mentions_ts_scope_blob); + RUN_TEST(doc_mentions_ts_resolution); + RUN_TEST(doc_mentions_ts_incremental_equals_full); + RUN_TEST(doc_mentions_ts_workers_agree); +} diff --git a/tests/test_main.c b/tests/test_main.c index 8c2c814d1..2db5d8184 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -1829,6 +1829,7 @@ extern void suite_doc_mentions_go(void); extern void suite_doc_mentions_jvm(void); extern void suite_doc_mentions_rs(void); extern void suite_doc_mentions_perl(void); +extern void suite_doc_mentions_ts(void); extern void suite_infrascan(void); extern void suite_cli(void); extern void suite_agent_clients(void); @@ -2304,6 +2305,7 @@ int main(int argc, char **argv) { RUN_SELECTED_SUITE(doc_mentions_jvm); RUN_SELECTED_SUITE(doc_mentions_rs); RUN_SELECTED_SUITE(doc_mentions_perl); + RUN_SELECTED_SUITE(doc_mentions_ts); /* Infrastructure scanning */ RUN_SELECTED_SUITE(infrascan); From 7a321cac16765b8cc497dbb787a2633661f14a66 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 10 Oct 2026 16:48:44 +0200 Subject: [PATCH 07/17] doc-links(php): PHPDoc @see references become MENTIONS edges, held until their audit A PHP doc block names code with `@see Ref` at the start of a tag line (a description may follow) and with inline `{@see Ref}`, both family `see`. doclink_php.c harvests them (code spans and fenced blocks hold none; @link, @uses and the other tags are not harvested; a URL is the core's href family) and writes each file's scope: namespace blocks, use imports, types with their supertypes already qualified in their own file, members, functions. The persisted scope drops lines and imports: other files read names only. Declarations inside a top-level conditional count as top level (`if (!function_exists('f')) { function f() {} }`), never a function body. doc_links_php.c resolves a reference as a FQSEN with PHP's own name rules: - a leading backslash is taken as written; otherwise the first segment is replaced when it is the alias of a class `use` of the namespace block the reference is written in, else the block's namespace is put in front (phpDocumentor's resolver); - beyond that tool, as PHP itself has them: an unqualified function name is taken from a `use function` import, else from the current namespace, else from the global namespace; and self/static/parent name the documented type and its superclass; - a bare `name()` that would bind a function but is written in a type that declares or inherits a method of that name is ambiguous (the method or the function; the reference does not say which); - members are looked up on the type, its traits, its superclass and its interfaces (depth first); type, function and method names compare without case, properties and constants with it; - properties, class constants and enum cases have no graph node: graph_gap; so has a type or function the graph folds into another one of the same name in its file (two namespace blocks), which the scope flags; - a name the repository does not declare is external outside the repository's namespace roots (built-ins, Composer dependencies), else missing. Residual: a function a dependency declares in the current namespace is invisible, so the fallback would bind a global function of that name. There is no scope_delta hook yet: a declaration change is a full run. The family is held (ships=false) until its held-out audit passes; the suite ships it through the test seam. Proof (ship build, Apple M5 Pro / 48 GB, base 196ca71cda60): - koel 04ed8beaec34 vs the field test's prototype: 4 of its 5 pairs bound alike; the fifth sits in a class constant's doc block, which has no node. - cakephp a75828db: 342 edge mentions + 39 unresolved rows = every one of the 381 non-URL references in function and type doc blocks; 36 more sit in property and constant doc blocks (no node, not harvested). - symfony 9917e115: 452 edge mentions (434 edges) + 205 rows = all 657; 24 in property, constant and other doc blocks. - edges only through the global fallback: 1 (cakephp `@see debug()` in Cake\Core, the global debug() helper: correct); one more on symfony was wrong (`@see run()` meaning the class's own method bound a global script function) and the method-name rule above makes it ambiguous. Through self/static/parent: 20, all correct (12 + 5 self, 3 parent). - a random sample of 24 edges (12 per corpus) read against their source lines: all correct. Tests (doc_mentions_php): tokens with their lines; the scope blob, its persisted form, namespace blocks and conditional declarations; every resolution rule; the global fallback and self/static/parent each in a test of its own; incremental equals full (a doc edit and a body edit repair one file, a removed function is a full run); 60 modules give the same edges and rows with one worker and several. Each rule red with it reverted (global fallback, self/static, parent, trait members, the folded-name gap, use aliases, method names without case, the bare-name method lookup, the namespace roots, code spans, the method-name guard, conditional declarations). Signed-off-by: Martin Vogel --- Makefile.cbm | 4 +- internal/cbm/doclink.c | 5 + internal/cbm/doclink.h | 36 ++ internal/cbm/doclink_php.c | 950 +++++++++++++++++++++++++++++ src/pipeline/doc_links.c | 2 +- src/pipeline/doc_links.h | 1 + src/pipeline/doc_links_php.c | 975 ++++++++++++++++++++++++++++++ tests/test_doc_mentions_helpers.h | 2 +- tests/test_doc_mentions_php.c | 464 ++++++++++++++ tests/test_main.c | 2 + 10 files changed, 2438 insertions(+), 3 deletions(-) create mode 100644 internal/cbm/doclink_php.c create mode 100644 src/pipeline/doc_links_php.c create mode 100644 tests/test_doc_mentions_php.c diff --git a/Makefile.cbm b/Makefile.cbm index b6341756c..4cfb47783 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -391,6 +391,7 @@ EXTRACTION_SRCS = \ $(CBM_DIR)/doclink_rs.c \ $(CBM_DIR)/doclink_perl.c \ $(CBM_DIR)/doclink_ts.c \ + $(CBM_DIR)/doclink_php.c \ $(CBM_DIR)/sql_values.c \ $(CBM_DIR)/helpers.c \ $(CBM_DIR)/cpp_branch_views.c \ @@ -522,6 +523,7 @@ PIPELINE_SRCS = \ src/pipeline/doc_links_rs.c \ src/pipeline/doc_links_perl.c \ src/pipeline/doc_links_ts.c \ + src/pipeline/doc_links_php.c \ src/pipeline/doc_links_msbuild.c \ src/pipeline/pass_route_nodes.c \ src/pipeline/pass_enrichment.c \ @@ -784,7 +786,7 @@ TEST_DISCOVER_SRCS = \ TEST_GRAPH_BUFFER_SRCS = tests/test_graph_buffer.c -TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_pipeline_conventions.c tests/test_pipeline_frozen.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_doclinks.c tests/test_doc_mentions.c tests/test_doc_links_md.c tests/test_doc_links_pdf.c tests/test_doc_links_rst.c tests/test_doc_links_adoc.c tests/test_doc_mentions_go.c tests/test_doc_mentions_jvm.c tests/test_doc_mentions_rs.c tests/test_doc_mentions_perl.c tests/test_doc_mentions_ts.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c +TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_pipeline_conventions.c tests/test_pipeline_frozen.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_doclinks.c tests/test_doc_mentions.c tests/test_doc_links_md.c tests/test_doc_links_pdf.c tests/test_doc_links_rst.c tests/test_doc_links_adoc.c tests/test_doc_mentions_go.c tests/test_doc_mentions_jvm.c tests/test_doc_mentions_rs.c tests/test_doc_mentions_perl.c tests/test_doc_mentions_ts.c tests/test_doc_mentions_php.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c TEST_WATCHER_SRCS = tests/test_watcher.c diff --git a/internal/cbm/doclink.c b/internal/cbm/doclink.c index 7231a2308..cb65126b1 100644 --- a/internal/cbm/doclink.c +++ b/internal/cbm/doclink.c @@ -234,6 +234,11 @@ static const doclink_lang_t DOCLINK_LANGS[] = { .scan_scope = cbm_doclink_ts_scan_scope, .scope_tag = CBM_DOCLINK_TS_SCOPE_TAG, .portable_scope = cbm_doclink_ts_portable_scope}, + {.lang = CBM_LANG_PHP, + .parse_doc = cbm_doclink_php_parse_doc, + .scan_scope = cbm_doclink_php_scan_scope, + .scope_tag = CBM_DOCLINK_PHP_SCOPE_TAG, + .portable_scope = cbm_doclink_php_portable_scope}, }; static const doclink_lang_t *doclink_lang(CBMLanguage lang) { diff --git a/internal/cbm/doclink.h b/internal/cbm/doclink.h index c1d868da5..8e3313009 100644 --- a/internal/cbm/doclink.h +++ b/internal/cbm/doclink.h @@ -104,6 +104,8 @@ X(CBM_DOCLINK_TS_LINK, CBM_LANG_TYPESCRIPT, "link", false, false, "MENTIONS") \ X(CBM_DOCLINK_TS_LINKCODE, CBM_LANG_TYPESCRIPT, "linkcode", false, false, "MENTIONS") \ X(CBM_DOCLINK_TS_SEE, CBM_LANG_TYPESCRIPT, "see", false, false, "MENTIONS") \ + /* php (doclink_php.c): PHPDoc @see and {@see}; held until their audit passes */ \ + X(CBM_DOCLINK_PHP_SEE, CBM_LANG_PHP, "see", false, false, "MENTIONS") \ /* markdown (doclink_md.c): what a document's text names explicitly */ \ X(CBM_DOCLINK_MD_LINK, CBM_LANG_MARKDOWN, "link", false, true, "MENTIONS") \ X(CBM_DOCLINK_MD_PATH, CBM_LANG_MARKDOWN, "path", false, true, "MENTIONS") \ @@ -555,6 +557,40 @@ char *cbm_doclink_ts_portable_scope(const char *scope); * this length and the resolver reports it as unparseable. */ enum { CBM_DOCLINK_TS_REF_MAX = 512 }; +/* ── PHP (doclink_php.c) ───────────────────────────────────────────── */ + +void cbm_doclink_php_parse_doc(CBMExtractCtx *ctx, const CBMDefinition *def, const char *doc, + uint32_t doc_line); +const char *cbm_doclink_php_scan_scope(CBMExtractCtx *ctx); +char *cbm_doclink_php_portable_scope(const char *scope); + +#define CBM_DOCLINK_PHP_SCOPE_TAG "php1" + +/* The longest reference a PHP token keeps; a longer one is cut just past this + * length and the resolver reports it as unparseable. */ +enum { CBM_DOCLINK_PHP_REF_MAX = 512 }; + +/* One `use` import of a PHP namespace block: kind 'c' (a class or namespace), + * 'f' (a function), 'k' (a constant); the name without a leading backslash. */ +typedef struct { + const char *alias; + size_t alias_len; + const char *name; + size_t name_len; + char kind; +} cbm_doclink_php_use_t; + +/* The fully qualified name (no leading backslash) of a type name `ref` (n + * bytes) written in namespace `ns` (ns_len 0: the global one) under these + * imports, as PHP resolves it: a leading backslash is taken as written, + * `namespace\` stands for the current namespace, else the first segment is + * replaced when it is the alias of a class import (compared without case), + * else the namespace is put in front. Writes NUL-terminated into out (cap + * bytes); returns the length, 0 when it does not fit. */ +size_t cbm_doclink_php_qualify(char *out, size_t cap, const char *ns, size_t ns_len, + const cbm_doclink_php_use_t *uses, int nuses, const char *ref, + size_t n); + /* ── Perl POD (doclink_perl.c) ─────────────────────────────────────── */ void cbm_doclink_perl_parse_doc(CBMExtractCtx *ctx, const CBMDefinition *def, const char *doc, diff --git a/internal/cbm/doclink_php.c b/internal/cbm/doclink_php.c new file mode 100644 index 000000000..8abb8a437 --- /dev/null +++ b/internal/cbm/doclink_php.c @@ -0,0 +1,950 @@ +/* + * doclink_php.c — PHP doc-comment references (PHPDoc) and the PHP doc-link + * scope. + * + * Tokens. Only a block comment that opens with a slash and two stars is a + * doc. Its lines lose their leading white space, one `*` and one space. A + * reference is + * - `@see Ref` at the start of a tag line, a description may follow; + * - `{@see Ref}` or `{@see Ref description}` anywhere on a line; + * both family `see`. The reference is the text up to white space (or the + * closing brace), sentence punctuation at its end dropped. A URL is the core's + * href family. Text in a code span (backticks on one line) or in a fenced + * code block holds no reference. @link, @uses and the other tags are not + * harvested. + * + * Scope blob, one record per line after the tag line "php1" (tab-separated): + * N blk name first last a namespace block, numbered from 0 in file order, + * its name ("-": the global namespace) and lines; a + * file without a namespace declaration has none, and + * its one block is number 0 + * U blk kind alias name a use import of block blk: kind c (a class or a + * namespace), f (a function), k (a constant); the + * alias as written, the name without its leading + * backslash + * T kind fqn flags first last a class (c), interface (i), trait (t) or enum + * (e) declared at the top level of a block, its fully + * qualified name and lines; flags `d` when the file + * declares another type or function of the same name + * (the graph keeps one node for both), else `-` + * B rel fqn a supertype of the T record before it: rel e + * (extends), i (implements), t (a used trait), the + * name qualified as PHP resolves it in this file + * K kind name a member of that type: m method, p property (no + * `$`; promoted constructor parameters too), c + * constant, e enum case + * F fqn flags a function declared at the top level of a block; + * flags as for T + * "At the top level" includes the branches of a top-level conditional + * (`if (!function_exists('f')) { function f() {} }`), never a function body. + * The persisted blob (cbm_doclink_php_portable_scope) drops the lines of N + * and T and the U records: other files read names only, and every name a + * supertype record holds is already qualified. + */ +#include "doclink.h" + +#include "arena.h" +#include "helpers.h" +#include "foundation/constants.h" +#include "foundation/hash_table.h" +#include "foundation/mem_core.h" +#include "tree_sitter/api.h" + +#include +#include +#include +#include +#include + +enum { + PHP_PREFIX_LEN = 3, + PHP_FENCE_LEN = 3, + PHP_SEE_LEN = 4, + PHP_INLINE_LEN = 5, + PHP_COND_DEPTH = 8, /* nested conditionals followed */ +}; + +/* ── Names ───────────────────────────────────────────────────────── */ + +static bool php_eq_nocase(const char *a, size_t an, const char *b, size_t bn) { + if (an != bn) { + return false; + } + for (size_t i = 0; i < an; i++) { + if (tolower((unsigned char)a[i]) != tolower((unsigned char)b[i])) { + return false; + } + } + return true; +} + +/* out[*w..] += s[0, n); false when it does not fit. */ +static bool php_put(char *out, size_t cap, size_t *w, const char *s, size_t n) { + if (*w + n + SKIP_ONE > cap) { + return false; + } + memcpy(out + *w, s, n); + *w += n; + out[*w] = '\0'; + return true; +} + +size_t cbm_doclink_php_qualify(char *out, size_t cap, const char *ns, size_t ns_len, + const cbm_doclink_php_use_t *uses, int nuses, const char *ref, + size_t n) { + size_t w = 0; + if (cap == 0 || n == 0) { + return 0; + } + out[0] = '\0'; + if (ref[0] == '\\') { + return php_put(out, cap, &w, ref + SKIP_ONE, n - SKIP_ONE) ? w : 0; + } + static const char rel[] = "namespace\\"; + size_t rl = sizeof(rel) - SKIP_ONE; + if (n > rl && php_eq_nocase(ref, rl, rel, rl)) { + ref += rl; /* namespace\Foo: the current namespace's Foo */ + n -= rl; + } else { + const char *bs = memchr(ref, '\\', n); + size_t first = bs ? (size_t)(bs - ref) : n; + for (int i = 0; i < nuses; i++) { + const cbm_doclink_php_use_t *u = &uses[i]; + if (u->kind == 'c' && php_eq_nocase(u->alias, u->alias_len, ref, first)) { + bool ok = php_put(out, cap, &w, u->name, u->name_len) && + php_put(out, cap, &w, ref + first, n - first); + return ok ? w : 0; + } + } + } + bool ok = + !ns_len || (php_put(out, cap, &w, ns, ns_len) && php_put(out, cap, &w, "\\", SKIP_ONE)); + return ok && php_put(out, cap, &w, ref, n) ? w : 0; +} + +/* ── Tokens ──────────────────────────────────────────────────────── */ + +typedef struct { + CBMExtractCtx *ctx; + const CBMDefinition *def; +} php_emit_t; + +static bool php_is_url(const char *t, size_t n) { + size_t i = 0; + while (i < n && (isalnum((unsigned char)t[i]) || t[i] == '+' || t[i] == '.' || t[i] == '-')) { + i++; + } + if (i > 0 && i + CBM_SZ_3 <= n && memcmp(t + i, "://", CBM_SZ_3) == 0) { + return true; + } + return (n > CBM_SZ_4 && strncmp(t, "www.", CBM_SZ_4) == 0) || + (n > CBM_SZ_7 && strncmp(t, "mailto:", CBM_SZ_7) == 0); +} + +static void php_push(const php_emit_t *em, const char *ref, size_t n, uint32_t line) { + while (n > 0 && (ref[n - SKIP_ONE] == '.' || ref[n - SKIP_ONE] == ',' || + ref[n - SKIP_ONE] == ';' || ref[n - SKIP_ONE] == ':')) { + n--; /* sentence punctuation */ + } + if (n == 0) { + return; + } + if (n > CBM_DOCLINK_PHP_REF_MAX) { + n = CBM_DOCLINK_PHP_REF_MAX + SKIP_ONE; /* clipped: the resolver calls it unparseable */ + } + int syntax = php_is_url(ref, n) ? CBM_DOCLINK_HREF : CBM_DOCLINK_PHP_SEE; + char *copy = cbm_arena_strndup(em->ctx->arena, ref, n); + if (!copy) { + return; + } + CBMDocLink link = { + .source_qn = em->def->qualified_name, + .raw = copy, + .line = line, + .def_line = em->def->start_line, + .syntax = (uint16_t)syntax, + }; + cbm_doclinks_push(&em->ctx->result->doc_links, em->ctx->arena, link); +} + +/* One doc line: its text after the leading `*`, and its source line. */ +typedef struct { + const char *p; + size_t n; + uint32_t line; +} php_line_t; + +/* The doc's lines, its opening and closing marks cut off. */ +static int php_doc_lines(CBMArena *a, const char *doc, uint32_t doc_line, php_line_t **out) { + size_t dl = strlen(doc); + if (dl < PHP_PREFIX_LEN || strncmp(doc, "/**", PHP_PREFIX_LEN) != 0) { + return 0; + } + const char *b = doc + PHP_PREFIX_LEN; + const char *e = doc + dl; + if (e - b >= PAIR_LEN && e[-SKIP_ONE] == '/' && e[-PAIR_LEN] == '*') { + e -= PAIR_LEN; + } + int cap = SKIP_ONE; + for (const char *p = b; p < e; p++) { + cap += *p == '\n'; + } + php_line_t *lines = (php_line_t *)cbm_arena_alloc(a, (size_t)cap * sizeof(php_line_t)); + if (!lines) { + return 0; + } + int n = 0; + uint32_t line = doc_line; + for (const char *p = b; p <= e && n < cap;) { + const char *le = p; + while (le < e && *le != '\n') { + le++; + } + const char *s = p; + while (s < le && (*s == ' ' || *s == '\t')) { + s++; + } + if (s < le && *s == '*') { + s++; + if (s < le && *s == ' ') { + s++; + } + } + const char *t = le; + while (t > s && (t[-SKIP_ONE] == ' ' || t[-SKIP_ONE] == '\t' || t[-SKIP_ONE] == '\r')) { + t--; + } + lines[n++] = (php_line_t){s, (size_t)(t - s), line}; + line++; + if (le >= e) { + break; + } + p = le + SKIP_ONE; + } + *out = lines; + return n; +} + +/* Mark the bytes of a line that sit in a code span (backticks paired on the + * line). */ +static void php_mask_spans(const php_line_t *ln, bool *mask) { + size_t i = 0; + while (i < ln->n) { + if (ln->p[i] != '`') { + i++; + continue; + } + size_t run = 0; + while (i + run < ln->n && ln->p[i + run] == '`') { + run++; + } + size_t j = i + run; + size_t close = SIZE_MAX; + while (j < ln->n) { + size_t r = 0; + while (j + r < ln->n && ln->p[j + r] == '`') { + r++; + } + if (r == run) { + close = j; + break; + } + j += r ? r : SKIP_ONE; + } + if (close == SIZE_MAX) { + i += run; + continue; + } + for (size_t k = i; k < close + run; k++) { + mask[k] = true; + } + i = close + run; + } +} + +/* The first position >= j of p[0, n) that is no white space. */ +static size_t php_skip_space(const char *p, size_t j, size_t n) { + while (j < n && isspace((unsigned char)p[j])) { + j++; + } + return j; +} + +/* `{@see Ref ...}` anywhere on the line, outside code spans. */ +static void php_line_inline(const php_emit_t *em, const php_line_t *ln, const bool *mask) { + for (size_t i = 0; i + PHP_INLINE_LEN <= ln->n; i++) { + if (mask[i] || memcmp(ln->p + i, "{@see", PHP_INLINE_LEN) != 0) { + continue; + } + size_t j = i + PHP_INLINE_LEN; + if (j < ln->n && ln->p[j] != '}' && !isspace((unsigned char)ln->p[j])) { + continue; /* {@seealso} or another word */ + } + j = php_skip_space(ln->p, j, ln->n); + size_t k = j; + while (k < ln->n && ln->p[k] != '}' && !isspace((unsigned char)ln->p[k])) { + k++; + } + php_push(em, ln->p + j, k - j, ln->line); + i = k; + } +} + +/* `@see Ref` at the start of a tag line. */ +static void php_line_see(const php_emit_t *em, const php_line_t *ln) { + if (ln->n < PHP_SEE_LEN || memcmp(ln->p, "@see", PHP_SEE_LEN) != 0 || + (ln->n > PHP_SEE_LEN && !isspace((unsigned char)ln->p[PHP_SEE_LEN]))) { + return; + } + size_t j = php_skip_space(ln->p, PHP_SEE_LEN, ln->n); + if (j < ln->n && ln->p[j] == '{') { + return; /* @see {@see Ref}: the inline tag is the reference */ + } + size_t k = j; + while (k < ln->n && !isspace((unsigned char)ln->p[k])) { + k++; + } + php_push(em, ln->p + j, k - j, ln->line); +} + +void cbm_doclink_php_parse_doc(CBMExtractCtx *ctx, const CBMDefinition *def, const char *doc, + uint32_t doc_line) { + if (!ctx || !ctx->result || !def || !doc || !def->qualified_name) { + return; + } + CBMArena *a = ctx->scratch ? ctx->scratch : ctx->arena; + php_line_t *lines = NULL; + int n = php_doc_lines(a, doc, doc_line, &lines); + php_emit_t em = {ctx, def}; + bool fence = false; + for (int i = 0; i < n; i++) { + const php_line_t *ln = &lines[i]; + if (ln->n >= PHP_FENCE_LEN && memcmp(ln->p, "```", PHP_FENCE_LEN) == 0) { + fence = !fence; + continue; + } + if (fence || ln->n == 0) { + continue; + } + bool *mask = (bool *)cbm_arena_calloc(a, ln->n + SKIP_ONE); + if (!mask) { + return; + } + php_mask_spans(ln, mask); + php_line_see(&em, ln); + php_line_inline(&em, ln, mask); + } +} + +/* ── Scope blob ──────────────────────────────────────────────────── */ + +typedef struct { + CBMArena *a; + char *buf; + size_t len; + size_t cap; + bool oom; +} php_sb_t; + +static void php_sb_put(php_sb_t *sb, const char *s, size_t n) { + if (sb->oom) { + return; + } + if (sb->len + n + SKIP_ONE > sb->cap) { + size_t ncap = sb->cap ? sb->cap * PAIR_LEN : CBM_SZ_256; + while (ncap < sb->len + n + SKIP_ONE) { + ncap *= PAIR_LEN; + } + char *grown = (char *)cbm_arena_alloc(sb->a, ncap); + if (!grown) { + sb->oom = true; + return; + } + if (sb->len) { + memcpy(grown, sb->buf, sb->len); + } + sb->buf = grown; + sb->cap = ncap; + } + memcpy(sb->buf + sb->len, s, n); + sb->len += n; + sb->buf[sb->len] = '\0'; +} + +static void php_sb_puts(php_sb_t *sb, const char *s) { + php_sb_put(sb, s, strlen(s)); +} + +/* A field: a tab, then the text (never empty: "-" stands for none). */ +static void php_sb_field(php_sb_t *sb, const char *s, size_t n) { + php_sb_put(sb, "\t", SKIP_ONE); + php_sb_put(sb, n ? s : "-", n ? n : SKIP_ONE); +} + +static void php_sb_num(php_sb_t *sb, uint32_t v) { + char num[CBM_SZ_32]; + int n = snprintf(num, sizeof(num), "%u", v); + php_sb_field(sb, num, n > 0 ? (size_t)n : 0); +} + +typedef struct { + CBMExtractCtx *ctx; + php_sb_t *sb; + CBMHashTable *names; /* short name of a top-level type or function -> count */ + int blk; + const char *ns; /* the current block's namespace ("" for the global one) */ + size_t ns_len; + cbm_doclink_php_use_t *uses; /* the current block's */ + int nuses; + int cap_uses; +} php_walk_t; + +static bool php_kind(TSNode n, const char *k) { + return !ts_node_is_null(n) && strcmp(ts_node_type(n), k) == 0; +} + +/* A name node's text without white space (`A \ B` is `A\B`), in the arena. */ +static const char *php_name_text(php_walk_t *w, TSNode n, size_t *len) { + uint32_t b = ts_node_start_byte(n); + uint32_t e = ts_node_end_byte(n); + char *out = (char *)cbm_arena_alloc(w->sb->a, (size_t)(e - b) + SKIP_ONE); + if (!out) { + w->sb->oom = true; + *len = 0; + return ""; + } + size_t k = 0; + for (uint32_t i = b; i < e; i++) { + char c = w->ctx->source[i]; + if (!isspace((unsigned char)c)) { + out[k++] = c; + } + } + out[k] = '\0'; + *len = k; + return out; +} + +/* All named children of a node, read with one cursor walk (indexing them is + * quadratic). */ +static TSNode *php_children(php_walk_t *w, TSNode parent, int *count) { + *count = 0; + uint32_t nc = ts_node_named_child_count(parent); + if (nc == 0) { + return NULL; + } + TSNode *kids = (TSNode *)cbm_arena_alloc(w->sb->a, (size_t)nc * sizeof(TSNode)); + if (!kids) { + w->sb->oom = true; + return NULL; + } + TSTreeCursor cur = ts_tree_cursor_new(parent); + for (bool more = ts_tree_cursor_goto_first_child(&cur); more && *count < (int)nc; + more = ts_tree_cursor_goto_next_sibling(&cur)) { + TSNode c = ts_tree_cursor_current_node(&cur); + if (ts_node_is_named(c)) { + kids[(*count)++] = c; + } + } + ts_tree_cursor_delete(&cur); + return kids; +} + +/* `function` or `const` among a node's anonymous children: 'f', 'k', else + * `dflt`. */ +static char php_use_kind(TSNode n, char dflt) { + uint32_t nc = ts_node_child_count(n); + for (uint32_t i = 0; i < nc; i++) { + TSNode c = ts_node_child(n, i); + if (ts_node_is_named(c)) { + continue; + } + if (php_kind(c, "function")) { + return 'f'; + } + if (php_kind(c, "const")) { + return 'k'; + } + } + return dflt; +} + +static void php_add_use(php_walk_t *w, char kind, const char *alias, size_t alias_len, + const char *name, size_t name_len) { + if (name_len && name[0] == '\\') { + name++; + name_len--; + } + if (!alias_len || !name_len) { + return; + } + if (w->nuses == w->cap_uses) { + int ncap = w->cap_uses ? w->cap_uses * PAIR_LEN : CBM_SZ_8; + cbm_doclink_php_use_t *g = (cbm_doclink_php_use_t *)cbm_arena_alloc( + w->sb->a, (size_t)ncap * sizeof(cbm_doclink_php_use_t)); + if (!g) { + w->sb->oom = true; + return; + } + if (w->nuses) { + memcpy(g, w->uses, (size_t)w->nuses * sizeof(cbm_doclink_php_use_t)); + } + w->uses = g; + w->cap_uses = ncap; + } + w->uses[w->nuses++] = (cbm_doclink_php_use_t){alias, alias_len, name, name_len, kind}; + php_sb_puts(w->sb, "\nU"); + php_sb_num(w->sb, (uint32_t)w->blk); + php_sb_field(w->sb, &kind, SKIP_ONE); + php_sb_field(w->sb, alias, alias_len); + php_sb_field(w->sb, name, name_len); +} + +/* One clause of a use declaration; `prefix` is a group's common prefix. */ +static void php_use_clause(php_walk_t *w, TSNode clause, char kind, const char *prefix, + size_t prefix_len) { + TSNode alias = ts_node_child_by_field_name(clause, TS_FIELD("alias")); + TSNode name = ts_node_named_child(clause, 0); + if (!php_kind(name, "name") && !php_kind(name, "qualified_name")) { + return; + } + size_t nl = 0; + const char *nt = php_name_text(w, name, &nl); + const char *full = nt; + size_t fl = nl; + if (prefix_len) { + char *joined = + cbm_arena_sprintf(w->sb->a, "%.*s\\%.*s", (int)prefix_len, prefix, (int)nl, nt); + if (!joined) { + w->sb->oom = true; + return; + } + full = joined; + fl = strlen(joined); + } + const char *al = NULL; + size_t all = 0; + if (!ts_node_is_null(alias)) { + al = php_name_text(w, alias, &all); + } else { + const char *bs = full + fl; + while (bs > full && bs[-SKIP_ONE] != '\\') { + bs--; + } + al = bs; + all = (size_t)(full + fl - bs); + } + php_add_use(w, php_use_kind(clause, kind), al, all, full, fl); +} + +static void php_use_decl(php_walk_t *w, TSNode decl) { + char kind = php_use_kind(decl, 'c'); + TSNode body = ts_node_child_by_field_name(decl, TS_FIELD("body")); + const char *prefix = NULL; + size_t pl = 0; + if (!ts_node_is_null(body)) { + TSNode pre = ts_node_named_child(decl, 0); + if (!php_kind(pre, "namespace_name")) { + return; + } + prefix = php_name_text(w, pre, &pl); + if (pl && prefix[0] == '\\') { + prefix++; + pl--; + } + } + int n = 0; + TSNode *kids = php_children(w, ts_node_is_null(body) ? decl : body, &n); + for (int i = 0; i < n; i++) { + if (php_kind(kids[i], "namespace_use_clause")) { + php_use_clause(w, kids[i], kind, prefix, pl); + } + } +} + +static bool php_is_type_decl(TSNode n) { + return php_kind(n, "class_declaration") || php_kind(n, "interface_declaration") || + php_kind(n, "trait_declaration") || php_kind(n, "enum_declaration"); +} + +static char php_type_kind(TSNode n) { + if (php_kind(n, "interface_declaration")) { + return 'i'; + } + if (php_kind(n, "trait_declaration")) { + return 't'; + } + return php_kind(n, "enum_declaration") ? 'e' : 'c'; +} + +/* The name of a type or function declaration, NUL-terminated in the arena. */ +static const char *php_decl_name(php_walk_t *w, TSNode decl, size_t *len) { + TSNode name = ts_node_child_by_field_name(decl, TS_FIELD("name")); + *len = 0; + return ts_node_is_null(name) ? NULL : php_name_text(w, name, len); +} + +/* `fqn` of a declared short name in the current block. */ +static const char *php_fqn(php_walk_t *w, const char *name, size_t len, size_t *out_len) { + if (!w->ns_len) { + *out_len = len; + return name; + } + char *s = cbm_arena_sprintf(w->sb->a, "%.*s\\%.*s", (int)w->ns_len, w->ns, (int)len, name); + if (!s) { + w->sb->oom = true; + *out_len = 0; + return ""; + } + *out_len = strlen(s); + return s; +} + +static char php_dup_flag(php_walk_t *w, const char *name) { + uintptr_t count = (uintptr_t)cbm_ht_get(w->names, name); + return count > SKIP_ONE ? 'd' : '-'; +} + +static void php_base(php_walk_t *w, char rel, TSNode name) { + size_t nl = 0; + const char *nt = php_name_text(w, name, &nl); + char buf[CBM_SZ_1K]; + size_t fl = + cbm_doclink_php_qualify(buf, sizeof(buf), w->ns, w->ns_len, w->uses, w->nuses, nt, nl); + if (!fl) { + return; + } + php_sb_puts(w->sb, "\nB"); + php_sb_field(w->sb, &rel, SKIP_ONE); + php_sb_field(w->sb, buf, fl); +} + +/* The names a base clause, interface clause or trait use lists. */ +static void php_bases(php_walk_t *w, char rel, TSNode clause) { + int n = 0; + TSNode *kids = php_children(w, clause, &n); + for (int i = 0; i < n; i++) { + if (php_kind(kids[i], "name") || php_kind(kids[i], "qualified_name")) { + php_base(w, rel, kids[i]); + } + } +} + +static void php_member(php_walk_t *w, char kind, TSNode name) { + size_t nl = 0; + const char *nt = php_name_text(w, name, &nl); + if (nl && nt[0] == '$') { + nt++; + nl--; + } + if (!nl) { + return; + } + php_sb_puts(w->sb, "\nK"); + php_sb_field(w->sb, &kind, SKIP_ONE); + php_sb_field(w->sb, nt, nl); +} + +/* Promoted constructor parameters are properties. */ +static void php_promoted(php_walk_t *w, TSNode method) { + TSNode params = ts_node_child_by_field_name(method, TS_FIELD("parameters")); + int n = 0; + TSNode *kids = ts_node_is_null(params) ? NULL : php_children(w, params, &n); + for (int i = 0; i < n; i++) { + if (php_kind(kids[i], "property_promotion_parameter")) { + TSNode name = ts_node_child_by_field_name(kids[i], TS_FIELD("name")); + if (!ts_node_is_null(name)) { + php_member(w, 'p', name); + } + } + } +} + +/* The elements of a property or constant declaration. */ +static void php_elements(php_walk_t *w, TSNode decl, const char *elem, char kind) { + int n = 0; + TSNode *kids = php_children(w, decl, &n); + for (int i = 0; i < n; i++) { + if (!php_kind(kids[i], elem)) { + continue; + } + TSNode name = ts_node_child_by_field_name(kids[i], TS_FIELD("name")); + if (ts_node_is_null(name)) { + name = ts_node_named_child(kids[i], 0); + } + if (php_kind(name, "name") || php_kind(name, "variable_name")) { + php_member(w, kind, name); + } + } +} + +static void php_type_body(php_walk_t *w, TSNode body) { + int n = 0; + TSNode *kids = php_children(w, body, &n); + for (int i = 0; i < n; i++) { + TSNode c = kids[i]; + if (php_kind(c, "use_declaration")) { + php_bases(w, 't', c); + } + } + for (int i = 0; i < n; i++) { + TSNode c = kids[i]; + if (php_kind(c, "method_declaration")) { + TSNode name = ts_node_child_by_field_name(c, TS_FIELD("name")); + if (!ts_node_is_null(name)) { + php_member(w, 'm', name); + } + php_promoted(w, c); + } else if (php_kind(c, "property_declaration")) { + php_elements(w, c, "property_element", 'p'); + } else if (php_kind(c, "const_declaration")) { + php_elements(w, c, "const_element", 'c'); + } else if (php_kind(c, "enum_case")) { + TSNode name = ts_node_child_by_field_name(c, TS_FIELD("name")); + if (!ts_node_is_null(name)) { + php_member(w, 'e', name); + } + } + } +} + +static void php_type(php_walk_t *w, TSNode decl) { + size_t nl = 0; + const char *name = php_decl_name(w, decl, &nl); + if (!name || !nl) { + return; + } + size_t fl = 0; + const char *fqn = php_fqn(w, name, nl, &fl); + char kind = php_type_kind(decl); + char flag = php_dup_flag(w, name); + php_sb_puts(w->sb, "\nT"); + php_sb_field(w->sb, &kind, SKIP_ONE); + php_sb_field(w->sb, fqn, fl); + php_sb_field(w->sb, &flag, SKIP_ONE); + php_sb_num(w->sb, ts_node_start_point(decl).row + SKIP_ONE); + php_sb_num(w->sb, ts_node_end_point(decl).row + SKIP_ONE); + int n = 0; + TSNode *kids = php_children(w, decl, &n); + for (int i = 0; i < n; i++) { + if (php_kind(kids[i], "base_clause")) { + php_bases(w, kind == 'i' ? 'i' : 'e', kids[i]); + } else if (php_kind(kids[i], "class_interface_clause")) { + php_bases(w, 'i', kids[i]); + } + } + TSNode body = ts_node_child_by_field_name(decl, TS_FIELD("body")); + if (!ts_node_is_null(body)) { + php_type_body(w, body); + } +} + +static void php_function(php_walk_t *w, TSNode decl) { + size_t nl = 0; + const char *name = php_decl_name(w, decl, &nl); + if (!name || !nl) { + return; + } + size_t fl = 0; + const char *fqn = php_fqn(w, name, nl, &fl); + char flag = php_dup_flag(w, name); + php_sb_puts(w->sb, "\nF"); + php_sb_field(w->sb, fqn, fl); + php_sb_field(w->sb, &flag, SKIP_ONE); +} + +static void php_count_name(php_walk_t *w, TSNode decl); + +static bool php_is_branch(TSNode n) { + return php_kind(n, "if_statement") || php_kind(n, "compound_statement") || + php_kind(n, "else_clause") || php_kind(n, "else_if_clause") || + php_kind(n, "colon_block"); +} + +/* The types and functions a top-level conditional declares + * (`if (!function_exists('f')) { function f() {} }`): PHP declares them when + * the branch runs, and the graph has their nodes. Counted for the dup flags + * (count) or written. */ +static void php_cond_decls(php_walk_t *w, TSNode n, int depth, bool count) { + if (depth > PHP_COND_DEPTH) { + return; + } + int k = 0; + TSNode *kids = php_children(w, n, &k); + for (int i = 0; i < k && !w->sb->oom; i++) { + TSNode c = kids[i]; + bool type = php_is_type_decl(c); + if (type || php_kind(c, "function_definition")) { + if (count) { + php_count_name(w, c); + } else if (type) { + php_type(w, c); + } else { + php_function(w, c); + } + } else if (php_is_branch(c)) { + php_cond_decls(w, c, depth + SKIP_ONE, count); + } + } +} + +/* One namespace block: kids[from, to) are its statements. Its use imports + * are read first: a supertype is qualified with all of them. */ +static void php_block(php_walk_t *w, const TSNode *kids, int from, int to) { + w->nuses = 0; + for (int i = from; i < to; i++) { + if (php_kind(kids[i], "namespace_use_declaration")) { + php_use_decl(w, kids[i]); + } + } + for (int i = from; i < to && !w->sb->oom; i++) { + if (php_is_type_decl(kids[i])) { + php_type(w, kids[i]); + } else if (php_kind(kids[i], "function_definition")) { + php_function(w, kids[i]); + } else if (php_kind(kids[i], "if_statement")) { + php_cond_decls(w, kids[i], 0, false); + } + } +} + +static void php_count_name(php_walk_t *w, TSNode decl) { + size_t nl = 0; + const char *name = php_decl_name(w, decl, &nl); + if (!name || !nl) { + return; + } + uintptr_t count = (uintptr_t)cbm_ht_get(w->names, name); + cbm_ht_set(w->names, name, (void *)(count + SKIP_ONE)); +} + +/* Count the short names of the top-level types and functions of kids[0, n) + * and of braced namespace bodies among them. */ +static void php_count_names(php_walk_t *w, const TSNode *kids, int n) { + for (int i = 0; i < n; i++) { + TSNode c = kids[i]; + if (php_is_type_decl(c) || php_kind(c, "function_definition")) { + php_count_name(w, c); + } else if (php_kind(c, "if_statement")) { + php_cond_decls(w, c, 0, true); + } else if (php_kind(c, "namespace_definition")) { + TSNode body = ts_node_child_by_field_name(c, TS_FIELD("body")); + int bn = 0; + TSNode *bk = ts_node_is_null(body) ? NULL : php_children(w, body, &bn); + php_count_names(w, bk, bn); + } + } +} + +static void php_namespace_rec(php_walk_t *w, TSNode def, uint32_t first, uint32_t last) { + TSNode name = ts_node_child_by_field_name(def, TS_FIELD("name")); + w->ns = ""; + w->ns_len = 0; + if (!ts_node_is_null(name)) { + w->ns = php_name_text(w, name, &w->ns_len); + if (w->ns_len && w->ns[0] == '\\') { + w->ns++; + w->ns_len--; + } + } + php_sb_puts(w->sb, "\nN"); + php_sb_num(w->sb, (uint32_t)w->blk); + php_sb_field(w->sb, w->ns, w->ns_len); + php_sb_num(w->sb, first); + php_sb_num(w->sb, last); +} + +static void php_walk_blocks(php_walk_t *w, const TSNode *kids, int n, uint32_t file_last) { + int first_ns = CBM_NOT_FOUND; + for (int i = 0; i < n && first_ns < 0; i++) { + first_ns = php_kind(kids[i], "namespace_definition") ? i : first_ns; + } + if (first_ns < 0) { + w->blk = 0; + php_block(w, kids, 0, n); + return; + } + w->blk = CBM_NOT_FOUND; + for (int i = first_ns; i < n && !w->sb->oom; i++) { + if (!php_kind(kids[i], "namespace_definition")) { + continue; + } + w->blk++; + TSNode def = kids[i]; + TSNode body = ts_node_child_by_field_name(def, TS_FIELD("body")); + uint32_t first = ts_node_start_point(def).row + SKIP_ONE; + if (!ts_node_is_null(body)) { + php_namespace_rec(w, def, first, ts_node_end_point(def).row + SKIP_ONE); + int bn = 0; + TSNode *bk = php_children(w, body, &bn); + php_block(w, bk, 0, bn); + continue; + } + int next = i + SKIP_ONE; + while (next < n && !php_kind(kids[next], "namespace_definition")) { + next++; + } + uint32_t last = next < n ? ts_node_start_point(kids[next]).row : file_last; + php_namespace_rec(w, def, first, last); + php_block(w, kids, i + SKIP_ONE, next); + i = next - SKIP_ONE; + } +} + +const char *cbm_doclink_php_scan_scope(CBMExtractCtx *ctx) { + if (!ctx || ts_node_is_null(ctx->root)) { + return NULL; + } + php_sb_t sb = {.a = ctx->arena}; + php_sb_puts(&sb, CBM_DOCLINK_PHP_SCOPE_TAG); + php_walk_t w = {.ctx = ctx, .sb = &sb, .ns = ""}; + w.names = cbm_ht_create_in(CBM_MEM_CLASS_OTHER, CBM_SZ_32); + if (!w.names) { + return NULL; + } + int n = 0; + TSNode *kids = php_children(&w, ctx->root, &n); + php_count_names(&w, kids, n); + php_walk_blocks(&w, kids, n, ts_node_end_point(ctx->root).row + SKIP_ONE); + cbm_ht_free(w.names); + return sb.oom ? NULL : sb.buf; +} + +/* The blob without lines and imports: "N blk name first last" -> "N blk + * name", "T kind fqn flags first last" -> "T kind fqn flags", U dropped. */ +char *cbm_doclink_php_portable_scope(const char *scope) { + size_t n = strlen(scope); + char *out = (char *)cbm_alloc(CBM_MEM_CLASS_OTHER, n + SKIP_ONE); + if (!out) { + return NULL; + } + size_t w = 0; + for (const char *p = scope; *p;) { + const char *nl = strchr(p, '\n'); + size_t len = nl ? (size_t)(nl - p) : strlen(p); + const char *next = nl ? nl + SKIP_ONE : p + len; + bool rec = len >= PAIR_LEN && p[SKIP_ONE] == '\t'; + if (rec && p[0] == 'U') { + p = next; + continue; + } + size_t keep = len; + if (rec && (p[0] == 'N' || p[0] == 'T')) { + int tabs = 0; + for (size_t k = 0; k < len; k++) { + if (p[k] == '\t' && ++tabs == (p[0] == 'N' ? CBM_SZ_3 : CBM_SZ_4)) { + keep = k; + break; + } + } + } + memcpy(out + w, p, keep); + w += keep; + if (nl) { + out[w++] = '\n'; + } + p = next; + } + out[w] = '\0'; + return out; +} diff --git a/src/pipeline/doc_links.c b/src/pipeline/doc_links.c index f01fff7c7..802665a35 100644 --- a/src/pipeline/doc_links.c +++ b/src/pipeline/doc_links.c @@ -48,7 +48,7 @@ static const cbm_doclink_resolver_t *const DOCLINK_RESOLVERS[] = { &cbm_doclink_cs_resolver, &cbm_doclink_md_resolver, &cbm_doclink_pdf_resolver, &cbm_doclink_rst_resolver, &cbm_doclink_adoc_resolver, &cbm_doclink_go_resolver, &cbm_doclink_jvm_resolver, &cbm_doclink_rs_resolver, &cbm_doclink_perl_resolver, - &cbm_doclink_ts_resolver, + &cbm_doclink_ts_resolver, &cbm_doclink_php_resolver, }; enum { DOCLINK_RESOLVER_COUNT = sizeof(DOCLINK_RESOLVERS) / sizeof(DOCLINK_RESOLVERS[0]) }; diff --git a/src/pipeline/doc_links.h b/src/pipeline/doc_links.h index f21049792..11ab9ec34 100644 --- a/src/pipeline/doc_links.h +++ b/src/pipeline/doc_links.h @@ -300,6 +300,7 @@ extern const cbm_doclink_resolver_t cbm_doclink_jvm_resolver; extern const cbm_doclink_resolver_t cbm_doclink_rs_resolver; extern const cbm_doclink_resolver_t cbm_doclink_perl_resolver; extern const cbm_doclink_resolver_t cbm_doclink_ts_resolver; +extern const cbm_doclink_resolver_t cbm_doclink_php_resolver; /* True when `rel_path` is a scope input of some language (scope_input). */ bool cbm_doclinks_is_scope_input(const char *rel_path); diff --git a/src/pipeline/doc_links_php.c b/src/pipeline/doc_links_php.c new file mode 100644 index 000000000..959595c6e --- /dev/null +++ b/src/pipeline/doc_links_php.c @@ -0,0 +1,975 @@ +/* + * doc_links_php.c — resolving PHP doc references (doclink_php.c). A reference + * is a FQSEN as phpDocumentor reads it, resolved with PHP's own name rules. + * + * Forms: `\Ns\Type`, `Type`, `Type::method()`, `Type::$property`, + * `Type::CONSTANT` (an enum case too), `function()` and `\Ns\function()`; + * `self::`, `static::` and `parent::` with a member. A member written without + * `()` and without `$` is a constant or enum case, else a method. + * + * Names. A leading backslash is taken as written. Otherwise the first segment + * of a type name is replaced when it is the alias of a class `use` of the + * namespace block the reference is written in (aliases compare without case), + * else the block's namespace is put in front: phpDocumentor's rule. Two rules + * go beyond that tool, as PHP itself has them: an unqualified function name + * is taken from a `use function` import, else from the current namespace, + * else from the global namespace; and `self` and `static` name the type the + * doc sits in, `parent` that type's superclass. A bare `name()` that would + * bind a function but is written in a type that declares or inherits a + * method of that name is ambiguous: the method or the function, and the + * reference does not say which. Type, function + * and method names compare without case, properties and constants with it. + * The global fallback cannot see a function a dependency declares in the + * current namespace: a reference to one binds the global function of that + * name (a residual). + * + * Members are looked up on the type, then on the traits it uses, its + * superclass and its interfaces, each of them the same way (depth first; the + * first declaration found wins). Nothing found where a supertype is outside + * the repository is external; nothing found with every supertype in the + * repository is missing. + * + * Outcomes. A type, function or method binds to its graph node. A property, + * constant or enum case has none (a graph gap), nor has a type or function + * the graph folds into another one of the same name in its file (the scope + * flags those). A type or qualified function the repository does not declare + * is external when the first segment of its name is no namespace the + * repository declares (PHP's built-ins, Composer dependencies), else missing; + * an unqualified function nobody declares is external (PHP's built-in + * functions). Two declarations of one name are ambiguous unless one is in the + * reference's own file. Test code binds only from test code. Edges are exact + * for a name written fully qualified or bound by a use import, unique + * otherwise. + */ +#include "pipeline/doc_links.h" +#include "cbm.h" +#include "doclink.h" +#include "helpers.h" /* cbm_fqn_module_source_lang, cbm_is_test_file */ +#include "foundation/arena.h" +#include "foundation/constants.h" +#include "foundation/hash_table.h" +#include "foundation/log.h" +#include "foundation/mem_core.h" +#include "graph_buffer/graph_buffer.h" + +#include +#include +#include +#include +#include +#include +#include + +enum { + PR_QN_MAX = 2048, + PR_NAME_MAX = 1024, + PR_FIELDS = 6, + PR_MAX_DEPTH = 16, + PR_VISIT_MAX = 64, + PR_USE_FIELDS = 5, /* U blk kind alias name */ + PR_LINE_FIELDS = 4, /* N blk name first last / T kind fqn flags first last: lines from here */ +}; + +/* ── The index ───────────────────────────────────────────────────── */ + +typedef struct { + char kind; /* m p c e */ + const char *name; +} pr_member_t; + +typedef struct { + char rel; /* e i t */ + const char *fqn; +} pr_base_t; + +typedef struct { + int file; + char kind; /* c i t e */ + bool dup; /* folded into another declaration of its file */ + const char *fqn; + const char *name; /* the tail of fqn */ + uint32_t first; /* lines: only in a fresh blob */ + uint32_t last; + pr_base_t *bases; + int nbases; + int cap_bases; + pr_member_t *members; + int nmembers; + int cap_members; + int next; /* the next type of the same name (without case), or -1 */ +} pr_type_t; + +typedef struct { + int file; + bool dup; + const char *fqn; + const char *name; + int next; +} pr_func_t; + +typedef struct { + int blk; + const char *name; + uint32_t first; + uint32_t last; +} pr_ns_t; + +typedef struct { + const char *path; + const char *module_qn; + bool test; + pr_ns_t *nss; + int nnss; + int cap_nss; + cbm_doclink_php_use_t *uses; /* only in a fresh blob; a block's are adjacent */ + int *use_blk; + int nuses; + int cap_uses; + int cap_use_blk; + int type_first; /* its types: types[type_first, type_first + type_count) */ + int type_count; +} pr_file_t; + +typedef struct { + CBMArena arena; + const char *project; + pr_file_t *files; + int nfiles; + int *run_to_file; + int run_count; + pr_type_t *types; + int ntypes; + int cap_types; + pr_func_t *funcs; + int nfuncs; + int cap_funcs; + CBMHashTable *type_at; /* lowercase fqn -> the first type's index + 1 */ + CBMHashTable *func_at; /* lowercase fqn -> the first function's index + 1 */ + CBMHashTable *roots; /* lowercase first segment of a declared namespace */ + CBMHashTable *by_path; /* rel_path -> file index + 1 */ + bool oom; +} pr_index_t; + +static const char *pr_copy(pr_index_t *ix, const char *s, size_t n) { + char *c = cbm_arena_strndup(&ix->arena, s, n); + ix->oom = ix->oom || !c; + return c; +} + +static const char *pr_lower(pr_index_t *ix, const char *s, size_t n) { + char *c = cbm_arena_strndup(&ix->arena, s, n); + if (!c) { + ix->oom = true; + return NULL; + } + for (size_t i = 0; i < n; i++) { + c[i] = (char)tolower((unsigned char)c[i]); + } + return c; +} + +/* Room for one more item in a growing arena array. */ +static bool pr_grow(pr_index_t *ix, void **items, int count, int *cap, size_t size) { + if (count < *cap) { + return true; + } + int ncap = *cap ? *cap * PAIR_LEN : CBM_SZ_8; + void *g = cbm_arena_alloc(&ix->arena, (size_t)ncap * size); + if (!g) { + ix->oom = true; + return false; + } + if (count) { + memcpy(g, *items, (size_t)count * size); + } + *items = g; + *cap = ncap; + return true; +} + +static bool pr_dash(const char *s, size_t n) { + return n == SKIP_ONE && s[0] == '-'; +} + +static uint32_t pr_num(const char *s) { + return (uint32_t)strtoul(s, NULL, CBM_DECIMAL_BASE); +} + +static pr_type_t *pr_last_type(pr_index_t *ix, const pr_file_t *f) { + return f->type_count ? &ix->types[f->type_first + f->type_count - SKIP_ONE] : NULL; +} + +static void pr_read_type(pr_index_t *ix, pr_file_t *f, int fi, const char **fl, const size_t *ln, + int n) { + if (!pr_grow(ix, (void **)&ix->types, ix->ntypes, &ix->cap_types, sizeof(pr_type_t))) { + return; + } + if (!f->type_count) { + f->type_first = ix->ntypes; + } + pr_type_t *t = &ix->types[ix->ntypes++]; + f->type_count++; + memset(t, 0, sizeof(*t)); + t->file = fi; + t->kind = fl[1][0]; + t->fqn = pr_copy(ix, fl[PAIR_LEN], ln[PAIR_LEN]); + t->dup = fl[CBM_SZ_3][0] == 'd'; + t->next = CBM_NOT_FOUND; + if (n > PR_LINE_FIELDS) { + t->first = pr_num(fl[PR_LINE_FIELDS]); + t->last = n > PR_LINE_FIELDS + SKIP_ONE ? pr_num(fl[PR_LINE_FIELDS + SKIP_ONE]) : 0; + } + if (t->fqn) { + const char *bs = strrchr(t->fqn, '\\'); + t->name = bs ? bs + SKIP_ONE : t->fqn; + } +} + +static void pr_read_record(pr_index_t *ix, pr_file_t *f, int fi, const char **fl, const size_t *ln, + int n) { + char tag = fl[0][0]; + pr_type_t *t = NULL; + if (tag == 'N' && n >= CBM_SZ_3 && + pr_grow(ix, (void **)&f->nss, f->nnss, &f->cap_nss, sizeof(pr_ns_t))) { + pr_ns_t *s = &f->nss[f->nnss++]; + s->blk = (int)pr_num(fl[1]); + s->name = + pr_dash(fl[PAIR_LEN], ln[PAIR_LEN]) ? "" : pr_copy(ix, fl[PAIR_LEN], ln[PAIR_LEN]); + s->first = n > CBM_SZ_3 ? pr_num(fl[CBM_SZ_3]) : 0; + s->last = n > CBM_SZ_4 ? pr_num(fl[CBM_SZ_4]) : 0; + } else if (tag == 'U' && n >= PR_USE_FIELDS && + pr_grow(ix, (void **)&f->uses, f->nuses, &f->cap_uses, + sizeof(cbm_doclink_php_use_t)) && + pr_grow(ix, (void **)&f->use_blk, f->nuses, &f->cap_use_blk, sizeof(int))) { + cbm_doclink_php_use_t *u = &f->uses[f->nuses]; + f->use_blk[f->nuses++] = (int)pr_num(fl[1]); + u->kind = fl[PAIR_LEN][0]; + u->alias = pr_copy(ix, fl[CBM_SZ_3], ln[CBM_SZ_3]); + u->alias_len = ln[CBM_SZ_3]; + u->name = pr_copy(ix, fl[CBM_SZ_4], ln[CBM_SZ_4]); + u->name_len = ln[CBM_SZ_4]; + } else if (tag == 'T' && n >= CBM_SZ_4) { + pr_read_type(ix, f, fi, fl, ln, n); + } else if (tag == 'B' && n >= CBM_SZ_3 && (t = pr_last_type(ix, f)) && + pr_grow(ix, (void **)&t->bases, t->nbases, &t->cap_bases, sizeof(pr_base_t))) { + t->bases[t->nbases++] = (pr_base_t){fl[1][0], pr_copy(ix, fl[PAIR_LEN], ln[PAIR_LEN])}; + } else if (tag == 'K' && n >= CBM_SZ_3 && (t = pr_last_type(ix, f)) && + pr_grow(ix, (void **)&t->members, t->nmembers, &t->cap_members, + sizeof(pr_member_t))) { + t->members[t->nmembers++] = + (pr_member_t){fl[1][0], pr_copy(ix, fl[PAIR_LEN], ln[PAIR_LEN])}; + } else if (tag == 'F' && n >= CBM_SZ_3 && + pr_grow(ix, (void **)&ix->funcs, ix->nfuncs, &ix->cap_funcs, sizeof(pr_func_t))) { + pr_func_t *fn = &ix->funcs[ix->nfuncs++]; + fn->file = fi; + fn->fqn = pr_copy(ix, fl[1], ln[1]); + fn->dup = fl[PAIR_LEN][0] == 'd'; + fn->next = CBM_NOT_FOUND; + const char *bs = fn->fqn ? strrchr(fn->fqn, '\\') : NULL; + fn->name = bs ? bs + SKIP_ONE : fn->fqn; + } +} + +static void pr_read_blob(pr_index_t *ix, pr_file_t *f, int fi, const char *blob) { + const char *line = strchr(blob, '\n'); /* the tag line */ + while (line && !ix->oom) { + const char *b = line + SKIP_ONE; + const char *e = strchr(b, '\n'); + if (!e) { + e = b + strlen(b); + } + const char *fl[PR_FIELDS]; + size_t ln[PR_FIELDS]; + int n = 0; + const char *s = b; + for (const char *p = b; p <= e && n < PR_FIELDS; p++) { + if (p == e || *p == '\t') { + fl[n] = s; + ln[n] = (size_t)(p - s); + n++; + s = p + SKIP_ONE; + } + } + if (n > 0 && ln[0] == SKIP_ONE) { + pr_read_record(ix, f, fi, fl, ln, n); + } + line = *e ? e : NULL; + } +} + +static void pr_destroy(void *index) { + pr_index_t *ix = (pr_index_t *)index; + if (!ix) { + return; + } + cbm_ht_free(ix->type_at); + cbm_ht_free(ix->func_at); + cbm_ht_free(ix->roots); + cbm_ht_free(ix->by_path); + cbm_free(CBM_MEM_CLASS_OTHER, ix->run_to_file); + cbm_arena_destroy(&ix->arena); + cbm_free(CBM_MEM_CLASS_OTHER, ix); +} + +/* The namespace root of a qualified name ("" for a global one). */ +static void pr_add_root(pr_index_t *ix, const char *name) { + const char *bs = name ? strchr(name, '\\') : NULL; + size_t n = bs ? (size_t)(bs - name) : (name ? strlen(name) : 0); + if (!n) { + return; + } + const char *key = pr_lower(ix, name, n); + if (key && !cbm_ht_get(ix->roots, key)) { + cbm_ht_set(ix->roots, key, (void *)(uintptr_t)SKIP_ONE); + } +} + +/* Chain every type and function into its name's list, and collect the + * namespace roots. */ +static bool pr_link(pr_index_t *ix) { + ix->type_at = cbm_ht_create_in(CBM_MEM_CLASS_OTHER, (uint32_t)(ix->ntypes + SKIP_ONE)); + ix->func_at = cbm_ht_create_in(CBM_MEM_CLASS_OTHER, (uint32_t)(ix->nfuncs + SKIP_ONE)); + ix->roots = cbm_ht_create_in(CBM_MEM_CLASS_OTHER, CBM_SZ_64); + if (!ix->type_at || !ix->func_at || !ix->roots) { + return false; + } + for (int i = 0; i < ix->ntypes && !ix->oom; i++) { + pr_type_t *t = &ix->types[i]; + const char *key = t->fqn ? pr_lower(ix, t->fqn, strlen(t->fqn)) : NULL; + if (!key) { + continue; + } + void *prev = cbm_ht_get(ix->type_at, key); + t->next = prev ? (int)((uintptr_t)prev - SKIP_ONE) : CBM_NOT_FOUND; + cbm_ht_set(ix->type_at, key, (void *)(uintptr_t)(i + SKIP_ONE)); + if (strchr(t->fqn, '\\')) { + pr_add_root(ix, t->fqn); + } + } + for (int i = 0; i < ix->nfuncs && !ix->oom; i++) { + pr_func_t *fn = &ix->funcs[i]; + const char *key = fn->fqn ? pr_lower(ix, fn->fqn, strlen(fn->fqn)) : NULL; + if (!key) { + continue; + } + void *prev = cbm_ht_get(ix->func_at, key); + fn->next = prev ? (int)((uintptr_t)prev - SKIP_ONE) : CBM_NOT_FOUND; + cbm_ht_set(ix->func_at, key, (void *)(uintptr_t)(i + SKIP_ONE)); + if (strchr(fn->fqn, '\\')) { + pr_add_root(ix, fn->fqn); + } + } + for (int i = 0; i < ix->nfiles && !ix->oom; i++) { + for (int k = 0; k < ix->files[i].nnss; k++) { + pr_add_root(ix, ix->files[i].nss[k].name); + } + } + return !ix->oom; +} + +static void *pr_build(const cbm_doclink_build_in_t *in) { + pr_index_t *ix = (pr_index_t *)cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*ix)); + if (!ix) { + return NULL; + } + cbm_arena_init(&ix->arena); + ix->project = in->ctx->project_name; + ix->nfiles = in->file_count; + ix->run_count = in->run_file_count; + ix->files = + in->file_count + ? (pr_file_t *)cbm_arena_calloc(&ix->arena, (size_t)in->file_count * sizeof(pr_file_t)) + : NULL; + ix->run_to_file = (int *)cbm_calloc(CBM_MEM_CLASS_OTHER, + (size_t)(in->run_file_count + SKIP_ONE) * sizeof(int)); + ix->by_path = cbm_ht_create_in(CBM_MEM_CLASS_OTHER, (uint32_t)(in->file_count + SKIP_ONE)); + bool ok = (!in->file_count || ix->files) && ix->run_to_file && ix->by_path; + for (int r = 0; ok && r < in->run_file_count; r++) { + ix->run_to_file[r] = CBM_NOT_FOUND; + } + for (int i = 0; ok && i < in->file_count && !ix->oom; i++) { + const cbm_doclink_file_t *src = &in->files[i]; + pr_file_t *f = &ix->files[i]; + f->path = src->rel_path; + f->module_qn = + cbm_fqn_module_source_lang(&ix->arena, ix->project, src->rel_path, CBM_LANG_PHP); + f->test = cbm_is_test_file(src->rel_path, CBM_LANG_PHP); + ix->oom = ix->oom || !f->module_qn; + cbm_ht_set(ix->by_path, src->rel_path, (void *)(uintptr_t)(i + SKIP_ONE)); + if (src->run_file >= 0 && src->run_file < in->run_file_count) { + ix->run_to_file[src->run_file] = i; + } + if (src->scope) { + pr_read_blob(ix, f, i, src->scope); + } + } + ok = ok && !ix->oom && pr_link(ix); + if (!ok) { + cbm_log_error("doc_links.php.error", "step", "index", "reason", "alloc"); + pr_destroy(ix); + return NULL; + } + char b[CBM_SZ_3][CBM_SZ_32]; + snprintf(b[0], sizeof(b[0]), "%d", ix->nfiles); + snprintf(b[SKIP_ONE], sizeof(b[SKIP_ONE]), "%d", ix->ntypes); + snprintf(b[PAIR_LEN], sizeof(b[PAIR_LEN]), "%d", ix->nfuncs); + cbm_log_info("doc_links.php.index", "files", b[0], "types", b[SKIP_ONE], "functions", + b[PAIR_LEN]); + return ix; +} + +/* ── References ──────────────────────────────────────────────────── */ + +typedef struct { + const char *cls; /* a type or function name, NULL for none */ + size_t cls_len; + const char *mem; /* a member name (no `$`), NULL for none */ + size_t mem_len; + bool prop; /* `::$name` */ + bool call; /* `()` at the end */ +} pr_ref_t; + +static bool pr_ident_start(char c) { + return isalpha((unsigned char)c) || c == '_' || (unsigned char)c >= 0x80; +} + +static bool pr_ident_char(char c) { + return isalnum((unsigned char)c) || c == '_' || (unsigned char)c >= 0x80; +} + +/* The end of an identifier at s[i] (i when there is none). */ +static size_t pr_ident_end(const char *s, size_t i, size_t n) { + if (i >= n || !pr_ident_start(s[i])) { + return i; + } + while (i < n && pr_ident_char(s[i])) { + i++; + } + return i; +} + +/* The end of a name (`\`-joined identifiers, a leading `\` allowed) at s[0]. */ +static size_t pr_name_end(const char *s, size_t n) { + size_t i = (n > 0 && s[0] == '\\') ? SKIP_ONE : 0; + size_t e = pr_ident_end(s, i, n); + if (e == i) { + return 0; + } + while (e + SKIP_ONE < n && s[e] == '\\' && pr_ident_start(s[e + SKIP_ONE])) { + e = pr_ident_end(s, e + SKIP_ONE, n); + } + return e; +} + +static bool pr_parse(const char *s, size_t n, pr_ref_t *r) { + memset(r, 0, sizeof(*r)); + size_t i = pr_name_end(s, n); + if (i == 0) { + return false; /* a member needs its type: `::name` is no FQSEN */ + } + r->cls = s; + r->cls_len = i; + if (i + PAIR_LEN <= n && s[i] == ':' && s[i + SKIP_ONE] == ':') { + i += PAIR_LEN; + if (i < n && s[i] == '$') { + r->prop = true; + i++; + } + size_t e = pr_ident_end(s, i, n); + if (e == i) { + return false; + } + r->mem = s + i; + r->mem_len = e - i; + i = e; + } + if (!r->prop && i + PAIR_LEN == n && s[i] == '(' && s[i + SKIP_ONE] == ')') { + r->call = true; + i = n; + } + return i == n; +} + +/* ── Resolution ──────────────────────────────────────────────────── */ + +typedef enum { + PR_NONE = 0, /* nothing found */ + PR_TYPE, /* a type of the index (`type`) */ + PR_NODE, + PR_GAP, + PR_EXTERNAL, + PR_MISSING, + PR_AMBIGUOUS, +} pr_kind_t; + +typedef struct { + pr_kind_t kind; + int type; + const cbm_gbuf_node_t *node; + bool exact; +} pr_res_t; + +typedef struct { + const pr_index_t *ix; + const cbm_gbuf_t *g; + int src; + const char *ns; /* the reference's namespace block */ + size_t ns_len; + const cbm_doclink_php_use_t *uses; /* that block's imports */ + int nuses; + int cur; /* the type the doc sits in, or -1 */ +} pr_ctx_t; + +static pr_res_t pr_mk(pr_kind_t k) { + return (pr_res_t){k, CBM_NOT_FOUND, NULL, false}; +} + +static bool pr_eq_nocase(const char *a, size_t an, const char *b, size_t bn) { + if (an != bn) { + return false; + } + for (size_t i = 0; i < an; i++) { + if (tolower((unsigned char)a[i]) != tolower((unsigned char)b[i])) { + return false; + } + } + return true; +} + +static void pr_lower_into(char *out, const char *s, size_t n) { + for (size_t i = 0; i < n; i++) { + out[i] = (char)tolower((unsigned char)s[i]); + } + out[n] = '\0'; +} + +/* The namespace block of `line` in file f and its imports. */ +static void pr_block_at(pr_ctx_t *c, const pr_file_t *f, uint32_t line) { + int blk = f->nnss ? CBM_NOT_FOUND : 0; + c->ns = ""; + c->ns_len = 0; + for (int i = 0; i < f->nnss; i++) { + if (line >= f->nss[i].first && line <= f->nss[i].last) { + blk = f->nss[i].blk; + c->ns = f->nss[i].name; + c->ns_len = strlen(c->ns); + break; + } + } + c->uses = NULL; + c->nuses = 0; + for (int i = 0; i < f->nuses; i++) { + if (f->use_blk[i] != blk) { + continue; + } + if (!c->uses) { + c->uses = &f->uses[i]; + } + c->nuses++; + } +} + +/* The type of file f whose lines hold `line`, or -1. */ +static int pr_type_at(const pr_index_t *ix, const pr_file_t *f, uint32_t line) { + for (int i = 0; i < f->type_count; i++) { + const pr_type_t *t = &ix->types[f->type_first + i]; + if (line >= t->first && line <= t->last) { + return f->type_first + i; + } + } + return CBM_NOT_FOUND; +} + +/* The node at `qn` in file `path`, or NULL. */ +static const cbm_gbuf_node_t *pr_node(const pr_ctx_t *c, const char *qn, const char *path) { + const cbm_gbuf_node_t *n = cbm_gbuf_find_by_qn(c->g, qn); + if (!n || !n->label || !n->file_path || strcmp(n->file_path, path) != 0) { + return NULL; + } + return n; +} + +/* A declared type's own node: a gap when the graph folded it or has none. */ +static pr_res_t pr_type_node(const pr_ctx_t *c, int ti) { + const pr_type_t *t = &c->ix->types[ti]; + const pr_file_t *f = &c->ix->files[t->file]; + char qn[PR_QN_MAX]; + int n = snprintf(qn, sizeof(qn), "%s.%s", f->module_qn, t->name); + const cbm_gbuf_node_t *node = + (!t->dup && n > 0 && (size_t)n < sizeof(qn)) ? pr_node(c, qn, f->path) : NULL; + pr_res_t r = pr_mk(node ? PR_NODE : PR_GAP); + r.node = node; + return r; +} + +/* The repository's type named `fqn`: one, several (ambiguous, unless one is + * in the reference's file) or none. */ +static pr_res_t pr_find_type(const pr_ctx_t *c, const char *fqn, size_t len) { + char key[PR_NAME_MAX]; + if (len >= sizeof(key)) { + return pr_mk(PR_NONE); + } + pr_lower_into(key, fqn, len); + void *v = cbm_ht_get(c->ix->type_at, key); + int first = v ? (int)((uintptr_t)v - SKIP_ONE) : CBM_NOT_FOUND; + int count = 0; + int own = CBM_NOT_FOUND; + for (int i = first; i >= 0; i = c->ix->types[i].next) { + count++; + own = c->ix->types[i].file == c->src ? i : own; + } + if (count == 0) { + return pr_mk(PR_NONE); + } + pr_res_t r = pr_mk(count == SKIP_ONE || own >= 0 ? PR_TYPE : PR_AMBIGUOUS); + r.type = count == SKIP_ONE ? first : own; + return r; +} + +/* A name the repository does not declare: missing inside one of its + * namespaces, else external. */ +static pr_res_t pr_absent(const pr_ctx_t *c, const char *fqn, size_t len) { + const char *bs = memchr(fqn, '\\', len); + char key[PR_NAME_MAX]; + size_t rl = bs ? (size_t)(bs - fqn) : 0; + if (!rl || rl >= sizeof(key)) { + return pr_mk(PR_EXTERNAL); + } + pr_lower_into(key, fqn, rl); + return pr_mk(cbm_ht_get(c->ix->roots, key) ? PR_MISSING : PR_EXTERNAL); +} + +/* True when the first segment of `name` is a class-use alias. */ +static bool pr_aliased(const pr_ctx_t *c, const char *name, size_t len) { + const char *bs = memchr(name, '\\', len); + size_t first = bs ? (size_t)(bs - name) : len; + for (int i = 0; i < c->nuses; i++) { + if (c->uses[i].kind == 'c' && + pr_eq_nocase(c->uses[i].alias, c->uses[i].alias_len, name, first)) { + return true; + } + } + return false; +} + +static bool pr_is(const char *s, size_t n, const char *word) { + return pr_eq_nocase(s, n, word, strlen(word)); +} + +/* The superclass of type ti, as a type of the index. */ +static pr_res_t pr_parent(const pr_ctx_t *c, int ti) { + const pr_type_t *t = &c->ix->types[ti]; + for (int i = 0; i < t->nbases; i++) { + if (t->bases[i].rel == 'e' && t->bases[i].fqn) { + pr_res_t r = pr_find_type(c, t->bases[i].fqn, strlen(t->bases[i].fqn)); + return r.kind == PR_NONE ? pr_mk(PR_EXTERNAL) : r; + } + } + return pr_mk(PR_MISSING); +} + +/* The type a reference's type part names. */ +static pr_res_t pr_type_of(const pr_ctx_t *c, const pr_ref_t *r) { + if (pr_is(r->cls, r->cls_len, "self") || pr_is(r->cls, r->cls_len, "static")) { + pr_res_t res = pr_mk(c->cur >= 0 ? PR_TYPE : PR_MISSING); + res.type = c->cur; + return res; + } + if (pr_is(r->cls, r->cls_len, "parent")) { + return c->cur >= 0 ? pr_parent(c, c->cur) : pr_mk(PR_MISSING); + } + char fqn[PR_NAME_MAX]; + size_t fl = cbm_doclink_php_qualify(fqn, sizeof(fqn), c->ns, c->ns_len, c->uses, c->nuses, + r->cls, r->cls_len); + if (!fl) { + return pr_mk(PR_MISSING); + } + pr_res_t res = pr_find_type(c, fqn, fl); + if (res.kind == PR_NONE) { + res = pr_absent(c, fqn, fl); + } + res.exact = r->cls[0] == '\\' || pr_aliased(c, r->cls, r->cls_len); + return res; +} + +/* ── Members across supertypes ───────────────────────────────────── */ + +typedef struct { + const pr_ctx_t *c; + char want; /* 'm' a method, 'p' a property, 'c' a constant or enum case */ + const char *name; + size_t len; + int visited[PR_VISIT_MAX]; + int nvisited; + bool open; /* a supertype outside the repository */ + bool amb; /* a supertype with several declarations */ + int owner; /* found: the declaring type and member */ + const pr_member_t *found; +} pr_search_t; + +static bool pr_member_is(const pr_search_t *s, const pr_member_t *m) { + size_t ml = strlen(m->name); + if (s->want == 'm') { + return m->kind == 'm' && pr_eq_nocase(m->name, ml, s->name, s->len); + } + bool kind = s->want == 'p' ? m->kind == 'p' : (m->kind == 'c' || m->kind == 'e'); + return kind && ml == s->len && memcmp(m->name, s->name, ml) == 0; +} + +static bool pr_seen(pr_search_t *s, int ti) { + for (int i = 0; i < s->nvisited; i++) { + if (s->visited[i] == ti) { + return true; + } + } + if (s->nvisited == PR_VISIT_MAX) { + s->open = true; /* too wide to follow: not every supertype was read */ + return true; + } + s->visited[s->nvisited++] = ti; + return false; +} + +static bool pr_search(pr_search_t *s, int ti, int depth) { + if (depth > PR_MAX_DEPTH) { + s->open = true; + return false; + } + if (pr_seen(s, ti)) { + return false; + } + const pr_type_t *t = &s->c->ix->types[ti]; + for (int i = 0; i < t->nmembers; i++) { + if (pr_member_is(s, &t->members[i])) { + s->owner = ti; + s->found = &t->members[i]; + return true; + } + } + static const char order[] = {'t', 'e', 'i'}; /* traits, superclass, interfaces */ + for (size_t o = 0; o < sizeof(order); o++) { + for (int i = 0; i < t->nbases; i++) { + if (t->bases[i].rel != order[o] || !t->bases[i].fqn) { + continue; + } + pr_res_t r = pr_find_type(s->c, t->bases[i].fqn, strlen(t->bases[i].fqn)); + if (r.kind == PR_TYPE && pr_search(s, r.type, depth + SKIP_ONE)) { + return true; + } + s->amb = s->amb || r.kind == PR_AMBIGUOUS; + s->open = s->open || r.kind == PR_NONE; + } + } + return false; +} + +/* The member's outcome: a method's node, else a graph gap. */ +static pr_res_t pr_member_node(const pr_ctx_t *c, const pr_search_t *s) { + const pr_type_t *t = &c->ix->types[s->owner]; + if (s->found->kind != 'm' || t->dup) { + return pr_mk(PR_GAP); + } + const pr_file_t *f = &c->ix->files[t->file]; + char qn[PR_QN_MAX]; + int n = snprintf(qn, sizeof(qn), "%s.%s.%s", f->module_qn, t->name, s->found->name); + const cbm_gbuf_node_t *node = + (n > 0 && (size_t)n < sizeof(qn)) ? pr_node(c, qn, f->path) : NULL; + pr_res_t r = pr_mk(node ? PR_NODE : PR_GAP); + r.node = node; + return r; +} + +static pr_res_t pr_member(const pr_ctx_t *c, int ti, const pr_ref_t *r) { + pr_search_t s = {.c = c, .name = r->mem, .len = r->mem_len, .owner = CBM_NOT_FOUND}; + s.want = r->prop ? 'p' : r->call ? 'm' : 'c'; + bool hit = pr_search(&s, ti, 0); + if (!hit && s.want == 'c') { + /* `Type::name` without parentheses names a method as often as a + * constant: no constant or case of that name, then a method */ + bool open = s.open; + bool amb = s.amb; + s.want = 'm'; + s.nvisited = 0; + hit = pr_search(&s, ti, 0); + s.open = s.open || open; + s.amb = s.amb || amb; + } + if (hit) { + return pr_member_node(c, &s); + } + return pr_mk(s.amb ? PR_AMBIGUOUS : s.open ? PR_EXTERNAL : PR_MISSING); +} + +/* ── Functions ───────────────────────────────────────────────────── */ + +static pr_res_t pr_find_func(const pr_ctx_t *c, const char *fqn, size_t len) { + char key[PR_NAME_MAX]; + if (len >= sizeof(key)) { + return pr_mk(PR_NONE); + } + pr_lower_into(key, fqn, len); + void *v = cbm_ht_get(c->ix->func_at, key); + int first = v ? (int)((uintptr_t)v - SKIP_ONE) : CBM_NOT_FOUND; + int count = 0; + int own = CBM_NOT_FOUND; + for (int i = first; i >= 0; i = c->ix->funcs[i].next) { + count++; + own = c->ix->funcs[i].file == c->src ? i : own; + } + if (count == 0) { + return pr_mk(PR_NONE); + } + if (count > SKIP_ONE && own < 0) { + return pr_mk(PR_AMBIGUOUS); + } + const pr_func_t *fn = &c->ix->funcs[count == SKIP_ONE ? first : own]; + const pr_file_t *f = &c->ix->files[fn->file]; + char qn[PR_QN_MAX]; + int n = snprintf(qn, sizeof(qn), "%s.%s", f->module_qn, fn->name); + const cbm_gbuf_node_t *node = + (!fn->dup && n > 0 && (size_t)n < sizeof(qn)) ? pr_node(c, qn, f->path) : NULL; + pr_res_t r = pr_mk(node ? PR_NODE : PR_GAP); + r.node = node; + return r; +} + +/* True when the type the doc sits in declares or inherits a method named + * like the reference. */ +static bool pr_type_has_method(const pr_ctx_t *c, const pr_ref_t *r) { + if (c->cur < 0) { + return false; + } + pr_search_t s = { + .c = c, .want = 'm', .name = r->cls, .len = r->cls_len, .owner = CBM_NOT_FOUND}; + return pr_search(&s, c->cur, 0); +} + +static pr_res_t pr_function(const pr_ctx_t *c, const pr_ref_t *r) { + char fqn[PR_NAME_MAX]; + if (r->cls[0] == '\\' || memchr(r->cls, '\\', r->cls_len)) { + /* qualified: the type rules (a namespace alias may start it) */ + size_t fl = cbm_doclink_php_qualify(fqn, sizeof(fqn), c->ns, c->ns_len, c->uses, c->nuses, + r->cls, r->cls_len); + pr_res_t res = fl ? pr_find_func(c, fqn, fl) : pr_mk(PR_MISSING); + if (res.kind == PR_NONE) { + res = pr_absent(c, fqn, fl); + } + res.exact = r->cls[0] == '\\' || pr_aliased(c, r->cls, r->cls_len); + return res; + } + for (int i = 0; i < c->nuses; i++) { + const cbm_doclink_php_use_t *u = &c->uses[i]; + if (u->kind == 'f' && pr_eq_nocase(u->alias, u->alias_len, r->cls, r->cls_len)) { + pr_res_t res = pr_find_func(c, u->name, u->name_len); + if (res.kind == PR_NONE) { + res = pr_absent(c, u->name, u->name_len); + } + res.exact = true; + return res; + } + } + pr_res_t res = pr_mk(PR_NONE); + if (c->ns_len) { + int n = snprintf(fqn, sizeof(fqn), "%.*s\\%.*s", (int)c->ns_len, c->ns, (int)r->cls_len, + r->cls); + res = (n > 0 && (size_t)n < sizeof(fqn)) ? pr_find_func(c, fqn, (size_t)n) : res; + } + if (res.kind == PR_NONE) { + res = pr_find_func(c, r->cls, r->cls_len); /* PHP's global fallback */ + } + if (res.kind == PR_NONE) { + return pr_mk(PR_EXTERNAL); /* a built-in function */ + } + if ((res.kind == PR_NODE || res.kind == PR_GAP) && pr_type_has_method(c, r)) { + /* written in a type that has a method of that name: the method or + * this function, and the reference does not pick one */ + return pr_mk(PR_AMBIGUOUS); + } + return res; +} + +/* ── Outcomes ────────────────────────────────────────────────────── */ + +static void pr_out(const pr_ctx_t *c, pr_res_t r, cbm_doclink_outcome_t *out) { + switch (r.kind) { + case PR_NODE: { + const pr_file_t *src = &c->ix->files[c->src]; + void *v = r.node->file_path ? cbm_ht_get(c->ix->by_path, r.node->file_path) : NULL; + int tf = v ? (int)((uintptr_t)v - SKIP_ONE) : CBM_NOT_FOUND; + if (tf >= 0 && c->ix->files[tf].test && !src->test) { + out->kind = CBM_DOCLINK_UNRESOLVED; + out->reason = CBM_DOCLINK_REASON_TEST_ONLY; + return; + } + out->kind = CBM_DOCLINK_EDGE; + out->target = r.node; + out->exact = r.exact; + return; + } + case PR_EXTERNAL: + out->reason = CBM_DOCLINK_REASON_EXTERNAL; + break; + case PR_AMBIGUOUS: + out->reason = CBM_DOCLINK_REASON_AMBIGUOUS; + break; + case PR_GAP: + out->reason = CBM_DOCLINK_REASON_GRAPH_GAP; + break; + default: + out->reason = CBM_DOCLINK_REASON_MISSING; + break; + } + out->kind = CBM_DOCLINK_UNRESOLVED; +} + +static void pr_resolve(const void *index, void *state, int run_file, const CBMDocLink *link, + const cbm_gbuf_t *graph, cbm_doclink_outcome_t *out) { + (void)state; /* PHP keeps no per-file working state */ + const pr_index_t *ix = (const pr_index_t *)index; + out->kind = CBM_DOCLINK_UNRESOLVED; + out->reason = CBM_DOCLINK_REASON_MISSING; + out->target = NULL; + out->exact = false; + if (!ix || !link->raw || run_file < 0 || run_file >= ix->run_count || + ix->run_to_file[run_file] < 0) { + return; + } + pr_ctx_t c = {.ix = ix, .g = graph, .src = ix->run_to_file[run_file]}; + const pr_file_t *f = &ix->files[c.src]; + size_t rl = strlen(link->raw); + pr_ref_t r; + if (rl > CBM_DOCLINK_PHP_REF_MAX || !pr_parse(link->raw, rl, &r)) { + out->reason = CBM_DOCLINK_REASON_UNPARSEABLE; + return; + } + pr_block_at(&c, f, link->line); + c.cur = pr_type_at(ix, f, link->def_line); + pr_res_t res; + if (!r.mem && r.call) { + res = pr_function(&c, &r); + } else if (!r.mem) { + res = pr_type_of(&c, &r); + if (res.kind == PR_TYPE) { + bool exact = res.exact; + res = pr_type_node(&c, res.type); + res.exact = exact; + } + } else { + res = pr_type_of(&c, &r); + if (res.kind == PR_TYPE) { + bool exact = res.exact; + res = pr_member(&c, res.type, &r); + res.exact = exact; + } + } + pr_out(&c, res, out); +} + +const cbm_doclink_resolver_t cbm_doclink_php_resolver = { + .langs = {CBM_LANG_PHP}, + .lang_count = SKIP_ONE, + .scope_tag = CBM_DOCLINK_PHP_SCOPE_TAG, + .build = pr_build, + .destroy = pr_destroy, + .resolve = pr_resolve, +}; diff --git a/tests/test_doc_mentions_helpers.h b/tests/test_doc_mentions_helpers.h index b6d502ddb..eb32af893 100644 --- a/tests/test_doc_mentions_helpers.h +++ b/tests/test_doc_mentions_helpers.h @@ -441,7 +441,7 @@ static const int DM_HELD_FAMILIES[] = { CBM_DOCLINK_KT_SAMPLE, CBM_DOCLINK_RS_INTRA_DOC, CBM_DOCLINK_RS_LINK_DEF, CBM_DOCLINK_PERL_POD_L, CBM_DOCLINK_TS_LINK, CBM_DOCLINK_TS_LINKCODE, - CBM_DOCLINK_TS_SEE, + CBM_DOCLINK_TS_SEE, CBM_DOCLINK_PHP_SEE, }; static inline void dm_ship_held_families(void) { diff --git a/tests/test_doc_mentions_php.c b/tests/test_doc_mentions_php.c new file mode 100644 index 000000000..c4342d90d --- /dev/null +++ b/tests/test_doc_mentions_php.c @@ -0,0 +1,464 @@ +/* + * test_doc_mentions_php.c -- PHP doc-comment references (PHPDoc @see and + * {@see}). + */ +#include "../src/foundation/compat.h" +#include "test_framework.h" +#include "test_helpers.h" +#include "test_doc_mentions_helpers.h" + +#include "cbm.h" +#include "doclink.h" + +#include +#include +#include + +/* ── extraction ──────────────────────────────────────────────────── */ + +static const char *const PHP_USER = + "line != line || t->syntax != syntax) { + printf(" token %s: line %u syntax %d\n", raw, t->line, t->syntax); + return false; + } + return true; +} + +TEST(doc_mentions_php_tokens) { + CBMFileResult *r = dm_extract(PHP_USER, CBM_LANG_PHP, "src/Model/User.php"); + ASSERT_NOT_NULL(r); + ASSERT_TRUE(php_token_is(r, "H::format()", 9, CBM_DOCLINK_PHP_SEE)); + ASSERT_TRUE(php_token_is(r, "Strings", 9, CBM_DOCLINK_PHP_SEE)); + /* sentence punctuation is dropped; a description may follow */ + ASSERT_TRUE(php_token_is(r, "Num::MAX", 14, CBM_DOCLINK_PHP_SEE)); + ASSERT_TRUE(php_token_is(r, "\\App\\Util\\Helper::$cache", 15, CBM_DOCLINK_PHP_SEE)); + ASSERT_TRUE(php_token_is(r, "clamp()", 16, CBM_DOCLINK_PHP_SEE)); + ASSERT_TRUE(php_token_is(r, "https://example.com/doc", 17, CBM_DOCLINK_HREF)); + /* an inline tag in an @see line is the tag's, once */ + ASSERT_TRUE(php_token_is(r, "Inline", 18, CBM_DOCLINK_PHP_SEE)); + ASSERT_TRUE(php_token_is(r, "parent::boot()", 26, CBM_DOCLINK_PHP_SEE)); + /* a code span, {@seealso}, a fenced block, @seealso: no token */ + ASSERT_EQ(dm_count_tokens(r, "Hidden"), 0); + ASSERT_EQ(dm_count_tokens(r, "Also"), 0); + ASSERT_EQ(dm_count_tokens(r, "Fenced"), 0); + ASSERT_EQ(dm_count_tokens(r, "Nope"), 0); + ASSERT_EQ(r->doc_links.count, 8); + cbm_free_result(r); + PASS(); +} + +static bool php_has_line(const char *blob, const char *line) { + size_t n = strlen(line); + for (const char *p = blob; p && *p;) { + const char *nl = strchr(p, '\n'); + size_t len = nl ? (size_t)(nl - p) : strlen(p); + if (len == n && memcmp(p, line, n) == 0) { + return true; + } + p = nl ? nl + 1 : NULL; + } + printf(" no line: %s\n", line); + return false; +} + +TEST(doc_mentions_php_scope_blob) { + CBMFileResult *r = dm_extract(PHP_USER, CBM_LANG_PHP, "src/Model/User.php"); + ASSERT_NOT_NULL(r); + const char *s = r->doc_scope; + ASSERT_NOT_NULL(s); + ASSERT(strncmp(s, "php1\n", 5) == 0); + ASSERT(php_has_line(s, "N\t0\tApp\\Model\t2\t35")); + ASSERT(php_has_line(s, "U\t0\tc\tH\tApp\\Util\\Helper")); + ASSERT(php_has_line(s, "U\t0\tc\tStrings\tApp\\Util\\Strings")); + ASSERT(php_has_line(s, "U\t0\tc\tNum\tApp\\Util\\Numbers")); + ASSERT(php_has_line(s, "U\t0\tf\ttwice\tApp\\Util\\twice")); + ASSERT(php_has_line(s, "U\t0\tf\tclamp\tApp\\Util\\clamp")); + /* supertypes qualified as PHP resolves them in the file */ + ASSERT(php_has_line(s, "T\tc\tApp\\Model\\User\t-\t21\t29")); + ASSERT(php_has_line(s, "B\te\tApp\\Model\\Base")); + ASSERT(php_has_line(s, "B\ti\tCountable")); + ASSERT(php_has_line(s, "B\ti\tApp\\Model\\Shape")); + ASSERT(php_has_line(s, "B\tt\tApp\\Model\\Named")); + ASSERT(php_has_line(s, "B\tt\tApp\\Util\\Loud")); + ASSERT(php_has_line(s, "K\tc\tLIMIT")); + ASSERT(php_has_line(s, "K\tp\tname")); + ASSERT(php_has_line(s, "K\tp\talias")); + ASSERT(php_has_line(s, "K\tm\tcount")); + ASSERT(php_has_line(s, "K\tp\tage")); + ASSERT(php_has_line(s, "T\ti\tApp\\Model\\Shape\t-\t31\t31")); + ASSERT(php_has_line(s, "B\ti\tJsonSerializable")); + ASSERT(php_has_line(s, "T\tt\tApp\\Model\\Named\t-\t32\t32")); + ASSERT(php_has_line(s, "K\tm\tlabel")); + ASSERT(php_has_line(s, "T\te\tApp\\Model\\Suit\t-\t33\t33")); + ASSERT(php_has_line(s, "K\te\tHearts")); + ASSERT(php_has_line(s, "F\tApp\\Model\\make_user\t-")); + /* the persisted blob keeps no line and no import */ + char *p = cbm_doclink_php_portable_scope(s); + ASSERT_NOT_NULL(p); + ASSERT(php_has_line(p, "N\t0\tApp\\Model")); + ASSERT(php_has_line(p, "T\tc\tApp\\Model\\User\t-")); + ASSERT(php_has_line(p, "B\te\tApp\\Model\\Base")); + ASSERT(strstr(p, "\nU\t") == NULL); + cbm_free(CBM_MEM_CLASS_OTHER, p); + cbm_free_result(r); + PASS(); +} + +/* Two namespace blocks of one file declare the same names: the graph keeps + * one node for each pair, so the scope flags them. */ +TEST(doc_mentions_php_scope_blocks) { + CBMFileResult *r = dm_extract("doc_scope; + ASSERT_NOT_NULL(s); + ASSERT(php_has_line(s, "N\t0\tA\\One\t2\t6")); + ASSERT(php_has_line(s, "N\t1\tA\\Two\t7\t14")); + ASSERT(php_has_line(s, "U\t0\tc\tY\tX\\Y")); + ASSERT(php_has_line(s, "T\tc\tA\\One\\Same\td\t4\t4")); + ASSERT(php_has_line(s, "B\te\tX\\Y")); + ASSERT(php_has_line(s, "T\tc\tA\\Two\\Same\td\t8\t8")); + ASSERT(php_has_line(s, "F\tA\\One\\same\td")); + ASSERT(php_has_line(s, "F\tA\\Two\\same\td")); + ASSERT(php_has_line(s, "F\tA\\Two\\lone\t-")); + /* a conditional declaration */ + ASSERT(php_has_line(s, "F\tA\\Two\\cond\t-")); + cbm_free_result(r); + PASS(); +} + +/* ── resolution ──────────────────────────────────────────────────── */ + +static void php_write_repo(const char *repo) { + th_write_file(TH_PATH(repo, "src/Util/Helper.php"), + " %s: %d found\n", src, tgt, n); + return false; + } + return true; +} + +static bool php_row(const char *db, const char *raw, const char *reason) { + char got[64]; + dm_row(db, "src/Model/User.php", raw, got, sizeof(got), NULL, 0); + if (strcmp(got, reason) != 0) { + printf(" row [%s]: \"%s\", expected \"%s\"\n", raw, got, reason); + return false; + } + return true; +} + +/* A repository indexed once for the resolution tests. */ +static char g_php_tmp[256]; +static char g_php_db[512]; + +static bool php_indexed(void) { + if (g_php_db[0]) { + return true; + } + snprintf(g_php_tmp, sizeof(g_php_tmp), "/tmp/cbm_dlphp_XXXXXX"); + if (!cbm_mkdtemp(g_php_tmp)) { + return false; + } + char repo[512]; + snprintf(repo, sizeof(repo), "%s/repo", g_php_tmp); + snprintf(g_php_db, sizeof(g_php_db), "%s/g.db", g_php_tmp); + php_write_repo(repo); + return dm_index(repo, g_php_db, NULL) == 0; +} + +TEST(doc_mentions_php_resolution) { + ASSERT_TRUE(php_indexed()); + const char *db = g_php_db; + const char *u = "src.Model.User.User"; + /* use imports: an alias, a group member, a function import */ + ASSERT_TRUE(php_edge(db, u, "src.Util.Helper.Helper.format")); + ASSERT_TRUE(php_edge(db, u, "src.Util.Helper.Strings")); + ASSERT_TRUE(php_edge(db, u, "src.Util.Helper.clamp")); + /* the current namespace; a fully qualified global type */ + ASSERT_TRUE(php_edge(db, u, "src.Model.Base.Base.boot")); + ASSERT_TRUE(php_edge(db, u, "src.global.GlobalThing")); + /* a member a used trait declares */ + ASSERT_TRUE(php_edge(db, u, "src.Util.Helper.Loud.shout")); + /* constants, properties and enum cases have no node */ + ASSERT_TRUE(php_row(db, "Num::MAX", "graph_gap")); + ASSERT_TRUE(php_row(db, "H::$cache", "graph_gap")); + ASSERT_TRUE(php_row(db, "Suit::Hearts", "graph_gap")); + /* an unqualified type is the namespace's: no global fallback */ + ASSERT_TRUE(php_row(db, "GlobalThing", "missing")); + ASSERT_TRUE(php_row(db, "App\\Util\\Strings", "missing")); + ASSERT_TRUE(php_row(db, "Nope", "missing")); + /* a dependency, a built-in class, a built-in function */ + ASSERT_TRUE(php_row(db, "LoggerInterface", "external")); + ASSERT_TRUE(php_row(db, "\\Exception", "external")); + ASSERT_TRUE(php_row(db, "strlen()", "external")); + /* every supertype in the repository: missing */ + ASSERT_TRUE(php_row(db, "Base::nope()", "missing")); + /* two files declare it; one file declares it twice (one node) */ + ASSERT_TRUE(php_row(db, "\\App\\Twin\\Twin", "ambiguous")); + ASSERT_TRUE(php_row(db, "\\App\\One\\Same", "graph_gap")); + ASSERT_TRUE(php_row(db, "\\App\\One\\same()", "graph_gap")); + /* aliases and method names compare without case */ + ASSERT_TRUE(php_edge(db, "src.Model.User.User.other", "src.Util.Helper.Helper.format")); + PASS(); +} + +/* PHP's global fallback for an unqualified function, beyond phpDocumentor's + * resolver: the namespace's own function first, then the global one. */ +TEST(doc_mentions_php_global_fallback) { + ASSERT_TRUE(php_indexed()); + const char *db = g_php_db; + const char *o = "src.Model.User.User.other"; + ASSERT_TRUE(php_edge(db, o, "src.Model.User.shadowed")); + ASSERT_TRUE(php_edge(db, o, "src.global.global_helper")); + ASSERT_TRUE(php_edge(db, o, "src.Model.User.helper_here")); + /* a function named like a method of the documented type: two readings; + * no such function: PHP's built-ins */ + ASSERT_TRUE(php_row(db, "other()", "ambiguous")); + ASSERT_TRUE(php_row(db, "size()", "external")); + char props[512]; + int n = 0; + dm_edge(db, o, "src.global.shadowed", props, sizeof(props), &n); + ASSERT_EQ(n, 0); + PASS(); +} + +/* self, static and parent, beyond phpDocumentor's resolver: the type the doc + * sits in and its superclass. */ +TEST(doc_mentions_php_self_parent) { + ASSERT_TRUE(php_indexed()); + const char *db = g_php_db; + const char *c = "src.Model.User.User.count"; + ASSERT_TRUE(php_edge(db, c, "src.Model.User.User.size")); + ASSERT_TRUE(php_edge(db, c, "src.Model.Base.Base.boot")); + /* a bare member: no constant of that name, so a method, without case */ + ASSERT_TRUE(php_edge(db, c, "src.Model.Base.Base.mixedCase")); + ASSERT_TRUE(php_row(db, "static::LIMIT", "graph_gap")); + ASSERT_TRUE(php_row(db, "self::$name", "graph_gap")); + ASSERT_TRUE(php_row(db, "parent::gone()", "missing")); + PASS(); +} + +static void php_indexed_cleanup(void) { + if (g_php_db[0]) { + th_cleanup(g_php_tmp); + g_php_db[0] = '\0'; + } +} + +/* ── incremental == full; one worker == several ──────────────────── */ + +TEST(doc_mentions_php_incremental_equals_full) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlphp_inc_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[512]; + char inc_db[512]; + char full_db[512]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + snprintf(inc_db, sizeof(inc_db), "%s/inc.db", tmp); + snprintf(full_db, sizeof(full_db), "%s/full.db", tmp); + th_write_file(TH_PATH(repo, "src/lib.php"), + " Date: Sat, 10 Oct 2026 18:07:27 +0200 Subject: [PATCH 08/17] doc-links(c): Doxygen @ref and @see names become MENTIONS edges, held until their audit A Doxygen comment in a .c or .h file (a block opening with slash-star-star or slash-star-bang, a run of /// or //! lines) names code with `@ref name` or `\ref name` (family doxygen_ref) and with the names an `@see`, `\see`, `@sa` or `\sa` paragraph lists that have the shape of a symbol (`name`, `name()`, `::name`, `Type::member`; family see). Code blocks, fences, code spans, quoted labels and the word after \c, \p, \a, \b, \e, \em hold none; a URL is the core's href family. A .h is parsed as C++, which does not change the rule; other C++ and Objective-C files are not part of the leg. C keeps no node for a function prototype, so the docs of a header's API were never stored anywhere. The doc-link driver gets an optional loose_docs hook, called after the definitions and the file's own doc: the C leg reads every Doxygen block outside function bodies that no definition took, and its references belong to the file (the File node is the source). The scope ("c1") lists the definitions with nodes (function, variable, type, enum constant, field, object-like and function-like macro) with their qualified names, their linkage (static, or a type, constant or macro of a .c file, is the file's own) and the parent of a constant or field, and the pages, groups, anchors and sections the file's docs define. doc_links_c.c resolves a name as Doxygen does, without imports: - a page, group, anchor or section name is no code entity (graph_gap) and never binds a symbol of the same name; - then the documented file's own definitions (static ones too), its fields, then what every file sees (external linkage, a header's types, constants and macros); a static definition of another file is not visible; - candidates count by qualified name (a prototype and its definition, #if variants: one node); with `()` written a function or function-like macro wins, then a definition wins over a macro; - `Type::member`: a field of the struct or union Type, or a constant of the enum Type; - one candidate binds, several are ambiguous; a word of an @see paragraph that names nothing and is a plain lowercase word is prose (Doxygen prints it as text): no row; test code binds only from test code. There is no scope_delta hook yet: a declaration or linkage change is a full run. The families are held (ships=false) until their held-out audit passes; the suite ships them through the test seam. Proof (ship build, Apple M5 Pro / 48 GB, base 7a321cac): - xxhash (redis 4f20cb489344, a clean copy), the field test's 174 audited links, 110 of which had hit a rename macro before C1: every harvested one now names the right entity (169 edges, 2 self-mentions: XXH_NAMESPACE's own doc); the other 3 are @copydoc, which this leg does not harvest. - redis: doxygen_ref 74 edges (179 mentions), see 25 (30); rows: graph_gap 50, missing 6, ambiguous 3, unparseable 1. The prose rule took the 75 rows of words like "for" and "details" out. - pico-sdk 079c6f39: doxygen_ref 488 (730), see 199 (219); rows: graph_gap 142 (groups and pages), missing 94, ambiguous 68 (one type per chip, rp2040 and rp2350), test_only_target 28 (assembly functions with only a C test version), unparseable 5. - libusb abba63ed: doxygen_ref 256 (284), see 39 (44); rows: graph_gap 62, missing 18, unparseable 2, ambiguous 2. 46 of the edges are Type::member references. - libevent d82464a2: see 176 (230); rows: missing 9. - a random sample of 24 edges (8 per corpus, pico-sdk, libusb, libevent) read against their source lines: all correct, among them a header prototype's doc linking to the definition in the .c file. Finding outside the leg: libevent's event_assign has no node (its return type and name are split by an #if block), so its references are missing. Tests (doc_mentions_c): tokens with their lines and the file-level source of a prototype's doc; C++ files out, headers in; the scope blob; every resolution rule; incremental equals full (a doc edit and a body edit repair one file, a definition made static is a full run); 60 modules give the same edges and rows with one worker and several. Each rule red with it reverted. Signed-off-by: Martin Vogel --- Makefile.cbm | 4 +- internal/cbm/doclink.c | 92 ++-- internal/cbm/doclink.h | 27 ++ internal/cbm/doclink_c.c | 760 ++++++++++++++++++++++++++++++ src/pipeline/doc_links.c | 2 +- src/pipeline/doc_links.h | 1 + src/pipeline/doc_links_c.c | 481 +++++++++++++++++++ tests/test_doc_mentions_c.c | 341 ++++++++++++++ tests/test_doc_mentions_helpers.h | 1 + tests/test_main.c | 2 + 10 files changed, 1680 insertions(+), 31 deletions(-) create mode 100644 internal/cbm/doclink_c.c create mode 100644 src/pipeline/doc_links_c.c create mode 100644 tests/test_doc_mentions_c.c diff --git a/Makefile.cbm b/Makefile.cbm index 4cfb47783..ffb0cb0e7 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -392,6 +392,7 @@ EXTRACTION_SRCS = \ $(CBM_DIR)/doclink_perl.c \ $(CBM_DIR)/doclink_ts.c \ $(CBM_DIR)/doclink_php.c \ + $(CBM_DIR)/doclink_c.c \ $(CBM_DIR)/sql_values.c \ $(CBM_DIR)/helpers.c \ $(CBM_DIR)/cpp_branch_views.c \ @@ -524,6 +525,7 @@ PIPELINE_SRCS = \ src/pipeline/doc_links_perl.c \ src/pipeline/doc_links_ts.c \ src/pipeline/doc_links_php.c \ + src/pipeline/doc_links_c.c \ src/pipeline/doc_links_msbuild.c \ src/pipeline/pass_route_nodes.c \ src/pipeline/pass_enrichment.c \ @@ -786,7 +788,7 @@ TEST_DISCOVER_SRCS = \ TEST_GRAPH_BUFFER_SRCS = tests/test_graph_buffer.c -TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_pipeline_conventions.c tests/test_pipeline_frozen.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_doclinks.c tests/test_doc_mentions.c tests/test_doc_links_md.c tests/test_doc_links_pdf.c tests/test_doc_links_rst.c tests/test_doc_links_adoc.c tests/test_doc_mentions_go.c tests/test_doc_mentions_jvm.c tests/test_doc_mentions_rs.c tests/test_doc_mentions_perl.c tests/test_doc_mentions_ts.c tests/test_doc_mentions_php.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c +TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_pipeline_conventions.c tests/test_pipeline_frozen.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_doclinks.c tests/test_doc_mentions.c tests/test_doc_links_md.c tests/test_doc_links_pdf.c tests/test_doc_links_rst.c tests/test_doc_links_adoc.c tests/test_doc_mentions_go.c tests/test_doc_mentions_jvm.c tests/test_doc_mentions_rs.c tests/test_doc_mentions_perl.c tests/test_doc_mentions_ts.c tests/test_doc_mentions_php.c tests/test_doc_mentions_c.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c TEST_WATCHER_SRCS = tests/test_watcher.c diff --git a/internal/cbm/doclink.c b/internal/cbm/doclink.c index cb65126b1..18a65cef1 100644 --- a/internal/cbm/doclink.c +++ b/internal/cbm/doclink.c @@ -152,6 +152,8 @@ typedef struct { /* A document language: the whole file is the documentation, so one scan * of its bytes replaces parse_doc, the file-level doc and the scope. */ void (*scan_file)(CBMExtractCtx *ctx); + void (*loose_docs)(CBMExtractCtx *ctx, const CBMDefinition *file_def, + cbm_doclink_taken_fn taken); } doclink_lang_t; static const doclink_lang_t DOCLINK_LANGS[] = { @@ -234,6 +236,17 @@ static const doclink_lang_t DOCLINK_LANGS[] = { .scan_scope = cbm_doclink_ts_scan_scope, .scope_tag = CBM_DOCLINK_TS_SCOPE_TAG, .portable_scope = cbm_doclink_ts_portable_scope}, + /* C, and the .h files C++ parses (the hooks skip every other file) */ + {.lang = CBM_LANG_C, + .parse_doc = cbm_doclink_c_parse_doc, + .scan_scope = cbm_doclink_c_scan_scope, + .scope_tag = CBM_DOCLINK_C_SCOPE_TAG, + .loose_docs = cbm_doclink_c_loose_docs}, + {.lang = CBM_LANG_CPP, + .parse_doc = cbm_doclink_c_parse_doc, + .scan_scope = cbm_doclink_c_scan_scope, + .scope_tag = CBM_DOCLINK_C_SCOPE_TAG, + .loose_docs = cbm_doclink_c_loose_docs}, {.lang = CBM_LANG_PHP, .parse_doc = cbm_doclink_php_parse_doc, .scan_scope = cbm_doclink_php_scan_scope, @@ -359,12 +372,6 @@ static doc_line_ent_t *doc_info_of(CBMExtractCtx *ctx, const char *doc) { return NULL; } -/* A doc not built by doc_run_text keeps the definition's own line. */ -static uint32_t doc_line_of(CBMExtractCtx *ctx, const char *doc) { - const doc_line_ent_t *info = doc_info_of(ctx, doc); - return info ? info->line : 0; -} - /* ── Persisted scope ─────────────────────────────────────────────── */ char *cbm_doclink_portable_scope(const char *scope) { @@ -461,6 +468,55 @@ static doclink_twin_key_t *twin_keys(CBMExtractCtx *ctx, const char *label, int return keys; } +/* True when a doc taken from a definition, or the file's own doc, starts on + * one of the lines [first, last]. */ +static bool doclink_taken(const CBMExtractCtx *ctx, uint32_t first, uint32_t last) { + const doc_line_map_t *m = (const doc_line_map_t *)ctx->doc_lines; + for (int i = 0; m && i < m->count; i++) { + if (m->items[i].parsed && m->items[i].line >= first && m->items[i].line <= last) { + return true; + } + } + return false; +} + +/* The file's own doc, then the language's loose docs: their references belong + * to the file. The parser gets a definition-shaped stand-in for the file; what + * it pushes is marked, and the resolving half takes the File node as the + * source. No qualified name for that node is computed on this side. */ +static void doclink_file_pass(CBMExtractCtx *ctx, const doclink_lang_t *L) { + const char *file_doc = ctx->result->module_doc; + bool has_doc = file_doc && file_doc[0]; + if (!has_doc && !L->loose_docs) { + return; + } + CBMDocLinkArray *arr = &ctx->result->doc_links; + int first = arr->count; + CBMDefinition file_def = { + .name = ctx->rel_path, + .qualified_name = ctx->module_qn ? ctx->module_qn : "", + .label = "File", + .file_path = ctx->rel_path, + .start_line = SKIP_ONE, + .end_line = SKIP_ONE, + .docstring = file_doc, + }; + if (has_doc) { + doc_line_ent_t *info = doc_info_of(ctx, file_doc); + uint32_t doc_line = info ? info->line : 0; + L->parse_doc(ctx, &file_def, file_doc, doc_line ? doc_line : file_def.start_line); + if (info) { + info->parsed = true; + } + } + if (L->loose_docs) { + L->loose_docs(ctx, &file_def, doclink_taken); + } + for (int i = first; i < arr->count; i++) { + arr->items[i].flags |= CBM_DOCLINK_FLAG_FILE; + } +} + void cbm_doclinks_extract(CBMExtractCtx *ctx) { if (!ctx || !ctx->result) { return; @@ -520,29 +576,7 @@ void cbm_doclinks_extract(CBMExtractCtx *ctx) { info->parsed = true; } } - /* The file's own doc: its references belong to the file. The parser gets - * a definition-shaped stand-in for the file; what it pushes is marked, and - * the resolving half takes the File node as the source. No qualified name - * for that node is computed on this side. */ - const char *file_doc = ctx->result->module_doc; - if (file_doc && file_doc[0]) { - CBMDocLinkArray *arr = &ctx->result->doc_links; - int first = arr->count; - CBMDefinition file_def = { - .name = ctx->rel_path, - .qualified_name = ctx->module_qn ? ctx->module_qn : "", - .label = "File", - .file_path = ctx->rel_path, - .start_line = SKIP_ONE, - .end_line = SKIP_ONE, - .docstring = file_doc, - }; - uint32_t doc_line = doc_line_of(ctx, file_doc); - L->parse_doc(ctx, &file_def, file_doc, doc_line ? doc_line : file_def.start_line); - for (int i = first; i < arr->count; i++) { - arr->items[i].flags |= CBM_DOCLINK_FLAG_FILE; - } - } + doclink_file_pass(ctx, L); if (L->scan_scope) { ctx->result->doc_scope = L->scan_scope(ctx); } diff --git a/internal/cbm/doclink.h b/internal/cbm/doclink.h index 8e3313009..cce493488 100644 --- a/internal/cbm/doclink.h +++ b/internal/cbm/doclink.h @@ -22,6 +22,10 @@ * identical on full and incremental runs. Optional. * scope_tag, the blob's tag line and its persisted form (line * portable_scope numbers dropped). Optional. + * loose_docs after the definitions and the file's own doc: the + * file's other doc comments (C: a prototype's doc, + * which the graph keeps no node for); what it pushes + * belongs to the file. Optional. * scan_file instead of parse_doc, for a DOCUMENT language whose * text is the documentation (Markdown): one pass over * the file's bytes pushes every reference, with the @@ -106,6 +110,9 @@ X(CBM_DOCLINK_TS_SEE, CBM_LANG_TYPESCRIPT, "see", false, false, "MENTIONS") \ /* php (doclink_php.c): PHPDoc @see and {@see}; held until their audit passes */ \ X(CBM_DOCLINK_PHP_SEE, CBM_LANG_PHP, "see", false, false, "MENTIONS") \ + /* c (doclink_c.c): Doxygen @ref and @see in .c and .h files; held until their audit */ \ + X(CBM_DOCLINK_C_REF, CBM_LANG_C, "doxygen_ref", false, false, "MENTIONS") \ + X(CBM_DOCLINK_C_SEE, CBM_LANG_C, "see", false, false, "MENTIONS") \ /* markdown (doclink_md.c): what a document's text names explicitly */ \ X(CBM_DOCLINK_MD_LINK, CBM_LANG_MARKDOWN, "link", false, true, "MENTIONS") \ X(CBM_DOCLINK_MD_PATH, CBM_LANG_MARKDOWN, "path", false, true, "MENTIONS") \ @@ -591,6 +598,26 @@ size_t cbm_doclink_php_qualify(char *out, size_t cap, const char *ns, size_t ns_ const cbm_doclink_php_use_t *uses, int nuses, const char *ref, size_t n); +/* ── C (doclink_c.c) ───────────────────────────────────────────────── */ + +/* Asked by a loose_docs hook for a doc comment on lines [first, last]: true + * when a definition's doc or the file's own doc there was already taken. */ +typedef bool (*cbm_doclink_taken_fn)(const CBMExtractCtx *ctx, uint32_t first, uint32_t last); + +void cbm_doclink_c_parse_doc(CBMExtractCtx *ctx, const CBMDefinition *def, const char *doc, + uint32_t doc_line); +void cbm_doclink_c_loose_docs(CBMExtractCtx *ctx, const CBMDefinition *file_def, + cbm_doclink_taken_fn taken); +const char *cbm_doclink_c_scan_scope(CBMExtractCtx *ctx); + +/* C files and the headers C++ parses share one scope tag: one resolver + * indexes them. The blob holds no line: it is its own persisted form. */ +#define CBM_DOCLINK_C_SCOPE_TAG "c1" + +/* The longest reference a C token keeps; a longer one is cut just past this + * length and the resolver reports it as unparseable. */ +enum { CBM_DOCLINK_C_REF_MAX = 256 }; + /* ── Perl POD (doclink_perl.c) ─────────────────────────────────────── */ void cbm_doclink_perl_parse_doc(CBMExtractCtx *ctx, const CBMDefinition *def, const char *doc, diff --git a/internal/cbm/doclink_c.c b/internal/cbm/doclink_c.c new file mode 100644 index 000000000..fa8ad5c22 --- /dev/null +++ b/internal/cbm/doclink_c.c @@ -0,0 +1,760 @@ +/* + * doclink_c.c — C doc-comment references (Doxygen) and the C doc-link scope. + * + * Files. Only .c and .h files: a .h is parsed as C++, which does not change + * that; other C++ and Objective-C sources are not part of this leg. + * + * Docs. The four Doxygen comment styles: blocks opening with slash-star-star + * or slash-star-bang, and runs of lines opening with three slashes or with + * slash-slash-bang (a `<` after the opener marks a member's trailing doc). A + * definition's stored doc goes through cbm_doclink_c_parse_doc. Every other + * Doxygen block of the file outside function bodies (a prototype's doc in a + * header, which the graph keeps no node for; a group's text) goes through + * cbm_doclink_c_loose_docs, and its references belong to the file. + * + * Tokens: + * - `@ref name` or `\ref name`, an optional quoted label after it: family + * doxygen_ref; + * - the names listed after `@see`, `\see`, `@sa` or `\sa` (the paragraph up + * to an empty line or the next command at the start of a line) that have + * the shape of a symbol: `name`, `name()`, `::name`: family see. An + * explicit @ref inside such a paragraph is a doxygen_ref token. + * Text inside \code, \verbatim, \dot, \msc or \startuml blocks, a Markdown + * fence or a code span holds none, nor does the word after \c, \p, \a, \b, \e + * or \em. A URL is the core's href family. + * + * Scope blob, one record per line after the tag line "c1" (tab-separated): + * D kind link name qn parent a definition the graph has a node for: + * kind f (function), v (variable), t (type: typedef, + * struct, union or enum), e (enum constant), r (a + * struct or union field), m (object-like macro), M + * (function-like macro); link x (visible in every file: + * external linkage, or a type, constant or macro of a + * header) or s (this file only: static, a type, + * constant or macro of a .c file, a field); qn the + * node's qualified name; parent the enum of a constant + * or the struct of a field ("-" for none) + * A name a Doxygen page, group, anchor or section the file's + * docs define + * The blob holds no line, so it is its own persisted form. + */ +#include "doclink.h" + +#include "arena.h" +#include "helpers.h" +#include "foundation/constants.h" +#include "foundation/hash_table.h" +#include "foundation/mem_core.h" +#include "tree_sitter/api.h" + +#include +#include +#include +#include +#include + +enum { C_OPENER_LEN = 3, C_FENCE_LEN = 3, C_CMD_MAX = 32 }; + +/* True for a .c or .h file: the only files this leg reads. */ +static bool c_leg_file(const char *rel_path) { + size_t n = rel_path ? strlen(rel_path) : 0; + return n > PAIR_LEN && rel_path[n - PAIR_LEN] == '.' && + (rel_path[n - SKIP_ONE] == 'c' || rel_path[n - SKIP_ONE] == 'h'); +} + +/* ── Doc lines ───────────────────────────────────────────────────── */ + +typedef struct { + const char *p; + size_t n; + uint32_t line; +} c_line_t; + +static bool c_has(const char *p, size_t n, const char *lit) { + size_t l = strlen(lit); + return n >= l && memcmp(p, lit, l) == 0; +} + +/* One raw line of a doc comment without its comment marks. */ +static c_line_t c_strip_line(const char *s, const char *le, bool first, bool *block, + uint32_t line) { + while (s < le && (*s == ' ' || *s == '\t')) { + s++; + } + size_t n = (size_t)(le - s); + if (first && (c_has(s, n, "/**") || c_has(s, n, "/*!"))) { + *block = true; + s += C_OPENER_LEN; + } else if (!*block && (c_has(s, n, "///") || c_has(s, n, "//!"))) { + s += C_OPENER_LEN; + } else if (*block && s < le && *s == '*' && !(s + SKIP_ONE < le && s[SKIP_ONE] == '/')) { + s++; + } + if (s < le && *s == '<') { + s++; /* a member's trailing doc */ + } + const char *t = le; + while (t > s && (t[-SKIP_ONE] == ' ' || t[-SKIP_ONE] == '\t' || t[-SKIP_ONE] == '\r')) { + t--; + } + if (*block && t - s >= PAIR_LEN && t[-SKIP_ONE] == '/' && t[-PAIR_LEN] == '*') { + t -= PAIR_LEN; + while (t > s && (t[-SKIP_ONE] == ' ' || t[-SKIP_ONE] == '\t')) { + t--; + } + } + while (s < t && (*s == ' ' || *s == '\t')) { + s++; + } + return (c_line_t){s, (size_t)(t - s), line}; +} + +/* The doc's lines without their comment marks. */ +static int c_doc_lines(CBMArena *a, const char *doc, uint32_t doc_line, c_line_t **out) { + int cap = SKIP_ONE; + for (const char *p = doc; *p; p++) { + cap += *p == '\n'; + } + c_line_t *lines = (c_line_t *)cbm_arena_alloc(a, (size_t)cap * sizeof(c_line_t)); + if (!lines) { + return 0; + } + int n = 0; + bool block = false; + for (const char *p = doc; n < cap;) { + const char *le = strchr(p, '\n'); + if (!le) { + le = p + strlen(p); + } + lines[n] = c_strip_line(p, le, n == 0, &block, doc_line + (uint32_t)n); + n++; + if (!*le) { + break; + } + p = le + SKIP_ONE; + } + *out = lines; + return n; +} + +/* ── Tokens ──────────────────────────────────────────────────────── */ + +typedef struct { + CBMExtractCtx *ctx; + const CBMDefinition *def; + bool see; /* inside an @see paragraph */ + int verbatim; /* inside a code block: the index of its end command, else -1 */ + bool fence; +} c_emit_t; + +static bool c_ident_start(char c) { + return isalpha((unsigned char)c) || c == '_'; +} + +static bool c_ident_char(char c) { + return isalnum((unsigned char)c) || c == '_'; +} + +static bool c_is_url(const char *t, size_t n) { + size_t i = 0; + while (i < n && (isalnum((unsigned char)t[i]) || t[i] == '+' || t[i] == '.' || t[i] == '-')) { + i++; + } + if (i > 0 && i + CBM_SZ_3 <= n && memcmp(t + i, "://", CBM_SZ_3) == 0) { + return true; + } + return (n > CBM_SZ_4 && strncmp(t, "www.", CBM_SZ_4) == 0) || + (n > CBM_SZ_7 && strncmp(t, "mailto:", CBM_SZ_7) == 0); +} + +static void c_push(const c_emit_t *em, const char *ref, size_t n, int syntax, uint32_t line) { + if (n == 0) { + return; + } + if (n > CBM_DOCLINK_C_REF_MAX) { + n = CBM_DOCLINK_C_REF_MAX + SKIP_ONE; /* clipped: the resolver calls it unparseable */ + } + if (c_is_url(ref, n)) { + syntax = CBM_DOCLINK_HREF; + } + char *copy = cbm_arena_strndup(em->ctx->arena, ref, n); + if (!copy) { + return; + } + CBMDocLink link = { + .source_qn = em->def->qualified_name, + .raw = copy, + .line = line, + .def_line = em->def->start_line, + .syntax = (uint16_t)syntax, + }; + cbm_doclinks_push(&em->ctx->result->doc_links, em->ctx->arena, link); +} + +/* Mark the bytes of a line that sit in a code span or a quoted label. */ +static void c_mask(const c_line_t *ln, bool *mask) { + for (size_t i = 0; i < ln->n;) { + char q = ln->p[i]; + if (q != '`' && q != '"') { + i++; + continue; + } + size_t j = i + SKIP_ONE; + while (j < ln->n && ln->p[j] != q) { + j++; + } + if (j >= ln->n) { + i++; /* unpaired */ + continue; + } + for (size_t k = i; k <= j; k++) { + mask[k] = true; + } + i = j + SKIP_ONE; + } +} + +/* A command at p[i] (`\word` or `@word`, not inside a word or an address): + * the word's length, else 0. */ +static size_t c_command_at(const char *p, size_t i, size_t n) { + if ((p[i] != '\\' && p[i] != '@') || + (i > 0 && + (c_ident_char(p[i - SKIP_ONE]) || p[i - SKIP_ONE] == '\\' || p[i - SKIP_ONE] == '@'))) { + return 0; + } + size_t k = i + SKIP_ONE; + while (k < n && isalpha((unsigned char)p[k])) { + k++; + } + return k - i - SKIP_ONE; +} + +static bool c_word_is(const char *w, size_t n, const char *lit) { + return strlen(lit) == n && memcmp(w, lit, n) == 0; +} + +static bool c_inline_cmd(const char *w, size_t n) { + return c_word_is(w, n, "c") || c_word_is(w, n, "p") || c_word_is(w, n, "a") || + c_word_is(w, n, "b") || c_word_is(w, n, "e") || c_word_is(w, n, "em"); +} + +static const char *const C_BLOCK_OPEN[] = {"code", "verbatim", "dot", "msc", "startuml", NULL}; +static const char *const C_BLOCK_CLOSE[] = {"endcode", "endverbatim", "enddot", + "endmsc", "enduml", NULL}; + +static int c_block_index(const char *const *list, const char *w, size_t n) { + for (int i = 0; list[i]; i++) { + if (c_word_is(w, n, list[i])) { + return i; + } + } + return CBM_NOT_FOUND; +} + +/* The end of the next non-space position of p[j, n). */ +static size_t c_skip_space(const char *p, size_t j, size_t n) { + while (j < n && isspace((unsigned char)p[j])) { + j++; + } + return j; +} + +/* `@ref name`: the name after the command (sentence punctuation dropped, + * `()` kept); returns the position after it. */ +static size_t c_ref(const c_emit_t *em, const c_line_t *ln, size_t j, const bool *mask) { + j = c_skip_space(ln->p, j, ln->n); + size_t k = j; + while (k < ln->n && !isspace((unsigned char)ln->p[k]) && ln->p[k] != '"' && !mask[k]) { + k++; + } + size_t e = k; + while (e > j && strchr(".,;:", ln->p[e - SKIP_ONE])) { + e--; + } + if (e > j && ln->p[e - SKIP_ONE] == ')' && !(e - j >= PAIR_LEN && ln->p[e - PAIR_LEN] == '(')) { + e--; /* a closing parenthesis of the prose around it */ + } + c_push(em, ln->p + j, e - j, CBM_DOCLINK_C_REF, ln->line); + return k; +} + +/* A word of an @see paragraph: a token when it has the shape of a symbol. */ +static void c_see_word(const c_emit_t *em, const char *w, size_t n, uint32_t line) { + while (n > 0 && strchr(".,;:", w[n - SKIP_ONE])) { + n--; + } + if (c_is_url(w, n)) { + c_push(em, w, n, CBM_DOCLINK_HREF, line); + return; + } + size_t i = (n >= PAIR_LEN && w[0] == ':' && w[SKIP_ONE] == ':') ? PAIR_LEN : 0; + if (i >= n || !c_ident_start(w[i])) { + return; + } + while (i < n && c_ident_char(w[i])) { + i++; + } + if (i + PAIR_LEN == n && w[i] == '(' && w[i + SKIP_ONE] == ')') { + i = n; + } + if (i == n) { + c_push(em, w, n, CBM_DOCLINK_C_SEE, line); + } +} + +/* One line outside code blocks: commands, and the words of an @see + * paragraph. */ +static void c_line(c_emit_t *em, const c_line_t *ln, const bool *mask) { + size_t i = 0; + bool line_start = true; + while (i < ln->n) { + if (mask[i] || isspace((unsigned char)ln->p[i]) || ln->p[i] == ',') { + i++; + continue; + } + size_t cl = c_command_at(ln->p, i, ln->n); + if (cl > 0) { + const char *w = ln->p + i + SKIP_ONE; + size_t j = i + SKIP_ONE + cl; + if (c_word_is(w, cl, "ref")) { + i = c_ref(em, ln, j, mask); + } else if (c_word_is(w, cl, "see") || c_word_is(w, cl, "sa")) { + em->see = true; + i = j; + } else if (c_inline_cmd(w, cl)) { + j = c_skip_space(ln->p, j, ln->n); + while (j < ln->n && !isspace((unsigned char)ln->p[j])) { + j++; /* the word it formats is text */ + } + i = j; + } else { + int open = c_block_index(C_BLOCK_OPEN, w, cl); + if (open >= 0) { + em->verbatim = open; + return; + } + if (line_start) { + em->see = false; /* a new paragraph */ + } + i = j; + } + line_start = false; + continue; + } + size_t k = i; + while (k < ln->n && !isspace((unsigned char)ln->p[k]) && ln->p[k] != ',' && !mask[k] && + c_command_at(ln->p, k, ln->n) == 0) { + k++; + } + if (k == i) { + k = i + SKIP_ONE; + } else if (em->see) { + c_see_word(em, ln->p + i, k - i, ln->line); + } + line_start = false; + i = k; + } +} + +/* The end command of the open code block on this line, if any. */ +static bool c_closes_block(const c_line_t *ln, int open) { + for (size_t i = 0; i < ln->n; i++) { + size_t cl = c_command_at(ln->p, i, ln->n); + if (cl && c_word_is(ln->p + i + SKIP_ONE, cl, C_BLOCK_CLOSE[open])) { + return true; + } + } + return false; +} + +static void c_harvest(CBMExtractCtx *ctx, const CBMDefinition *def, const char *doc, + uint32_t doc_line) { + CBMArena *a = ctx->scratch ? ctx->scratch : ctx->arena; + c_line_t *lines = NULL; + int n = c_doc_lines(a, doc, doc_line, &lines); + c_emit_t em = {ctx, def, false, CBM_NOT_FOUND, false}; + for (int i = 0; i < n; i++) { + const c_line_t *ln = &lines[i]; + if (em.verbatim >= 0) { + if (c_closes_block(ln, em.verbatim)) { + em.verbatim = CBM_NOT_FOUND; + } + continue; + } + if (ln->n >= C_FENCE_LEN && + (memcmp(ln->p, "```", C_FENCE_LEN) == 0 || memcmp(ln->p, "~~~", C_FENCE_LEN) == 0)) { + em.fence = !em.fence; + continue; + } + if (em.fence) { + continue; + } + if (ln->n == 0) { + em.see = false; /* an empty line ends a paragraph */ + continue; + } + bool *mask = (bool *)cbm_arena_calloc(a, ln->n + SKIP_ONE); + if (!mask) { + return; + } + c_mask(ln, mask); + c_line(&em, ln, mask); + } +} + +void cbm_doclink_c_parse_doc(CBMExtractCtx *ctx, const CBMDefinition *def, const char *doc, + uint32_t doc_line) { + if (!ctx || !ctx->result || !def || !doc || !def->qualified_name || + !c_leg_file(ctx->rel_path)) { + return; + } + c_harvest(ctx, def, doc, doc_line); +} + +/* ── Loose docs ──────────────────────────────────────────────────── */ + +static bool c_kind(TSNode n, const char *k) { + return !ts_node_is_null(n) && strcmp(ts_node_type(n), k) == 0; +} + +/* A comment node's Doxygen style: 'b' a block, 'l' a line, 0 none. */ +static char c_doc_style(const char *src, TSNode n) { + uint32_t b = ts_node_start_byte(n); + uint32_t e = ts_node_end_byte(n); + const char *p = src + b; + size_t len = e - b; + if (c_has(p, len, "/**/") || c_has(p, len, "/***")) { + return 0; + } + if (c_has(p, len, "/**") || c_has(p, len, "/*!")) { + return 'b'; + } + if (c_has(p, len, "////")) { + return 0; + } + return (c_has(p, len, "///") || c_has(p, len, "//!")) ? 'l' : 0; +} + +typedef struct { + CBMExtractCtx *ctx; + cbm_doclink_taken_fn taken; + const CBMDefinition *file_def; + uint32_t start_byte; /* a pending run of line comments */ + uint32_t end_byte; + uint32_t first; + uint32_t last; + bool pending; +} c_loose_t; + +static void c_loose_emit(c_loose_t *lw, uint32_t sb, uint32_t eb, uint32_t first, uint32_t last) { + if (lw->taken(lw->ctx, first, last)) { + return; /* a definition's doc: harvested with the definition */ + } + CBMArena *a = lw->ctx->scratch ? lw->ctx->scratch : lw->ctx->arena; + char *doc = cbm_arena_strndup(a, lw->ctx->source + sb, eb - sb); + if (doc) { + c_harvest(lw->ctx, lw->file_def, doc, first); + } +} + +static void c_loose_flush(c_loose_t *lw) { + if (lw->pending) { + c_loose_emit(lw, lw->start_byte, lw->end_byte, lw->first, lw->last); + lw->pending = false; + } +} + +static void c_loose_comment(c_loose_t *lw, TSNode n) { + char style = c_doc_style(lw->ctx->source, n); + uint32_t row = ts_node_start_point(n).row + SKIP_ONE; + if (style == 'l' && lw->pending && row == lw->last + SKIP_ONE) { + lw->end_byte = ts_node_end_byte(n); /* the run goes on */ + lw->last = row; + return; + } + c_loose_flush(lw); + if (style == 'l') { + lw->pending = true; + lw->start_byte = ts_node_start_byte(n); + lw->end_byte = ts_node_end_byte(n); + lw->first = row; + lw->last = row; + } else if (style == 'b') { + c_loose_emit(lw, ts_node_start_byte(n), ts_node_end_byte(n), row, + ts_node_end_point(n).row + SKIP_ONE); + } +} + +void cbm_doclink_c_loose_docs(CBMExtractCtx *ctx, const CBMDefinition *file_def, + cbm_doclink_taken_fn taken) { + if (!ctx || !ctx->result || !file_def || !taken || ts_node_is_null(ctx->root) || + !c_leg_file(ctx->rel_path)) { + return; + } + c_loose_t lw = {.ctx = ctx, .taken = taken, .file_def = file_def}; + TSTreeCursor cur = ts_tree_cursor_new(ctx->root); + for (bool more = ts_tree_cursor_goto_first_child(&cur); more;) { + TSNode n = ts_tree_cursor_current_node(&cur); + if (c_kind(n, "comment")) { + c_loose_comment(&lw, n); + } else if (ts_node_is_named(n)) { + c_loose_flush(&lw); /* code between: the run ends */ + } + /* Doxygen reads no comment inside a function body */ + if (!c_kind(n, "compound_statement") && ts_tree_cursor_goto_first_child(&cur)) { + continue; + } + while (!ts_tree_cursor_goto_next_sibling(&cur)) { + if (!ts_tree_cursor_goto_parent(&cur)) { + more = false; + break; + } + } + } + ts_tree_cursor_delete(&cur); + c_loose_flush(&lw); +} + +/* ── Scope blob ──────────────────────────────────────────────────── */ + +typedef struct { + CBMArena *a; + char *buf; + size_t len; + size_t cap; + bool oom; +} c_sb_t; + +static void c_sb_put(c_sb_t *sb, const char *s, size_t n) { + if (sb->oom) { + return; + } + if (sb->len + n + SKIP_ONE > sb->cap) { + size_t ncap = sb->cap ? sb->cap * PAIR_LEN : CBM_SZ_256; + while (ncap < sb->len + n + SKIP_ONE) { + ncap *= PAIR_LEN; + } + char *grown = (char *)cbm_arena_alloc(sb->a, ncap); + if (!grown) { + sb->oom = true; + return; + } + if (sb->len) { + memcpy(grown, sb->buf, sb->len); + } + sb->buf = grown; + sb->cap = ncap; + } + memcpy(sb->buf + sb->len, s, n); + sb->len += n; + sb->buf[sb->len] = '\0'; +} + +static void c_sb_field(c_sb_t *sb, const char *s, size_t n) { + c_sb_put(sb, "\t", SKIP_ONE); + for (size_t i = 0; i < n; i++) { + char c = (s[i] == '\t' || s[i] == '\n' || s[i] == '\r') ? ' ' : s[i]; + c_sb_put(sb, &c, SKIP_ONE); + } +} + +/* The name a declarator declares (through pointers, arrays, functions and + * parentheses). */ +static TSNode c_declarator_name(TSNode d) { + for (int guard = 0; guard < CBM_SZ_32 && !ts_node_is_null(d); guard++) { + if (c_kind(d, "identifier") || c_kind(d, "type_identifier")) { + return d; + } + TSNode inner = ts_node_child_by_field_name(d, TS_FIELD("declarator")); + if (ts_node_is_null(inner)) { + inner = ts_node_named_child(d, 0); /* a parenthesized declarator */ + } + d = inner; + } + return (TSNode){0}; +} + +static bool c_is_static(const char *src, TSNode decl) { + uint32_t nc = ts_node_named_child_count(decl); + for (uint32_t i = 0; i < nc; i++) { + TSNode c = ts_node_named_child(decl, i); + if (!c_kind(c, "storage_class_specifier")) { + continue; + } + uint32_t b = ts_node_start_byte(c); + uint32_t e = ts_node_end_byte(c); + if (e - b == CBM_SZ_6 && memcmp(src + b, "static", CBM_SZ_6) == 0) { + return true; + } + } + return false; +} + +/* The names a static declaration or definition declares. */ +static void c_static_names(CBMExtractCtx *ctx, CBMHashTable *names, TSNode decl) { + if (!c_is_static(ctx->source, decl)) { + return; + } + TSTreeCursor cur = ts_tree_cursor_new(decl); + for (bool more = ts_tree_cursor_goto_first_child(&cur); more; + more = ts_tree_cursor_goto_next_sibling(&cur)) { + const char *field = ts_tree_cursor_current_field_name(&cur); + if (!field || strcmp(field, "declarator") != 0) { + continue; + } + TSNode name = c_declarator_name(ts_tree_cursor_current_node(&cur)); + if (ts_node_is_null(name)) { + continue; + } + uint32_t b = ts_node_start_byte(name); + char *key = cbm_arena_strndup(ctx->arena, ctx->source + b, ts_node_end_byte(name) - b); + if (key) { + cbm_ht_set(names, key, (void *)(uintptr_t)SKIP_ONE); + } + } + ts_tree_cursor_delete(&cur); +} + +/* The static names of the file: top-level declarations, also inside + * preprocessor conditionals and extern "C" blocks. */ +static void c_collect_static(CBMExtractCtx *ctx, CBMHashTable *names) { + TSTreeCursor cur = ts_tree_cursor_new(ctx->root); + for (bool more = ts_tree_cursor_goto_first_child(&cur); more;) { + TSNode n = ts_tree_cursor_current_node(&cur); + bool container = c_kind(n, "preproc_if") || c_kind(n, "preproc_ifdef") || + c_kind(n, "preproc_else") || c_kind(n, "preproc_elif") || + c_kind(n, "preproc_elifdef") || c_kind(n, "linkage_specification") || + c_kind(n, "declaration_list"); + if (c_kind(n, "function_definition") || c_kind(n, "declaration")) { + c_static_names(ctx, names, n); + } + if (container && ts_tree_cursor_goto_first_child(&cur)) { + continue; + } + while (!ts_tree_cursor_goto_next_sibling(&cur)) { + if (!ts_tree_cursor_goto_parent(&cur)) { + more = false; + break; + } + } + } + ts_tree_cursor_delete(&cur); +} + +static char c_def_kind(const CBMDefinition *d) { + const char *l = d->label; + if (strcmp(l, "Function") == 0) { + return 'f'; + } + if (strcmp(l, "Variable") == 0) { + return d->parent_class ? 'e' : 'v'; + } + if (strcmp(l, "Field") == 0) { + return 'r'; + } + if (strcmp(l, "Macro") == 0) { + return d->signature ? 'M' : 'm'; + } + if (strcmp(l, "Type") == 0 || strcmp(l, "Class") == 0 || strcmp(l, "Enum") == 0) { + return 't'; + } + return 0; +} + +/* `\page name`, `\defgroup name` ... in one doc text: A records. */ +static void c_anchors(c_sb_t *sb, const char *doc) { + static const char *const cmds[] = {"page", "defgroup", "addtogroup", "anchor", + "section", "subsection", "subsubsection", NULL}; + size_t n = strlen(doc); + for (size_t i = 0; i < n; i++) { + size_t cl = c_command_at(doc, i, n); + if (!cl) { + continue; + } + bool hit = false; + for (int k = 0; cmds[k] && !hit; k++) { + hit = c_word_is(doc + i + SKIP_ONE, cl, cmds[k]); + } + size_t j = i + SKIP_ONE + cl; + if (!hit || j >= n || (doc[j] != ' ' && doc[j] != '\t')) { + continue; + } + j = c_skip_space(doc, j, n); + size_t k = j; + while (k < n && (c_ident_char(doc[k]) || doc[k] == '.' || doc[k] == '-')) { + k++; + } + if (k > j) { + c_sb_put(sb, "\nA", PAIR_LEN); + c_sb_field(sb, doc + j, k - j); + } + i = k; + } +} + +/* The anchors every Doxygen comment of the file defines. */ +static void c_scope_anchors(CBMExtractCtx *ctx, c_sb_t *sb) { + TSTreeCursor cur = ts_tree_cursor_new(ctx->root); + for (bool more = ts_tree_cursor_goto_first_child(&cur); more;) { + TSNode n = ts_tree_cursor_current_node(&cur); + if (c_kind(n, "comment") && c_doc_style(ctx->source, n)) { + uint32_t b = ts_node_start_byte(n); + char *text = cbm_arena_strndup(sb->a, ctx->source + b, ts_node_end_byte(n) - b); + if (text) { + c_anchors(sb, text); + } + } + if (!c_kind(n, "compound_statement") && ts_tree_cursor_goto_first_child(&cur)) { + continue; + } + while (!ts_tree_cursor_goto_next_sibling(&cur)) { + if (!ts_tree_cursor_goto_parent(&cur)) { + more = false; + break; + } + } + } + ts_tree_cursor_delete(&cur); +} + +const char *cbm_doclink_c_scan_scope(CBMExtractCtx *ctx) { + if (!ctx || !ctx->result || ts_node_is_null(ctx->root) || !c_leg_file(ctx->rel_path)) { + return NULL; + } + c_sb_t sb = {.a = ctx->arena}; + c_sb_put(&sb, CBM_DOCLINK_C_SCOPE_TAG, strlen(CBM_DOCLINK_C_SCOPE_TAG)); + CBMHashTable *statics = cbm_ht_create_in(CBM_MEM_CLASS_OTHER, CBM_SZ_32); + CBMHashTable *seen = cbm_ht_create_in(CBM_MEM_CLASS_OTHER, CBM_SZ_64); + if (!statics || !seen) { + cbm_ht_free(statics); + cbm_ht_free(seen); + return NULL; + } + c_collect_static(ctx, statics); + size_t pl = strlen(ctx->rel_path); + bool header = ctx->rel_path[pl - SKIP_ONE] == 'h'; + const CBMDefArray *defs = &ctx->result->defs; + for (int i = 0; i < defs->count && !sb.oom; i++) { + const CBMDefinition *d = &defs->items[i]; + char kind = (d->label && d->name && d->qualified_name) ? c_def_kind(d) : 0; + if (!kind || cbm_ht_get(seen, d->qualified_name)) { + continue; /* one node per qualified name (#if variants) */ + } + cbm_ht_set(seen, d->qualified_name, (void *)(uintptr_t)SKIP_ONE); + bool object = kind == 'f' || kind == 'v'; + char link = object ? (cbm_ht_get(statics, d->name) ? 's' : 'x') + : (header && kind != 'r' ? 'x' : 's'); + c_sb_put(&sb, "\nD", PAIR_LEN); + c_sb_field(&sb, &kind, SKIP_ONE); + c_sb_field(&sb, &link, SKIP_ONE); + c_sb_field(&sb, d->name, strlen(d->name)); + c_sb_field(&sb, d->qualified_name, strlen(d->qualified_name)); + const char *parent = + (kind == 'e' || kind == 'r') && d->parent_class ? d->parent_class : "-"; + c_sb_field(&sb, parent, strlen(parent)); + } + cbm_ht_free(statics); + cbm_ht_free(seen); + c_scope_anchors(ctx, &sb); + return sb.oom ? NULL : sb.buf; +} diff --git a/src/pipeline/doc_links.c b/src/pipeline/doc_links.c index 802665a35..a71aefb5c 100644 --- a/src/pipeline/doc_links.c +++ b/src/pipeline/doc_links.c @@ -48,7 +48,7 @@ static const cbm_doclink_resolver_t *const DOCLINK_RESOLVERS[] = { &cbm_doclink_cs_resolver, &cbm_doclink_md_resolver, &cbm_doclink_pdf_resolver, &cbm_doclink_rst_resolver, &cbm_doclink_adoc_resolver, &cbm_doclink_go_resolver, &cbm_doclink_jvm_resolver, &cbm_doclink_rs_resolver, &cbm_doclink_perl_resolver, - &cbm_doclink_ts_resolver, &cbm_doclink_php_resolver, + &cbm_doclink_ts_resolver, &cbm_doclink_php_resolver, &cbm_doclink_c_resolver, }; enum { DOCLINK_RESOLVER_COUNT = sizeof(DOCLINK_RESOLVERS) / sizeof(DOCLINK_RESOLVERS[0]) }; diff --git a/src/pipeline/doc_links.h b/src/pipeline/doc_links.h index 11ab9ec34..9ae1e07c7 100644 --- a/src/pipeline/doc_links.h +++ b/src/pipeline/doc_links.h @@ -301,6 +301,7 @@ extern const cbm_doclink_resolver_t cbm_doclink_rs_resolver; extern const cbm_doclink_resolver_t cbm_doclink_perl_resolver; extern const cbm_doclink_resolver_t cbm_doclink_ts_resolver; extern const cbm_doclink_resolver_t cbm_doclink_php_resolver; +extern const cbm_doclink_resolver_t cbm_doclink_c_resolver; /* True when `rel_path` is a scope input of some language (scope_input). */ bool cbm_doclinks_is_scope_input(const char *rel_path); diff --git a/src/pipeline/doc_links_c.c b/src/pipeline/doc_links_c.c new file mode 100644 index 000000000..044d69852 --- /dev/null +++ b/src/pipeline/doc_links_c.c @@ -0,0 +1,481 @@ +/* + * doc_links_c.c — resolving C doc references (doclink_c.c): Doxygen @ref and + * @see names, with the lookup Doxygen itself has, which knows no imports. + * + * A name (`name`, `name()`, `::name`) is looked up + * 1. among the Doxygen pages, groups, anchors and sections of the + * repository's docs: such a name is no code entity (a graph gap), and + * never binds to a symbol of the same name; + * 2. among the definitions of the documented file, static ones included; + * 3. among the definitions every file sees: external linkage, and the types, + * enum constants and macros of headers. A static definition of another + * file is not visible. + * A field of the documented file (`@ref field`) comes after its other + * definitions and before what every file sees. `Type::member` names a field + * of the struct or union Type, or a constant of the enum Type, Type found by + * the same levels. + * The first level with a candidate decides. Its candidates are counted by + * qualified name: a prototype and its definition, and the #if variants of + * one definition, are one node, the one the graph keeps for that name. A + * definition (function, variable, type, enum constant) wins over a macro of + * the same name; with `()` written, a function or function-like macro wins + * over the others. One candidate binds (exact in the documented file, + * unique elsewhere), several are ambiguous, none is missing; but a word of an + * @see paragraph that names nothing and is a plain lowercase word (no `_`, + * digit, capital, `()` or `::`) is prose, which Doxygen prints as text: no + * row. Test code binds only from test code. + */ +#include "pipeline/doc_links.h" +#include "cbm.h" +#include "doclink.h" +#include "helpers.h" /* cbm_is_test_file */ +#include "foundation/arena.h" +#include "foundation/constants.h" +#include "foundation/hash_table.h" +#include "foundation/log.h" +#include "foundation/mem_core.h" +#include "graph_buffer/graph_buffer.h" + +#include +#include +#include +#include +#include +#include +#include + +enum { CR_FIELDS = 6, CR_NAME_MAX = 256, CR_CANDS = 64, CR_QN_MAX = 2048 }; + +/* ── The index ───────────────────────────────────────────────────── */ + +typedef struct { + int file; + char kind; /* f v t e r m M */ + bool global; + const char *name; + const char *qn; + const char *parent; /* the enum of a constant, the struct of a field; NULL */ + int next; /* the next definition of the same name, or -1 */ +} cr_def_t; + +typedef struct { + const char *path; + bool test; +} cr_file_t; + +typedef struct { + CBMArena arena; + cr_file_t *files; + int nfiles; + int *run_to_file; + int run_count; + cr_def_t *defs; + int ndefs; + int cap_defs; + CBMHashTable *by_name; /* name -> the first definition's index + 1 */ + CBMHashTable *anchors; /* a Doxygen page, group, anchor or section name */ + CBMHashTable *by_path; /* rel_path -> file index + 1 */ + bool oom; +} cr_index_t; + +static const char *cr_copy(cr_index_t *ix, const char *s, size_t n) { + char *c = cbm_arena_strndup(&ix->arena, s, n); + ix->oom = ix->oom || !c; + return c; +} + +static bool cr_grow(cr_index_t *ix) { + if (ix->ndefs < ix->cap_defs) { + return true; + } + int ncap = ix->cap_defs ? ix->cap_defs * PAIR_LEN : CBM_SZ_64; + cr_def_t *g = (cr_def_t *)cbm_arena_alloc(&ix->arena, (size_t)ncap * sizeof(cr_def_t)); + if (!g) { + ix->oom = true; + return false; + } + if (ix->ndefs) { + memcpy(g, ix->defs, (size_t)ix->ndefs * sizeof(cr_def_t)); + } + ix->defs = g; + ix->cap_defs = ncap; + return true; +} + +static void cr_read_record(cr_index_t *ix, int fi, const char **fl, const size_t *ln, int n) { + if (fl[0][0] == 'D' && n >= CR_FIELDS - SKIP_ONE && cr_grow(ix)) { /* parent optional */ + cr_def_t *d = &ix->defs[ix->ndefs++]; + d->file = fi; + d->kind = fl[1][0]; + d->global = fl[PAIR_LEN][0] == 'x'; + d->name = cr_copy(ix, fl[CBM_SZ_3], ln[CBM_SZ_3]); + d->qn = cr_copy(ix, fl[CBM_SZ_4], ln[CBM_SZ_4]); + bool none = n <= CBM_SZ_5 || (ln[CBM_SZ_5] == SKIP_ONE && fl[CBM_SZ_5][0] == '-'); + d->parent = none ? NULL : cr_copy(ix, fl[CBM_SZ_5], ln[CBM_SZ_5]); + d->next = CBM_NOT_FOUND; + } else if (fl[0][0] == 'A' && n >= PAIR_LEN) { + const char *name = cr_copy(ix, fl[1], ln[1]); + if (name && !cbm_ht_get(ix->anchors, name)) { + cbm_ht_set(ix->anchors, name, (void *)(uintptr_t)SKIP_ONE); + } + } +} + +static void cr_read_blob(cr_index_t *ix, int fi, const char *blob) { + const char *line = strchr(blob, '\n'); /* the tag line */ + while (line && !ix->oom) { + const char *b = line + SKIP_ONE; + const char *e = strchr(b, '\n'); + if (!e) { + e = b + strlen(b); + } + const char *fl[CR_FIELDS]; + size_t ln[CR_FIELDS]; + int n = 0; + const char *s = b; + for (const char *p = b; p <= e && n < CR_FIELDS; p++) { + if (p == e || *p == '\t') { + fl[n] = s; + ln[n] = (size_t)(p - s); + n++; + s = p + SKIP_ONE; + } + } + if (n > 0 && ln[0] == SKIP_ONE) { + cr_read_record(ix, fi, fl, ln, n); + } + line = *e ? e : NULL; + } +} + +static void cr_destroy(void *index) { + cr_index_t *ix = (cr_index_t *)index; + if (!ix) { + return; + } + cbm_ht_free(ix->by_name); + cbm_ht_free(ix->anchors); + cbm_ht_free(ix->by_path); + cbm_free(CBM_MEM_CLASS_OTHER, ix->run_to_file); + cbm_arena_destroy(&ix->arena); + cbm_free(CBM_MEM_CLASS_OTHER, ix); +} + +/* Chain every definition into its name's list, in index order. */ +static bool cr_link(cr_index_t *ix) { + ix->by_name = cbm_ht_create_in(CBM_MEM_CLASS_OTHER, (uint32_t)(ix->ndefs + SKIP_ONE)); + if (!ix->by_name) { + return false; + } + for (int i = ix->ndefs - SKIP_ONE; i >= 0; i--) { + cr_def_t *d = &ix->defs[i]; + if (!d->name || !d->qn) { + continue; + } + void *prev = cbm_ht_get(ix->by_name, d->name); + d->next = prev ? (int)((uintptr_t)prev - SKIP_ONE) : CBM_NOT_FOUND; + cbm_ht_set(ix->by_name, d->name, (void *)(uintptr_t)(i + SKIP_ONE)); + } + return true; +} + +static void *cr_build(const cbm_doclink_build_in_t *in) { + cr_index_t *ix = (cr_index_t *)cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*ix)); + if (!ix) { + return NULL; + } + cbm_arena_init(&ix->arena); + ix->nfiles = in->file_count; + ix->run_count = in->run_file_count; + ix->files = + in->file_count + ? (cr_file_t *)cbm_arena_calloc(&ix->arena, (size_t)in->file_count * sizeof(cr_file_t)) + : NULL; + ix->run_to_file = (int *)cbm_calloc(CBM_MEM_CLASS_OTHER, + (size_t)(in->run_file_count + SKIP_ONE) * sizeof(int)); + ix->by_path = cbm_ht_create_in(CBM_MEM_CLASS_OTHER, (uint32_t)(in->file_count + SKIP_ONE)); + ix->anchors = cbm_ht_create_in(CBM_MEM_CLASS_OTHER, CBM_SZ_64); + bool ok = (!in->file_count || ix->files) && ix->run_to_file && ix->by_path && ix->anchors; + for (int r = 0; ok && r < in->run_file_count; r++) { + ix->run_to_file[r] = CBM_NOT_FOUND; + } + for (int i = 0; ok && i < in->file_count && !ix->oom; i++) { + const cbm_doclink_file_t *src = &in->files[i]; + cr_file_t *f = &ix->files[i]; + f->path = src->rel_path; + f->test = cbm_is_test_file(src->rel_path, CBM_LANG_C); + cbm_ht_set(ix->by_path, src->rel_path, (void *)(uintptr_t)(i + SKIP_ONE)); + if (src->run_file >= 0 && src->run_file < in->run_file_count) { + ix->run_to_file[src->run_file] = i; + } + if (src->scope) { + cr_read_blob(ix, i, src->scope); + } + } + ok = ok && !ix->oom && cr_link(ix); + if (!ok) { + cbm_log_error("doc_links.c.error", "step", "index", "reason", "alloc"); + cr_destroy(ix); + return NULL; + } + char b[CBM_SZ_3][CBM_SZ_32]; + snprintf(b[0], sizeof(b[0]), "%d", ix->nfiles); + snprintf(b[SKIP_ONE], sizeof(b[SKIP_ONE]), "%d", ix->ndefs); + snprintf(b[PAIR_LEN], sizeof(b[PAIR_LEN]), "%u", cbm_ht_count(ix->anchors)); + cbm_log_info("doc_links.c.index", "files", b[0], "definitions", b[SKIP_ONE], "anchors", + b[PAIR_LEN]); + return ix; +} + +/* ── Resolution ──────────────────────────────────────────────────── */ + +/* The candidates of one level, one per qualified name. */ +typedef struct { + const cr_def_t *items[CR_CANDS]; + int n; + bool full; /* more than CR_CANDS: ambiguous */ +} cr_cands_t; + +static void cr_add(cr_cands_t *c, const cr_def_t *d) { + for (int i = 0; i < c->n; i++) { + if (strcmp(c->items[i]->qn, d->qn) == 0) { + return; + } + } + if (c->n == CR_CANDS) { + c->full = true; + return; + } + c->items[c->n++] = d; +} + +static bool cr_macro(const cr_def_t *d) { + return d->kind == 'm' || d->kind == 'M'; +} + +static bool cr_callable(const cr_def_t *d) { + return d->kind == 'f' || d->kind == 'M'; +} + +/* Keep the candidates `keep` accepts, when there is one. */ +static void cr_prefer(cr_cands_t *c, bool (*keep)(const cr_def_t *), bool want) { + int k = 0; + for (int i = 0; i < c->n; i++) { + k += keep(c->items[i]) == want; + } + if (k == 0 || k == c->n) { + return; + } + int w = 0; + for (int i = 0; i < c->n; i++) { + if (keep(c->items[i]) == want) { + c->items[w++] = c->items[i]; + } + } + c->n = w; +} + +enum { CR_LEVEL_OWN = 0, CR_LEVEL_FIELD = 1, CR_LEVEL_GLOBAL = 2, CR_LEVELS = 3 }; + +/* The definitions of `name` one level sees: the documented file's own (any + * linkage, fields apart), its fields, or those every file sees. */ +static void cr_level(const cr_index_t *ix, const char *name, int src, int level, cr_cands_t *c) { + void *v = cbm_ht_get(ix->by_name, name); + for (int i = v ? (int)((uintptr_t)v - SKIP_ONE) : CBM_NOT_FOUND; i >= 0; i = ix->defs[i].next) { + const cr_def_t *d = &ix->defs[i]; + bool field = d->kind == 'r'; + bool in = level == CR_LEVEL_GLOBAL ? d->global + : d->file == src && field == (level == CR_LEVEL_FIELD); + if (in) { + cr_add(c, d); + } + } +} + +static void cr_out_node(const cr_index_t *ix, int src, const cr_def_t *d, bool exact, + const cbm_gbuf_t *graph, cbm_doclink_outcome_t *out) { + const cbm_gbuf_node_t *node = cbm_gbuf_find_by_qn(graph, d->qn); + if (!node || !node->label) { + out->reason = CBM_DOCLINK_REASON_GRAPH_GAP; + return; + } + if (ix->files[d->file].test && !ix->files[src].test) { + out->reason = CBM_DOCLINK_REASON_TEST_ONLY; + return; + } + out->kind = CBM_DOCLINK_EDGE; + out->target = node; + out->exact = exact; +} + +static bool cr_ident(const char *s, size_t n) { + if (n == 0 || !(isalpha((unsigned char)s[0]) || s[0] == '_')) { + return false; + } + for (size_t i = 0; i < n; i++) { + if (!isalnum((unsigned char)s[i]) && s[i] != '_') { + return false; + } + } + return true; +} + +/* The name of a reference, `::` in front and `()` behind dropped, and the + * type of `Type::member` (scope, "" for none); false when it is no C name. */ +static bool cr_name(const char *raw, char *out, char *scope, size_t cap, bool *call) { + size_t n = strlen(raw); + if (n >= PAIR_LEN && raw[0] == ':' && raw[SKIP_ONE] == ':') { + raw += PAIR_LEN; + n -= PAIR_LEN; + } + *call = n >= PAIR_LEN && raw[n - PAIR_LEN] == '(' && raw[n - SKIP_ONE] == ')'; + if (*call) { + n -= PAIR_LEN; + } + scope[0] = '\0'; + const char *sep = NULL; + for (size_t i = 0; i + SKIP_ONE < n && !sep; i++) { + sep = raw[i] == ':' && raw[i + SKIP_ONE] == ':' ? raw + i : NULL; + } + if (sep) { + size_t sl = (size_t)(sep - raw); + if (sl >= cap || !cr_ident(raw, sl)) { + return false; + } + memcpy(scope, raw, sl); + scope[sl] = '\0'; + n -= sl + PAIR_LEN; + raw = sep + PAIR_LEN; + } + if (n >= cap || !cr_ident(raw, n)) { + return false; + } + memcpy(out, raw, n); + out[n] = '\0'; + return true; +} + +static bool cr_type(const cr_def_t *d) { + return d->kind == 't'; +} + +/* `Type::member`: the type by the usual levels, then its field (a node at + * the type's qualified name + `.` + member) or its enum constant. */ +static void cr_member(const cr_index_t *ix, int src, const char *scope, const char *name, + const cbm_gbuf_t *graph, cbm_doclink_outcome_t *out) { + const cr_def_t *type = NULL; + for (int level = 0; level < CR_LEVELS && !type; level++) { + cr_cands_t c = {.n = 0}; + cr_level(ix, scope, src, level, &c); + cr_prefer(&c, cr_type, true); + if (c.n == 0 || !cr_type(c.items[0])) { + continue; + } + if (c.full || c.n > SKIP_ONE) { + out->reason = CBM_DOCLINK_REASON_AMBIGUOUS; + return; + } + type = c.items[0]; + } + if (!type) { + return; /* missing */ + } + char qn[CR_QN_MAX]; + int n = snprintf(qn, sizeof(qn), "%s.%s", type->qn, name); + cr_def_t field = {.file = type->file, .kind = 'r', .name = name, .qn = qn}; + if (n > 0 && (size_t)n < sizeof(qn) && cbm_gbuf_find_by_qn(graph, qn)) { + cr_out_node(ix, src, &field, true, graph, out); + return; + } + void *v = cbm_ht_get(ix->by_name, name); + for (int i = v ? (int)((uintptr_t)v - SKIP_ONE) : CBM_NOT_FOUND; i >= 0; i = ix->defs[i].next) { + const cr_def_t *d = &ix->defs[i]; + if (d->kind == 'e' && d->parent && strcmp(d->parent, type->qn) == 0) { + cr_out_node(ix, src, d, true, graph, out); + return; + } + } +} + +/* A plain lowercase word: what an @see paragraph's prose is made of. */ +static bool cr_plain_word(const char *s) { + for (; *s; s++) { + if (*s < 'a' || *s > 'z') { + return false; + } + } + return true; +} + +static void cr_resolve_name(const cr_index_t *ix, int src, const CBMDocLink *link, + const cbm_gbuf_t *graph, cbm_doclink_outcome_t *out); + +static void cr_resolve(const void *index, void *state, int run_file, const CBMDocLink *link, + const cbm_gbuf_t *graph, cbm_doclink_outcome_t *out) { + (void)state; /* C keeps no per-file working state */ + const cr_index_t *ix = (const cr_index_t *)index; + out->kind = CBM_DOCLINK_UNRESOLVED; + out->reason = CBM_DOCLINK_REASON_MISSING; + out->target = NULL; + out->exact = false; + if (!ix || !link->raw || run_file < 0 || run_file >= ix->run_count || + ix->run_to_file[run_file] < 0) { + return; + } + int src = ix->run_to_file[run_file]; + cr_resolve_name(ix, src, link, graph, out); + if (link->syntax == CBM_DOCLINK_C_SEE && out->kind == CBM_DOCLINK_UNRESOLVED && + out->reason == CBM_DOCLINK_REASON_MISSING && cr_plain_word(link->raw)) { + out->kind = CBM_DOCLINK_LOCAL; /* prose: Doxygen prints it as text */ + } +} + +static void cr_resolve_name(const cr_index_t *ix, int src, const CBMDocLink *link, + const cbm_gbuf_t *graph, cbm_doclink_outcome_t *out) { + if (cbm_ht_get(ix->anchors, link->raw)) { + out->reason = CBM_DOCLINK_REASON_GRAPH_GAP; /* a page, group or anchor */ + return; + } + char name[CR_NAME_MAX]; + char scope[CR_NAME_MAX]; + bool call = false; + if (strlen(link->raw) > CBM_DOCLINK_C_REF_MAX || + !cr_name(link->raw, name, scope, sizeof(name), &call)) { + out->reason = CBM_DOCLINK_REASON_UNPARSEABLE; + return; + } + if (scope[0]) { + cr_member(ix, src, scope, name, graph, out); + return; + } + if (cbm_ht_get(ix->anchors, name)) { + out->reason = CBM_DOCLINK_REASON_GRAPH_GAP; + return; + } + for (int level = 0; level < CR_LEVELS; level++) { + cr_cands_t c = {.n = 0}; + cr_level(ix, name, src, level, &c); + if (c.n == 0) { + continue; + } + if (call) { + cr_prefer(&c, cr_callable, true); /* `()` written: what can be called */ + } + cr_prefer(&c, cr_macro, false); /* then a definition before a macro */ + if (c.full || c.n > SKIP_ONE) { + out->reason = CBM_DOCLINK_REASON_AMBIGUOUS; + return; + } + cr_out_node(ix, src, c.items[0], level != CR_LEVEL_GLOBAL, graph, out); + return; + } +} + +const cbm_doclink_resolver_t cbm_doclink_c_resolver = { + .langs = {CBM_LANG_C, CBM_LANG_CPP}, + .lang_count = PAIR_LEN, + .scope_tag = CBM_DOCLINK_C_SCOPE_TAG, + .build = cr_build, + .destroy = cr_destroy, + .resolve = cr_resolve, +}; diff --git a/tests/test_doc_mentions_c.c b/tests/test_doc_mentions_c.c new file mode 100644 index 000000000..b4fe96e84 --- /dev/null +++ b/tests/test_doc_mentions_c.c @@ -0,0 +1,341 @@ +/* + * test_doc_mentions_c.c -- C doc-comment references (Doxygen @ref, @see). + */ +#include "../src/foundation/compat.h" +#include "test_framework.h" +#include "test_helpers.h" +#include "test_doc_mentions_helpers.h" + +#include "cbm.h" +#include "doclink.h" + +#include +#include +#include + +/* ── extraction ──────────────────────────────────────────────────── */ + +static const char *const C_WIDGET = + "#include \"api.h\"\n" /* 1 */ + "\n" /* 2 */ + "/**\n" /* 3 */ + " * Makes widgets. @ref widget_free \"free it\" and \\ref api_counter.\n" /* 4 */ + " * Not `@ref Hidden` nor \"@ref Quoted\".\n" /* 5 */ + " * @code\n" /* 6 */ + " * @ref InCode\n" /* 7 */ + " * @endcode\n" /* 8 */ + " * ```\n" /* 9 */ + " * @ref Fenced\n" /* 10 */ + " * ```\n" /* 11 */ + " * @see widget_free(), helper, ::api_counter, https://example.com/w\n" /* 12 */ + " * also_this\n" /* 13 */ + " *\n" /* 14 */ + " * after_blank\n" /* 15 */ + " * \\sa \\c not_this, Other_thing.\n" /* 16 */ + " * @param n not_a_see\n" /* 17 */ + " */\n" /* 18 */ + "int widget_new(int n) { return n; }\n" /* 19 */ + "\n" /* 20 */ + "/// Line doc. @ref API_MAX\n" /* 21 */ + "/// @sa widget_new()\n" /* 22 */ + "static int helper(int x) { return x; }\n" /* 23 */ + "\n" /* 24 */ + "/** @defgroup grp The group */\n" /* 25 */ + "\n" /* 26 */ + "/** A prototype's doc: @ref grp and @ref widget_new. */\n" /* 27 */ + "int f2(void);\n" /* 28 */ + "#define API_MAX 3\n" /* 29 */ + "#define TWICE(x) ((x) * 2)\n" /* 30 */ + "static int counter;\n" /* 31 */ + "int api_counter = 0;\n" /* 32 */ + "enum mode { MODE_A };\n"; /* 33 */ + +static bool c_token_is(const CBMFileResult *r, const char *raw, uint32_t line, int syntax, + bool file) { + if (dm_count_tokens(r, raw) != 1) { + printf(" token %s: %d found\n", raw, dm_count_tokens(r, raw)); + return false; + } + const CBMDocLink *t = dm_find_token(r, raw); + bool is_file = (t->flags & CBM_DOCLINK_FLAG_FILE) != 0; + if (t->line != line || t->syntax != syntax || is_file != file) { + printf(" token %s: line %u syntax %d file %d\n", raw, t->line, t->syntax, is_file); + return false; + } + return true; +} + +TEST(doc_mentions_c_tokens) { + CBMFileResult *r = dm_extract(C_WIDGET, CBM_LANG_C, "src/widget.c"); + ASSERT_NOT_NULL(r); + ASSERT_TRUE(c_token_is(r, "widget_free", 4, CBM_DOCLINK_C_REF, false)); + ASSERT_TRUE(c_token_is(r, "api_counter", 4, CBM_DOCLINK_C_REF, false)); + /* the names an @see paragraph lists, to its empty line */ + ASSERT_TRUE(c_token_is(r, "widget_free()", 12, CBM_DOCLINK_C_SEE, false)); + ASSERT_TRUE(c_token_is(r, "helper", 12, CBM_DOCLINK_C_SEE, false)); + ASSERT_TRUE(c_token_is(r, "::api_counter", 12, CBM_DOCLINK_C_SEE, false)); + ASSERT_TRUE(c_token_is(r, "https://example.com/w", 12, CBM_DOCLINK_HREF, false)); + ASSERT_TRUE(c_token_is(r, "also_this", 13, CBM_DOCLINK_C_SEE, false)); + ASSERT_TRUE(c_token_is(r, "Other_thing", 16, CBM_DOCLINK_C_SEE, false)); + /* a /// run */ + ASSERT_TRUE(c_token_is(r, "API_MAX", 21, CBM_DOCLINK_C_REF, false)); + ASSERT_TRUE(c_token_is(r, "widget_new()", 22, CBM_DOCLINK_C_SEE, false)); + /* a prototype has no node: its doc belongs to the file */ + ASSERT_TRUE(c_token_is(r, "grp", 27, CBM_DOCLINK_C_REF, true)); + ASSERT_TRUE(c_token_is(r, "widget_new", 27, CBM_DOCLINK_C_REF, true)); + /* code spans, quotes, code blocks, fences, a new paragraph, \c */ + ASSERT_EQ(dm_count_tokens(r, "Hidden"), 0); + ASSERT_EQ(dm_count_tokens(r, "Quoted"), 0); + ASSERT_EQ(dm_count_tokens(r, "InCode"), 0); + ASSERT_EQ(dm_count_tokens(r, "Fenced"), 0); + ASSERT_EQ(dm_count_tokens(r, "after_blank"), 0); + ASSERT_EQ(dm_count_tokens(r, "not_this"), 0); + ASSERT_EQ(dm_count_tokens(r, "not_a_see"), 0); + ASSERT_EQ(r->doc_links.count, 12); + cbm_free_result(r); + PASS(); +} + +/* A C++ file is not part of the leg; a header is. */ +TEST(doc_mentions_c_files) { + const char *src = "/** See @ref other_fn. */\nint fn(void) { return 0; }\n"; + CBMFileResult *r = dm_extract(src, CBM_LANG_CPP, "src/x.cpp"); + ASSERT_NOT_NULL(r); + ASSERT_EQ(r->doc_links.count, 0); + ASSERT_NULL(r->doc_scope); + cbm_free_result(r); + r = dm_extract(src, CBM_LANG_CPP, "include/x.h"); + ASSERT_NOT_NULL(r); + ASSERT_EQ(dm_count_tokens(r, "other_fn"), 1); + cbm_free_result(r); + PASS(); +} + +static bool c_has_prefix_line(const char *blob, const char *prefix) { + size_t n = strlen(prefix); + for (const char *p = blob; p && *p;) { + const char *nl = strchr(p, '\n'); + size_t len = nl ? (size_t)(nl - p) : strlen(p); + if (len >= n && memcmp(p, prefix, n) == 0) { + return true; + } + p = nl ? nl + 1 : NULL; + } + printf(" no line: %s...\n", prefix); + return false; +} + +TEST(doc_mentions_c_scope_blob) { + CBMFileResult *r = dm_extract(C_WIDGET, CBM_LANG_C, "src/widget.c"); + ASSERT_NOT_NULL(r); + const char *s = r->doc_scope; + ASSERT_NOT_NULL(s); + ASSERT(strncmp(s, "c1\n", 3) == 0); + ASSERT(c_has_prefix_line(s, "D\tf\tx\twidget_new\t")); + ASSERT(c_has_prefix_line(s, "D\tf\ts\thelper\t")); + ASSERT(c_has_prefix_line(s, "D\tv\ts\tcounter\t")); + ASSERT(c_has_prefix_line(s, "D\tv\tx\tapi_counter\t")); + /* types, constants and macros of a .c file are its own */ + ASSERT(c_has_prefix_line(s, "D\tm\ts\tAPI_MAX\t")); + ASSERT(c_has_prefix_line(s, "D\tM\ts\tTWICE\t")); + ASSERT(c_has_prefix_line(s, "D\tt\ts\tmode\t")); + ASSERT(c_has_prefix_line(s, "D\te\ts\tMODE_A\t")); + ASSERT(c_has_prefix_line(s, "A\tgrp")); + cbm_free_result(r); + PASS(); +} + +/* ── resolution ──────────────────────────────────────────────────── */ + +static void c_write_repo(const char *repo) { + th_write_file(TH_PATH(repo, "include/api.h"), "#ifndef API_H\n" + "#define API_H\n" + "/** @defgroup widgets Widgets */\n" + "/** Makes one, see @ref widget_free. */\n" + "int widget_new(int n);\n" + "void widget_free(int w);\n" + "#define API_MAX 16\n" + "#define api_twice(x) ((x) * 2)\n" + "int dual(int x);\n" + "#define dual(x) ((x) + 1)\n" + "#define level(x) (x)\n" + "enum color { RED, GREEN };\n" + "struct st { int fld; };\n" + "typedef struct widget { int n; } widget_t;\n" + "extern int api_counter;\n" + "#endif\n"); + th_write_file(TH_PATH(repo, "src/widget.c"), + "#include \"api.h\"\n" + "int api_counter;\n" + "int level;\n" + "static int helper(int x) { return x; }\n" + "int widget_new(int n) { return n; }\n" + "void widget_free(int w) { (void)w; }\n" + "int (dual)(int x) { return x; }\n" + "/**\n" + " * The core. @ref helper, @ref api_counter, @ref API_MAX, @ref dual,\n" + " * @ref RED, @ref widgets, @ref other_static, @ref twin, @ref nope,\n" + " * @ref widget_t, @ref test_only_fn, @ref st::fld, @ref color::GREEN,\n" + " * @ref lfield, @ref st::nofield.\n" + " * @see api_twice() for details, unknown_fn()\n" + " */\n" + "int core(void) { return helper(0); }\n" + "struct local { int lfield; };\n"); + th_write_file(TH_PATH(repo, "src/other.c"), "#include \"api.h\"\n" + "static int other_static(void) { return 0; }\n" + "int twin(void) { return 1; }\n" + "/** Other: @ref dual, @ref level and\n" + " * @ref level(). */\n" + "int other(void) { return other_static(); }\n"); + th_write_file(TH_PATH(repo, "src/third.c"), "int twin(void) { return 2; }\n"); + th_write_file(TH_PATH(repo, "tests/test_widget.c"), "int test_only_fn(void) { return 0; }\n"); +} + +static bool c_edge(const char *db, const char *src, const char *tgt) { + char props[512]; + int n = 0; + dm_edge(db, src, tgt, props, sizeof(props), &n); + if (n != 1) { + printf(" edge %s -> %s: %d found\n", src, tgt, n); + return false; + } + return true; +} + +static bool c_row(const char *db, const char *raw, const char *reason) { + char got[64]; + dm_row(db, "src/widget.c", raw, got, sizeof(got), NULL, 0); + if (strcmp(got, reason) != 0) { + printf(" row [%s]: \"%s\", expected \"%s\"\n", raw, got, reason); + return false; + } + return true; +} + +TEST(doc_mentions_c_resolution) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlc_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[512]; + char db[512]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + snprintf(db, sizeof(db), "%s/g.db", tmp); + c_write_repo(repo); + ASSERT_EQ(dm_index(repo, db, NULL), 0); + const char *core = "src.widget.core"; + /* the documented file's own definitions, a static one too */ + ASSERT_TRUE(c_edge(db, core, "src.widget.helper")); + ASSERT_TRUE(c_edge(db, core, "src.widget.api_counter")); + ASSERT_TRUE(c_edge(db, core, "src.widget.dual")); + /* what every file sees: a header's macros, constants and types */ + ASSERT_TRUE(c_edge(db, core, "include.api.API_MAX#macro")); + ASSERT_TRUE(c_edge(db, core, "include.api.RED")); + ASSERT_TRUE(c_edge(db, core, "include.api.widget_t")); + ASSERT_TRUE(c_edge(db, core, "include.api.api_twice#macro")); + /* a group is no code entity; another file's static is not visible */ + ASSERT_TRUE(c_row(db, "widgets", "graph_gap")); + ASSERT_TRUE(c_row(db, "other_static", "missing")); + ASSERT_TRUE(c_row(db, "nope", "missing")); + ASSERT_TRUE(c_row(db, "twin", "ambiguous")); + ASSERT_TRUE(c_row(db, "test_only_fn", "test_only_target")); + /* Type::member: a struct field, an enum constant; a field of the file */ + ASSERT_TRUE(c_edge(db, core, "include.api.st.fld")); + ASSERT_TRUE(c_edge(db, core, "include.api.GREEN")); + ASSERT_TRUE(c_edge(db, core, "src.widget.local.lfield")); + ASSERT_TRUE(c_row(db, "st::nofield", "missing")); + /* prose in an @see paragraph is text; a code-shaped name is missing */ + ASSERT_TRUE(c_row(db, "for", "")); + ASSERT_TRUE(c_row(db, "details", "")); + ASSERT_TRUE(c_row(db, "unknown_fn()", "missing")); + /* a definition before a macro; with (), a function-like macro first */ + ASSERT_TRUE(c_edge(db, "src.other.other", "src.widget.dual")); + ASSERT_TRUE(c_edge(db, "src.other.other", "src.widget.level")); + ASSERT_TRUE(c_edge(db, "src.other.other", "include.api.level#macro")); + /* a header prototype's doc: the header file is its source */ + ASSERT_TRUE(c_edge(db, "include.api.h.__file__", "src.widget.widget_free")); + th_cleanup(tmp); + PASS(); +} + +/* ── incremental == full; one worker == several ──────────────────── */ + +TEST(doc_mentions_c_incremental_equals_full) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlc_inc_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[512]; + char inc_db[512]; + char full_db[512]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + snprintf(inc_db, sizeof(inc_db), "%s/inc.db", tmp); + snprintf(full_db, sizeof(full_db), "%s/full.db", tmp); + th_write_file(TH_PATH(repo, "src/lib.c"), + "int a(void) { return 0; }\nint b(void) { return 1; }\n"); + th_write_file(TH_PATH(repo, "src/use.c"), + "/** Uses @ref a. */\nint user(void) { return 0; }\n"); + ASSERT_EQ(dm_index(repo, inc_db, NULL), 0); + ASSERT_TRUE(c_edge(inc_db, "src.use.user", "src.lib.a")); + + /* a doc edit: the file is repaired alone */ + th_write_file(TH_PATH(repo, "src/use.c"), + "/** Uses @ref a and @ref nope. */\nint user(void) { return 0; }\n"); + ASSERT_EQ(dm_step(repo, inc_db, full_db, "doc edit", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0); + + /* a body edit in the target file: its scope is unchanged */ + th_write_file(TH_PATH(repo, "src/lib.c"), + "int a(void) { return 2; }\nint b(void) { return 1; }\n"); + ASSERT_EQ(dm_step(repo, inc_db, full_db, "body edit", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0); + ASSERT_TRUE(c_edge(inc_db, "src.use.user", "src.lib.a")); + + /* a becomes static: without a scope_delta hook that is a full run */ + th_write_file(TH_PATH(repo, "src/lib.c"), + "static int a(void) { return 0; }\nint b(void) { return 1; }\n"); + ASSERT_EQ(dm_step(repo, inc_db, full_db, "made static", CBM_INCREMENTAL_ROUTE_FORCED_FULL), 0); + char props[512]; + int n = 0; + dm_edge(inc_db, "src.use.user", "src.lib.a", props, sizeof(props), &n); + ASSERT_EQ(n, 0); + th_cleanup(tmp); + PASS(); +} + +TEST(doc_mentions_c_workers_agree) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlc_par_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[512]; + char par_db[512]; + char seq_db[512]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + snprintf(par_db, sizeof(par_db), "%s/par.db", tmp); + snprintf(seq_db, sizeof(seq_db), "%s/seq.db", tmp); + enum { MODULES = 60 }; /* past the parallel threshold */ + for (int i = 0; i < MODULES; i++) { + char rel[64]; + char src[1024]; + int next = (i + 1) % MODULES; + snprintf(rel, sizeof(rel), "src/m%02d.c", i); + snprintf(src, sizeof(src), + "/** F%02d; see @ref f%02d, @ref K%02d, @ref nope.\n" + " * @see g%02d() */\n" + "int f%02d(void) { return 0; }\n" + "int g%02d(void) { return 1; }\n" + "#define K%02d 1\n", + i, next, next, next, i, i, i); + th_write_file(TH_PATH(repo, rel), src); + } + ASSERT_EQ(dm_workers_agree(repo, par_db, seq_db), 0); + ASSERT_TRUE(c_edge(par_db, "src.m00.f00", "src.m01.f01")); + th_cleanup(tmp); + PASS(); +} + +SUITE(doc_mentions_c) { + dm_ship_held_families(); + RUN_TEST(doc_mentions_c_tokens); + RUN_TEST(doc_mentions_c_files); + RUN_TEST(doc_mentions_c_scope_blob); + RUN_TEST(doc_mentions_c_resolution); + RUN_TEST(doc_mentions_c_incremental_equals_full); + RUN_TEST(doc_mentions_c_workers_agree); +} diff --git a/tests/test_doc_mentions_helpers.h b/tests/test_doc_mentions_helpers.h index eb32af893..95e5f0503 100644 --- a/tests/test_doc_mentions_helpers.h +++ b/tests/test_doc_mentions_helpers.h @@ -442,6 +442,7 @@ static const int DM_HELD_FAMILIES[] = { CBM_DOCLINK_RS_LINK_DEF, CBM_DOCLINK_PERL_POD_L, CBM_DOCLINK_TS_LINK, CBM_DOCLINK_TS_LINKCODE, CBM_DOCLINK_TS_SEE, CBM_DOCLINK_PHP_SEE, + CBM_DOCLINK_C_REF, CBM_DOCLINK_C_SEE, }; static inline void dm_ship_held_families(void) { diff --git a/tests/test_main.c b/tests/test_main.c index cff0ee468..de3d80508 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -1831,6 +1831,7 @@ extern void suite_doc_mentions_rs(void); extern void suite_doc_mentions_perl(void); extern void suite_doc_mentions_ts(void); extern void suite_doc_mentions_php(void); +extern void suite_doc_mentions_c(void); extern void suite_infrascan(void); extern void suite_cli(void); extern void suite_agent_clients(void); @@ -2308,6 +2309,7 @@ int main(int argc, char **argv) { RUN_SELECTED_SUITE(doc_mentions_perl); RUN_SELECTED_SUITE(doc_mentions_ts); RUN_SELECTED_SUITE(doc_mentions_php); + RUN_SELECTED_SUITE(doc_mentions_c); /* Infrastructure scanning */ RUN_SELECTED_SUITE(infrascan); From 2fdcbfe288281f7f0a2db227d078a20aa27348f9 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 10 Oct 2026 18:20:13 +0200 Subject: [PATCH 09/17] doc-links(php): a property's, constant's or case's doc block belongs to its type PHP properties, class constants and enum cases have no graph node, so their doc blocks were never read: cakephp has 36 @see references there, symfony 21. The PHP leg now uses the driver's loose_docs hook: a /** */ block right above a property, constant or case is harvested with the type around it as the source (as the field test's prototype attached it), and resolved like any other reference written in that type (self:: is the type). The driver marks as the file's only what a hook pushes with the file stand-in (the C leg's prototype docs); a hook may push with a definition of the file instead. Proof (ship build, Apple M5 Pro / 48 GB, base 80e000a9): - koel 04ed8beaec34: all 5 of the prototype's pairs bound alike (the fifth sits in a class constant's doc). - cakephp a75828db: 373 edge mentions + 44 rows = every one of the 417 non-URL references (was 381 of 417). - symfony 9917e115: 459 + 219 = 678 of 681 (the other 3 sit in blocks before code the census does not classify). Tests: doc_mentions_php resolves a property's doc block to an edge from its class; red with the hook's walk reverted, and red when the driver marks every token of the file pass as the file's. Signed-off-by: Martin Vogel --- internal/cbm/doclink.c | 16 +++-- internal/cbm/doclink.h | 20 ++++--- internal/cbm/doclink_php.c | 109 ++++++++++++++++++++++++++++++++++ tests/test_doc_mentions_php.c | 19 +++--- 4 files changed, 144 insertions(+), 20 deletions(-) diff --git a/internal/cbm/doclink.c b/internal/cbm/doclink.c index 18a65cef1..8c6612b0e 100644 --- a/internal/cbm/doclink.c +++ b/internal/cbm/doclink.c @@ -251,7 +251,8 @@ static const doclink_lang_t DOCLINK_LANGS[] = { .parse_doc = cbm_doclink_php_parse_doc, .scan_scope = cbm_doclink_php_scan_scope, .scope_tag = CBM_DOCLINK_PHP_SCOPE_TAG, - .portable_scope = cbm_doclink_php_portable_scope}, + .portable_scope = cbm_doclink_php_portable_scope, + .loose_docs = cbm_doclink_php_loose_docs}, }; static const doclink_lang_t *doclink_lang(CBMLanguage lang) { @@ -480,10 +481,11 @@ static bool doclink_taken(const CBMExtractCtx *ctx, uint32_t first, uint32_t las return false; } -/* The file's own doc, then the language's loose docs: their references belong - * to the file. The parser gets a definition-shaped stand-in for the file; what - * it pushes is marked, and the resolving half takes the File node as the - * source. No qualified name for that node is computed on this side. */ +/* The file's own doc, then the language's loose docs. The parser gets a + * definition-shaped stand-in for the file; what it pushes with the stand-in + * is marked, and the resolving half takes the File node as the source (a + * loose-doc hook may push with a definition of the file instead). No qualified + * name for that node is computed on this side. */ static void doclink_file_pass(CBMExtractCtx *ctx, const doclink_lang_t *L) { const char *file_doc = ctx->result->module_doc; bool has_doc = file_doc && file_doc[0]; @@ -513,7 +515,9 @@ static void doclink_file_pass(CBMExtractCtx *ctx, const doclink_lang_t *L) { L->loose_docs(ctx, &file_def, doclink_taken); } for (int i = first; i < arr->count; i++) { - arr->items[i].flags |= CBM_DOCLINK_FLAG_FILE; + if (arr->items[i].source_qn == file_def.qualified_name) { + arr->items[i].flags |= CBM_DOCLINK_FLAG_FILE; /* not a definition's */ + } } } diff --git a/internal/cbm/doclink.h b/internal/cbm/doclink.h index cce493488..faccd4dcf 100644 --- a/internal/cbm/doclink.h +++ b/internal/cbm/doclink.h @@ -23,9 +23,11 @@ * scope_tag, the blob's tag line and its persisted form (line * portable_scope numbers dropped). Optional. * loose_docs after the definitions and the file's own doc: the - * file's other doc comments (C: a prototype's doc, - * which the graph keeps no node for); what it pushes - * belongs to the file. Optional. + * file's other doc comments, which the graph keeps no + * node for. What it pushes with the file stand-in + * belongs to the file (C: a prototype's doc); it may push + * with a definition of the file instead (PHP: a + * property's doc belongs to its class). Optional. * scan_file instead of parse_doc, for a DOCUMENT language whose * text is the documentation (Markdown): one pass over * the file's bytes pushes every reference, with the @@ -564,12 +566,20 @@ char *cbm_doclink_ts_portable_scope(const char *scope); * this length and the resolver reports it as unparseable. */ enum { CBM_DOCLINK_TS_REF_MAX = 512 }; +/* Asked by a loose_docs hook for a doc comment on lines [first, last]: true + * when a definition's doc or the file's own doc there was already taken. */ +typedef bool (*cbm_doclink_taken_fn)(const CBMExtractCtx *ctx, uint32_t first, uint32_t last); + /* ── PHP (doclink_php.c) ───────────────────────────────────────────── */ void cbm_doclink_php_parse_doc(CBMExtractCtx *ctx, const CBMDefinition *def, const char *doc, uint32_t doc_line); const char *cbm_doclink_php_scan_scope(CBMExtractCtx *ctx); char *cbm_doclink_php_portable_scope(const char *scope); +/* Doc blocks on properties, class constants and enum cases (they have no + * node): their references belong to the type around them. */ +void cbm_doclink_php_loose_docs(CBMExtractCtx *ctx, const CBMDefinition *file_def, + cbm_doclink_taken_fn taken); #define CBM_DOCLINK_PHP_SCOPE_TAG "php1" @@ -600,10 +610,6 @@ size_t cbm_doclink_php_qualify(char *out, size_t cap, const char *ns, size_t ns_ /* ── C (doclink_c.c) ───────────────────────────────────────────────── */ -/* Asked by a loose_docs hook for a doc comment on lines [first, last]: true - * when a definition's doc or the file's own doc there was already taken. */ -typedef bool (*cbm_doclink_taken_fn)(const CBMExtractCtx *ctx, uint32_t first, uint32_t last); - void cbm_doclink_c_parse_doc(CBMExtractCtx *ctx, const CBMDefinition *def, const char *doc, uint32_t doc_line); void cbm_doclink_c_loose_docs(CBMExtractCtx *ctx, const CBMDefinition *file_def, diff --git a/internal/cbm/doclink_php.c b/internal/cbm/doclink_php.c index 8abb8a437..35f14788e 100644 --- a/internal/cbm/doclink_php.c +++ b/internal/cbm/doclink_php.c @@ -37,6 +37,10 @@ * flags as for T * "At the top level" includes the branches of a top-level conditional * (`if (!function_exists('f')) { function f() {} }`), never a function body. + * Doc blocks on properties, class constants and enum cases, which have no + * node, go through cbm_doclink_php_loose_docs: their references belong to + * the type around them. + * * The persisted blob (cbm_doclink_php_portable_scope) drops the lines of N * and T and the U records: other files read names only, and every name a * supertype record holds is already qualified. @@ -910,6 +914,111 @@ const char *cbm_doclink_php_scan_scope(CBMExtractCtx *ctx) { return sb.oom ? NULL : sb.buf; } +/* ── Loose docs ──────────────────────────────────────────────────── */ + +/* The definition of a type declaration: its name, at its line. */ +static const CBMDefinition *php_type_def(const CBMExtractCtx *ctx, TSNode decl) { + TSNode name = ts_node_child_by_field_name(decl, TS_FIELD("name")); + if (ts_node_is_null(name)) { + return NULL; + } + uint32_t b = ts_node_start_byte(name); + size_t n = ts_node_end_byte(name) - b; + uint32_t line = ts_node_start_point(decl).row + SKIP_ONE; + const CBMDefArray *defs = &ctx->result->defs; + for (int i = 0; i < defs->count; i++) { + const CBMDefinition *d = &defs->items[i]; + if (d->name && d->qualified_name && d->start_line == line && strlen(d->name) == n && + memcmp(d->name, ctx->source + b, n) == 0) { + return d; + } + } + return NULL; +} + +static bool php_member_decl(TSNode n) { + return php_kind(n, "property_declaration") || php_kind(n, "const_declaration") || + php_kind(n, "enum_case"); +} + +static bool php_doc_comment(const CBMExtractCtx *ctx, TSNode n) { + if (!php_kind(n, "comment")) { + return false; + } + uint32_t b = ts_node_start_byte(n); + size_t len = ts_node_end_byte(n) - b; + return len > PHP_PREFIX_LEN + SKIP_ONE && memcmp(ctx->source + b, "/**", PHP_PREFIX_LEN) == 0 && + ctx->source[b + PHP_PREFIX_LEN] != '*'; +} + +/* The doc blocks right above a type's properties, constants and cases. */ +static void php_loose_type(CBMExtractCtx *ctx, TSNode decl, cbm_doclink_taken_fn taken) { + const CBMDefinition *def = php_type_def(ctx, decl); + TSNode body = ts_node_child_by_field_name(decl, TS_FIELD("body")); + if (!def || ts_node_is_null(body)) { + return; + } + TSNode doc = {0}; + TSTreeCursor cur = ts_tree_cursor_new(body); + for (bool more = ts_tree_cursor_goto_first_child(&cur); more; + more = ts_tree_cursor_goto_next_sibling(&cur)) { + TSNode c = ts_tree_cursor_current_node(&cur); + if (!ts_node_is_named(c)) { + continue; + } + if (php_doc_comment(ctx, c)) { + doc = c; + continue; + } + uint32_t first = ts_node_is_null(doc) ? 0 : ts_node_start_point(doc).row + SKIP_ONE; + uint32_t last = ts_node_is_null(doc) ? 0 : ts_node_end_point(doc).row + SKIP_ONE; + if (first && php_member_decl(c) && + ts_node_start_point(c).row + SKIP_ONE <= last + SKIP_ONE && !taken(ctx, first, last)) { + CBMArena *a = ctx->scratch ? ctx->scratch : ctx->arena; + uint32_t b = ts_node_start_byte(doc); + char *text = cbm_arena_strndup(a, ctx->source + b, ts_node_end_byte(doc) - b); + if (text) { + cbm_doclink_php_parse_doc(ctx, def, text, first); + } + } + doc = (TSNode){0}; + } + ts_tree_cursor_delete(&cur); +} + +/* The types of the file: top level, namespace bodies, top-level conditionals. */ +static void php_loose_walk(CBMExtractCtx *ctx, TSNode parent, int depth, + cbm_doclink_taken_fn taken) { + if (depth > PHP_COND_DEPTH) { + return; + } + TSTreeCursor cur = ts_tree_cursor_new(parent); + for (bool more = ts_tree_cursor_goto_first_child(&cur); more; + more = ts_tree_cursor_goto_next_sibling(&cur)) { + TSNode c = ts_tree_cursor_current_node(&cur); + if (php_is_type_decl(c)) { + php_loose_type(ctx, c, taken); + } else if (php_kind(c, "namespace_definition")) { + TSNode body = ts_node_child_by_field_name(c, TS_FIELD("body")); + if (!ts_node_is_null(body)) { + php_loose_walk(ctx, body, depth + SKIP_ONE, taken); + } + } else if (php_is_branch(c)) { + php_loose_walk(ctx, c, depth + SKIP_ONE, taken); + } + } + ts_tree_cursor_delete(&cur); +} + +void cbm_doclink_php_loose_docs(CBMExtractCtx *ctx, const CBMDefinition *file_def, + cbm_doclink_taken_fn taken) { + (void)file_def; /* a member's doc belongs to its type, never to the file */ + if (!ctx || !ctx->result || !taken || ts_node_is_null(ctx->root)) { + return; + } + php_loose_walk(ctx, ctx->root, 0, taken); +} + /* The blob without lines and imports: "N blk name first last" -> "N blk * name", "T kind fqn flags first last" -> "T kind fqn flags", U dropped. */ char *cbm_doclink_php_portable_scope(const char *scope) { diff --git a/tests/test_doc_mentions_php.c b/tests/test_doc_mentions_php.c index c4342d90d..c1352b0e5 100644 --- a/tests/test_doc_mentions_php.c +++ b/tests/test_doc_mentions_php.c @@ -193,13 +193,16 @@ static void php_write_repo(const char *repo) { "class Numbers { const MAX = 9; }\n" "trait Loud { public function shout() {} }\n" "function clamp($v) { return $v; }\n"); - th_write_file(TH_PATH(repo, "src/Model/Base.php"), " Date: Sat, 10 Oct 2026 19:55:29 +0200 Subject: [PATCH 10/17] doc-links(php): a bare name() in a type's doc is the type's method phpDocumentor reads `@see name()` as a function. Authors write it for the documented class's own method: symfony has 97 such references that ended as external rows (Command `@see setCode()`, ProgressBar `{@see resumeAll()}`), and one that bound a global build-script function (ProcessHelper `@see run()`, meaning its own run()). In the doc of a type or of one of its members, a bare name() is now a method the type declares or inherits; a `use function` import of the name still wins, and outside a type the function rules stay. The ambiguity guard this replaces is gone. Proof (ship build, Apple M5 Pro / 48 GB): symfony 9917e115 +91 edges (external bare name() rows 97 -> 6), cakephp a75828db +6 (7 -> 1); a sample of 14 new edges read against their lines: all correct, inherited methods included (UploadedFile -> File::guessExtension). Test: doc_mentions_php_current_class (the method wins over a global function of the same name; the documented method itself is a self-mention), red with the rule reverted. Signed-off-by: Martin Vogel --- src/pipeline/doc_links_php.c | 37 ++++++++++++----------------------- tests/test_doc_mentions_php.c | 24 +++++++++++++++++++---- 2 files changed, 33 insertions(+), 28 deletions(-) diff --git a/src/pipeline/doc_links_php.c b/src/pipeline/doc_links_php.c index 959595c6e..2f8e03f5a 100644 --- a/src/pipeline/doc_links_php.c +++ b/src/pipeline/doc_links_php.c @@ -14,10 +14,10 @@ * go beyond that tool, as PHP itself has them: an unqualified function name * is taken from a `use function` import, else from the current namespace, * else from the global namespace; and `self` and `static` name the type the - * doc sits in, `parent` that type's superclass. A bare `name()` that would - * bind a function but is written in a type that declares or inherits a - * method of that name is ambiguous: the method or the function, and the - * reference does not say which. Type, function + * doc sits in, `parent` that type's superclass. Beyond both, as authors + * write it: a bare `name()` in the doc of a type (or of its members) that + * declares or inherits a method of that name is that method; a `use + * function` import of the name still wins. Type, function * and method names compare without case, properties and constants with it. * The global fallback cannot see a function a dependency declares in the * current namespace: a reference to one binds the global function of that @@ -833,17 +833,6 @@ static pr_res_t pr_find_func(const pr_ctx_t *c, const char *fqn, size_t len) { return r; } -/* True when the type the doc sits in declares or inherits a method named - * like the reference. */ -static bool pr_type_has_method(const pr_ctx_t *c, const pr_ref_t *r) { - if (c->cur < 0) { - return false; - } - pr_search_t s = { - .c = c, .want = 'm', .name = r->cls, .len = r->cls_len, .owner = CBM_NOT_FOUND}; - return pr_search(&s, c->cur, 0); -} - static pr_res_t pr_function(const pr_ctx_t *c, const pr_ref_t *r) { char fqn[PR_NAME_MAX]; if (r->cls[0] == '\\' || memchr(r->cls, '\\', r->cls_len)) { @@ -868,6 +857,14 @@ static pr_res_t pr_function(const pr_ctx_t *c, const pr_ref_t *r) { return res; } } + if (c->cur >= 0) { + /* a method the documented type declares or inherits */ + pr_search_t s = { + .c = c, .want = 'm', .name = r->cls, .len = r->cls_len, .owner = CBM_NOT_FOUND}; + if (pr_search(&s, c->cur, 0)) { + return pr_member_node(c, &s); + } + } pr_res_t res = pr_mk(PR_NONE); if (c->ns_len) { int n = snprintf(fqn, sizeof(fqn), "%.*s\\%.*s", (int)c->ns_len, c->ns, (int)r->cls_len, @@ -877,15 +874,7 @@ static pr_res_t pr_function(const pr_ctx_t *c, const pr_ref_t *r) { if (res.kind == PR_NONE) { res = pr_find_func(c, r->cls, r->cls_len); /* PHP's global fallback */ } - if (res.kind == PR_NONE) { - return pr_mk(PR_EXTERNAL); /* a built-in function */ - } - if ((res.kind == PR_NODE || res.kind == PR_GAP) && pr_type_has_method(c, r)) { - /* written in a type that has a method of that name: the method or - * this function, and the reference does not pick one */ - return pr_mk(PR_AMBIGUOUS); - } - return res; + return res.kind == PR_NONE ? pr_mk(PR_EXTERNAL) : res; /* none: a built-in */ } /* ── Outcomes ────────────────────────────────────────────────────── */ diff --git a/tests/test_doc_mentions_php.c b/tests/test_doc_mentions_php.c index c1352b0e5..1e31b2c5d 100644 --- a/tests/test_doc_mentions_php.c +++ b/tests/test_doc_mentions_php.c @@ -216,6 +216,7 @@ static void php_write_repo(const char *repo) { "function global_helper() {}\n" "function shadowed() {}\n" "function other() {}\n" + "function size() {}\n" "class GlobalThing {}\n"); th_write_file( TH_PATH(repo, "src/Model/User.php"), @@ -342,10 +343,6 @@ TEST(doc_mentions_php_global_fallback) { ASSERT_TRUE(php_edge(db, o, "src.Model.User.shadowed")); ASSERT_TRUE(php_edge(db, o, "src.global.global_helper")); ASSERT_TRUE(php_edge(db, o, "src.Model.User.helper_here")); - /* a function named like a method of the documented type: two readings; - * no such function: PHP's built-ins */ - ASSERT_TRUE(php_row(db, "other()", "ambiguous")); - ASSERT_TRUE(php_row(db, "size()", "external")); char props[512]; int n = 0; dm_edge(db, o, "src.global.shadowed", props, sizeof(props), &n); @@ -369,6 +366,24 @@ TEST(doc_mentions_php_self_parent) { PASS(); } +/* A bare name() in a type's doc: the type's own method first, though a + * global function has the name too. */ +TEST(doc_mentions_php_current_class) { + ASSERT_TRUE(php_indexed()); + const char *db = g_php_db; + const char *o = "src.Model.User.User.other"; + ASSERT_TRUE(php_edge(db, o, "src.Model.User.User.size")); + char props[512]; + int n = 0; + dm_edge(db, o, "src.global.size", props, sizeof(props), &n); + ASSERT_EQ(n, 0); + /* the documented method itself: a self-mention, neither edge nor row */ + ASSERT_TRUE(php_row(db, "other()", "")); + dm_edge(db, o, "src.global.other", props, sizeof(props), &n); + ASSERT_EQ(n, 0); + PASS(); +} + static void php_indexed_cleanup(void) { if (g_php_db[0]) { th_cleanup(g_php_tmp); @@ -463,6 +478,7 @@ SUITE(doc_mentions_php) { RUN_TEST(doc_mentions_php_resolution); RUN_TEST(doc_mentions_php_global_fallback); RUN_TEST(doc_mentions_php_self_parent); + RUN_TEST(doc_mentions_php_current_class); php_indexed_cleanup(); RUN_TEST(doc_mentions_php_incremental_equals_full); RUN_TEST(doc_mentions_php_workers_agree); From 06e69d0c562c924ddd2388735ed7e157040e1a10 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 10 Oct 2026 19:55:29 +0200 Subject: [PATCH 11/17] doc-links(ts): bare specifiers through tsconfig paths and workspace packages A bare import specifier was always external. The TS resolver now asks the pipeline what it already knows for import resolution, without indexing package.json or tsconfig.json: the nearest tsconfig/jsconfig `paths` and `baseUrl` (the alias collection; the first target, as TypeScript tries it first), then the workspace package whose `name` the specifier is (the package map), bound to the package's entry module (`exports`, `main`, `module`) when the repository indexes it. Every other bare specifier, a package's subpath included, stays external. Incremental runs stay exact: a tsconfig change re-resolves the files under it, a package.json change is a manifest change. path_alias gets cbm_path_alias_resolve_into, which writes into the caller's buffer; cbm_path_alias_resolve now wraps it (two raw allocations fewer, the memory-core baseline lowered to match). Also fixed: a declaration in an index file has no `index` in its QN (fqn.c strips it from a symbol's QN, not from the module's), so a reference to one missed its node; the resolver now names a file's declarations with that prefix. Proof (ship build, Apple M5 Pro / 48 GB): the TypeScript compiler b465fdbfe175 unchanged (69 edges, 57 of the prototype's 61 pairs); discord.js 4dc1acc6 unchanged (no doc reference names a workspace import; its packages' entries are build output). Tests: a tsconfig alias, a workspace package's entry and an index file's declaration each bind, each red with its rule reverted; path_alias, pipeline, edge_imports, incremental and all doc suites pass (1298). Signed-off-by: Martin Vogel --- scripts/memory-core-baseline.txt | 2 +- src/pipeline/doc_links_ts.c | 122 ++++++++++++++++++++++++------- src/pipeline/path_alias.c | 50 ++++++------- src/pipeline/path_alias.h | 6 ++ tests/test_doc_mentions_ts.c | 21 +++++- 5 files changed, 146 insertions(+), 55 deletions(-) diff --git a/scripts/memory-core-baseline.txt b/scripts/memory-core-baseline.txt index 251920e2f..7095384a0 100644 --- a/scripts/memory-core-baseline.txt +++ b/scripts/memory-core-baseline.txt @@ -75,7 +75,7 @@ src/pipeline/pass_pkgmap.c 82 src/pipeline/pass_semantic.c 3 src/pipeline/pass_tests.c 3 src/pipeline/pass_usages.c 4 -src/pipeline/path_alias.c 42 +src/pipeline/path_alias.c 40 src/pipeline/pipeline.c 134 src/pipeline/pipeline_delta.c 19 src/pipeline/pipeline_incremental.c 143 diff --git a/src/pipeline/doc_links_ts.c b/src/pipeline/doc_links_ts.c index db007f37e..c52d2e6da 100644 --- a/src/pipeline/doc_links_ts.c +++ b/src/pipeline/doc_links_ts.c @@ -14,8 +14,12 @@ * through `export { a as b }`, `export ... from` and `export *` barrels * (a cycle guard and a depth bound; two barrels that answer differently * are ambiguous); a relative specifier resolves with the usual extension - * and index rules, every bare one is external (package.json and - * tsconfig.json are not indexed); + * and index rules; a bare one through the nearest tsconfig/jsconfig + * `paths` (the first target, as TypeScript tries it first) or + * `baseUrl`, else, when it is the `name` of a workspace package (the + * pipeline's package map), to that package's entry module (`exports`, + * `main`, `module`) when the repository indexes it; every other bare + * specifier, a package's subpath included, is external; * 5. the standard library's global names: external; * 6. the top-level declarations of the repository's script files (files * without an import or an export are global): one binds, several are @@ -29,7 +33,8 @@ * declares or imports, unique otherwise. */ #include "pipeline/doc_links.h" -#include "pipeline/pipeline_internal.h" /* cbm_pipeline_file_node */ +#include "pipeline/path_alias.h" +#include "pipeline/pipeline_internal.h" /* cbm_pipeline_file_node, cbm_pipeline_get_pkgmap */ #include "cbm.h" #include "doclink.h" #include "helpers.h" /* cbm_fqn_module_source_lang, cbm_is_test_file */ @@ -83,6 +88,8 @@ typedef struct { typedef struct { const char *path; const char *module_qn; + const char *sym_qn; /* what its declarations' QNs start with: an index file's + * module QN without `.index` (fqn.c, strip_init_or_index) */ bool test; bool module_syntax; tr_rec_t *imps; /* I */ @@ -110,8 +117,11 @@ typedef struct { int nfiles; int *run_to_file; int run_count; - CBMHashTable *by_path; /* rel_path -> (file index + 1) as a pointer */ - CBMHashTable *globals; /* name -> tr_global_t */ + CBMHashTable *by_path; /* rel_path -> (file index + 1) as a pointer */ + CBMHashTable *globals; /* name -> tr_global_t */ + CBMHashTable *by_module; /* module QN -> (first file index + 1) */ + const cbm_path_alias_collection_t *aliases; /* the pipeline's, borrowed */ + CBMHashTable *pkgmap; /* the pipeline's, borrowed */ bool oom; } tr_index_t; @@ -266,6 +276,7 @@ static void tr_destroy(void *index) { } cbm_ht_free(ix->by_path); cbm_ht_free(ix->globals); + cbm_ht_free(ix->by_module); cbm_free(CBM_MEM_CLASS_OTHER, ix->run_to_file); cbm_arena_destroy(&ix->arena); cbm_free(CBM_MEM_CLASS_OTHER, ix); @@ -298,6 +309,18 @@ static bool tr_add_globals(tr_index_t *ix) { return true; } +/* The QN prefix of a file's declarations: an index module's QN loses its + * last segment, as a symbol QN does in fqn.c. */ +static const char *tr_symbol_prefix(tr_index_t *ix, const char *module_qn) { + static const char suffix[] = ".index"; + size_t sl = sizeof(suffix) - SKIP_ONE; + size_t n = module_qn ? strlen(module_qn) : 0; + if (n <= sl || strcmp(module_qn + n - sl, suffix) != 0) { + return module_qn; + } + return tr_copy(ix, module_qn, n - sl); +} + static void *tr_build(const cbm_doclink_build_in_t *in) { tr_index_t *ix = (tr_index_t *)cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*ix)); if (!ix) { @@ -326,6 +349,7 @@ static void *tr_build(const cbm_doclink_build_in_t *in) { cbm_fqn_module_source_lang(&ix->arena, ix->project, src->rel_path, CBM_LANG_TYPESCRIPT); f->test = cbm_is_test_file(src->rel_path, tr_lang(src->rel_path)); ix->oom = ix->oom || !f->module_qn; + f->sym_qn = tr_symbol_prefix(ix, f->module_qn); cbm_ht_set(ix->by_path, src->rel_path, (void *)(uintptr_t)(i + SKIP_ONE)); if (src->run_file >= 0 && src->run_file < in->run_file_count) { ix->run_to_file[src->run_file] = i; @@ -335,6 +359,16 @@ static void *tr_build(const cbm_doclink_build_in_t *in) { } } ok = ok && !ix->oom && tr_add_globals(ix); + ix->aliases = in->ctx->path_aliases; + ix->pkgmap = cbm_pipeline_get_pkgmap(); + ix->by_module = + ok ? cbm_ht_create_in(CBM_MEM_CLASS_OTHER, (uint32_t)(ix->nfiles + SKIP_ONE)) : NULL; + ok = ok && ix->by_module; + for (int i = ix->nfiles - SKIP_ONE; ok && i >= 0; i--) { + if (ix->files[i].module_qn) { + cbm_ht_set(ix->by_module, ix->files[i].module_qn, (void *)(uintptr_t)(i + SKIP_ONE)); + } + } if (!ok) { cbm_log_error("doc_links.ts.error", "step", "index", "reason", "alloc"); tr_destroy(ix); @@ -410,7 +444,7 @@ static const tr_rec_t *tr_decl(const tr_file_t *f, const char *name) { /* A top-level declaration of file f: its node, or a graph gap. */ static tr_res_t tr_decl_res(const tr_ctx_t *c, const tr_file_t *f, const char *name) { char qn[TR_QN_MAX]; - if (!tr_join(qn, sizeof(qn), f->module_qn, name, NULL)) { + if (!tr_join(qn, sizeof(qn), f->sym_qn, name, NULL)) { return tr_mk(TR_GAP); } const cbm_gbuf_node_t *n = tr_node(c, qn, f); @@ -461,22 +495,9 @@ static bool tr_norm(const char *dir, size_t dl, const char *rel, char *out, size return w > 0; } -/* The file a module specifier written in file `from` names: -1 with - * *external for a bare specifier (a package). */ -static int tr_spec_file(const tr_index_t *ix, int from, const char *spec, bool *external) { - *external = false; - if (!(spec[0] == '.' && - (spec[SKIP_ONE] == '/' || spec[SKIP_ONE] == '\0' || - (spec[SKIP_ONE] == '.' && (spec[PAIR_LEN] == '/' || spec[PAIR_LEN] == '\0'))))) { - *external = true; - return CBM_NOT_FOUND; - } - const char *path = ix->files[from].path; - const char *slash = strrchr(path, '/'); - char base[TR_PATH_MAX]; - if (!tr_norm(path, slash ? (size_t)(slash - path) : 0, spec, base, sizeof(base))) { - return CBM_NOT_FOUND; - } +/* The file a repository-relative module path names: the path itself, the + * path with a module extension, or its index file. */ +static int tr_path_file(const tr_index_t *ix, const char *base) { static const char *const strip[] = {".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", NULL}; static const char *const exts[] = {".ts", ".tsx", ".d.ts", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs", NULL}; @@ -508,6 +529,58 @@ static int tr_spec_file(const tr_index_t *ix, int from, const char *spec, bool * return hit; } +/* The workspace package a bare specifier names (`name` or `@scope/name`, no + * subpath): the file of its entry module, as the pipeline's package map + * holds it. */ +static int tr_package_file(const tr_index_t *ix, const char *spec) { + const char *slash = strchr(spec, '/'); + if (spec[0] == '@' && slash) { + slash = strchr(slash + SKIP_ONE, '/'); + } + if (slash || !ix->pkgmap || !ix->by_module) { + return CBM_NOT_FOUND; + } + const char *qn = (const char *)cbm_ht_get(ix->pkgmap, spec); + void *v = qn ? cbm_ht_get(ix->by_module, qn) : NULL; + return v ? (int)((uintptr_t)v - SKIP_ONE) : CBM_NOT_FOUND; +} + +/* A bare specifier the repository answers itself: the nearest + * tsconfig/jsconfig `paths` or `baseUrl`, else a workspace package. */ +static int tr_bare_file(const tr_index_t *ix, int from, const char *spec) { + const cbm_path_alias_map_t *map = + ix->aliases ? cbm_path_alias_find_for_file(ix->aliases, ix->files[from].path) : NULL; + char aliased[TR_PATH_MAX]; + if (map && cbm_path_alias_resolve_into(map, spec, aliased, sizeof(aliased))) { + int hit = tr_path_file(ix, aliased); + if (hit >= 0) { + return hit; + } + } + return tr_package_file(ix, spec); +} + +/* The file a module specifier written in file `from` names: -1 with + * *external for a bare specifier the repository does not answer (a + * package). */ +static int tr_spec_file(const tr_index_t *ix, int from, const char *spec, bool *external) { + *external = false; + if (!(spec[0] == '.' && + (spec[SKIP_ONE] == '/' || spec[SKIP_ONE] == '\0' || + (spec[SKIP_ONE] == '.' && (spec[PAIR_LEN] == '/' || spec[PAIR_LEN] == '\0'))))) { + int hit = tr_bare_file(ix, from, spec); + *external = hit < 0; + return hit; + } + const char *path = ix->files[from].path; + const char *slash = strrchr(path, '/'); + char base[TR_PATH_MAX]; + if (!tr_norm(path, slash ? (size_t)(slash - path) : 0, spec, base, sizeof(base))) { + return CBM_NOT_FOUND; + } + return tr_path_file(ix, base); +} + static tr_res_t tr_export_of(tr_ctx_t *c, int file, const char *name, int depth); /* What a specifier's file exports as `name`. */ @@ -833,9 +906,8 @@ static tr_res_t tr_head(tr_ctx_t *c, const CBMDocLink *link, const char *name) { } for (int k = nb - SKIP_ONE; k >= 0; k--) { /* nested ones come later in the blob */ char qn[TR_QN_MAX]; - const cbm_gbuf_node_t *n = tr_join(qn, sizeof(qn), src->module_qn, best[k]->path, name) - ? tr_node(c, qn, src) - : NULL; + const cbm_gbuf_node_t *n = + tr_join(qn, sizeof(qn), src->sym_qn, best[k]->path, name) ? tr_node(c, qn, src) : NULL; if (n) { tr_res_t r = tr_mk(TR_NODE); r.node = n; diff --git a/src/pipeline/path_alias.c b/src/pipeline/path_alias.c index ef999e14d..7ef54c369 100644 --- a/src/pipeline/path_alias.c +++ b/src/pipeline/path_alias.c @@ -317,11 +317,13 @@ void cbm_path_alias_collection_free(cbm_path_alias_collection_t *coll) { free(coll); } -char *cbm_path_alias_resolve(const cbm_path_alias_map_t *map, const char *module_path) { - if (!map || !module_path) { - return NULL; +bool cbm_path_alias_resolve_into(const cbm_path_alias_map_t *map, const char *module_path, + char *out, size_t cap) { + if (!map || !module_path || !out || cap == 0) { + return false; } size_t mod_len = strlen(module_path); + int n = CBM_NOT_FOUND; for (int i = 0; i < map->count; i++) { const cbm_path_alias_t *e = &map->entries[i]; @@ -339,41 +341,35 @@ char *cbm_path_alias_resolve(const cbm_path_alias_map_t *map, const char *module strcmp(module_path + mod_len - suffix_len, e->alias_suffix) != 0) { continue; } - size_t wild_len = mod_len - prefix_len - suffix_len; - const char *wild_start = module_path + prefix_len; - size_t tp_len = strlen(e->target_prefix); - size_t ts_len = strlen(e->target_suffix); - char *result = malloc(tp_len + wild_len + ts_len + 1); - if (!result) { - return NULL; - } - memcpy(result, e->target_prefix, tp_len); - memcpy(result + tp_len, wild_start, wild_len); - memcpy(result + tp_len + wild_len, e->target_suffix, ts_len); - result[tp_len + wild_len + ts_len] = '\0'; - return strip_resolved_ext(result); + int wild_len = (int)(mod_len - prefix_len - suffix_len); + n = snprintf(out, cap, "%s%.*s%s", e->target_prefix, wild_len, module_path + prefix_len, + e->target_suffix); + break; } if (strcmp(module_path, e->alias_prefix) == 0) { - return strip_resolved_ext(strdup(e->target_prefix)); + n = snprintf(out, cap, "%s", e->target_prefix); + break; } } /* baseUrl fallback. Apply only to non-relative imports that look * sub-path-ish (contain '/' but don't start with '.' or '@'); skips * obvious package names like "react" or "lodash". */ - if (map->base_url && module_path[0] != '.' && module_path[0] != '@' && + if (n < 0 && map->base_url && module_path[0] != '.' && module_path[0] != '@' && strchr(module_path, '/') != NULL) { - size_t bu_len = strlen(map->base_url); - size_t need = bu_len + 1 + mod_len + 1; - char *result = malloc(need); - if (!result) { - return NULL; - } - snprintf(result, need, "%s/%s", map->base_url, module_path); - return strip_resolved_ext(result); + n = snprintf(out, cap, "%s/%s", map->base_url, module_path); } - return NULL; + if (n < 0 || (size_t)n >= cap) { + return false; + } + strip_resolved_ext(out); + return true; +} + +char *cbm_path_alias_resolve(const cbm_path_alias_map_t *map, const char *module_path) { + char buf[CBM_PATH_ALIAS_RESULT_MAX]; + return cbm_path_alias_resolve_into(map, module_path, buf, sizeof(buf)) ? strdup(buf) : NULL; } /* ── Repo walk ─────────────────────────────────────────────────── */ diff --git a/src/pipeline/path_alias.h b/src/pipeline/path_alias.h index bc462e524..1f981c79e 100644 --- a/src/pipeline/path_alias.h +++ b/src/pipeline/path_alias.h @@ -93,4 +93,10 @@ const cbm_path_alias_map_t *cbm_path_alias_find_for_file(const cbm_path_alias_co * pipeline can consume it. */ char *cbm_path_alias_resolve(const cbm_path_alias_map_t *map, const char *module_path); +/* The same, written into out (cap bytes, NUL-terminated) instead of a heap + * string; false when no alias applies or the path does not fit. */ +enum { CBM_PATH_ALIAS_RESULT_MAX = 4096 }; +bool cbm_path_alias_resolve_into(const cbm_path_alias_map_t *map, const char *module_path, + char *out, size_t cap); + #endif /* CBM_PATH_ALIAS_H */ diff --git a/tests/test_doc_mentions_ts.c b/tests/test_doc_mentions_ts.c index 32581e10d..c48c6bf7f 100644 --- a/tests/test_doc_mentions_ts.c +++ b/tests/test_doc_mentions_ts.c @@ -144,7 +144,8 @@ static void ts_write_repo(const char *repo) { "function internal() {}\n"); th_write_file(TH_PATH(repo, "src/def.ts"), "export default class Def {}\n"); th_write_file(TH_PATH(repo, "src/barrel/index.ts"), - "export * from \"../helper\";\nexport * from \"./more\";\n"); + "export * from \"../helper\";\nexport * from \"./more\";\n" + "export function inBarrel() {}\n"); th_write_file(TH_PATH(repo, "src/barrel/more.ts"), "export function extra() {}\n"); th_write_file(TH_PATH(repo, "src/dup1.ts"), "export function twin() {}\n"); th_write_file(TH_PATH(repo, "src/dup2.ts"), "export function twin() {}\n"); @@ -164,7 +165,17 @@ static void ts_write_repo(const char *repo) { "}\n"); /* a script: no import, no export, so its names are global */ th_write_file(TH_PATH(repo, "src/global.js"), "function legacyGlobal() {}\n"); + /* a tsconfig path alias and a workspace package */ + th_write_file(TH_PATH(repo, "tsconfig.json"), + "{ \"compilerOptions\": { \"paths\": { \"@lib/*\": [\"src/lib/*\"] } } }\n"); + th_write_file(TH_PATH(repo, "src/lib/aliased.ts"), "export function aliased() {}\n"); + th_write_file(TH_PATH(repo, "packages/util/package.json"), + "{ \"name\": \"@acme/util\", \"main\": \"src/index.ts\" }\n"); + th_write_file(TH_PATH(repo, "packages/util/src/index.ts"), "export function utilFn() {}\n"); th_write_file(TH_PATH(repo, "src/widget.ts"), + "import { aliased } from \"@lib/aliased\";\n" + "import { utilFn } from \"@acme/util\";\n" + "import { inBarrel } from \"./barrel\";\n" "import { Helper, gadget as g } from \"./helper\";\n" "import * as ns from \"./helper\";\n" "import Def from \"./def\";\n" @@ -177,7 +188,8 @@ static void ts_write_repo(const char *repo) { " * Links: {@link Helper}, {@link Helper.run}, {@link g}, {@link ns.gadget},\n" " * {@link Def}, {@link extra}, {@link twin}, {@link Thing}, {@link Gone},\n" " * {@link Promise}, {@link legacyGlobal}, {@link Nope}, {@link spin},\n" - " * {@link Space.Inner.deep}, {@link Helper#missingMember}.\n" + " * {@link Space.Inner.deep}, {@link Helper#missingMember}, {@link aliased},\n" + " * {@link utilFn}, {@link inBarrel}.\n" " */\n" "export class Widget {\n" " spin(): void {}\n" @@ -247,6 +259,11 @@ TEST(doc_mentions_ts_resolution) { ASSERT_TRUE(ts_edge(db, w, "src.widget.Space.Inner.deep")); /* a class member without a node of its own */ ASSERT_TRUE(ts_row(db, "Helper#missingMember", "graph_gap")); + /* a tsconfig path alias; a workspace package's entry module */ + ASSERT_TRUE(ts_edge(db, w, "src.lib.aliased.aliased")); + ASSERT_TRUE(ts_edge(db, w, "packages.util.src.utilFn")); + /* a declaration of an index file (its QN has no `index`) */ + ASSERT_TRUE(ts_edge(db, w, "src.barrel.inBarrel")); /* this. is the container's; a namespace's own names first */ ASSERT_TRUE(ts_edge(db, "src.widget.Widget.turn", "src.widget.Widget.spin")); ASSERT_TRUE(ts_edge(db, "src.widget.Space.Inner.deep", "src.widget.Space.Inner.deep2")); From f4a49feba1a8325c4751de130b5166fb3f3fe986 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 11 Oct 2026 02:08:58 +0200 Subject: [PATCH 12/17] feat(docs): diagrams and OpenAPI operations in the graph Diagrams name the parts of a system and how they connect; an OpenAPI document names its endpoints and their handlers. Both now enter the graph. Diagrams: - a Diagram node per diagram: a fenced block of a Markdown file (under its Section), a directive of a reST file (`.. mermaid::`, `.. uml::`, `.. graphviz::`, Sphinx `.. digraph::`), an Asciidoctor Diagram block of an AsciiDoc file (`[plantuml]`, `[source,mermaid]`, ...), or a diagram file of its own, named by its title, else by its file; - a DiagramElement node per element, DEFINES from its Diagram, and a DIAGRAM_ARROW edge per arrow (label, style, line); - formats: Mermaid (flowchart, sequence, class, ER, state, C4, architecture), PlantUML, Graphviz DOT, D2, draw.io (plain, compressed, .drawio.svg) and Excalidraw (.excalidraw, .excalidraw.json, .excalidraw.svg); .drawio.svg and .excalidraw.svg are no longer dropped as images; - an element's explicit link (Mermaid click/link, PlantUML [[url]], url of and !include, draw.io and Excalidraw link, DOT URL/href, D2 link) is a doc link read as a Markdown link, MENTIONS when it resolves. OpenAPI: - each operation of an OpenAPI or Swagger document (openapi.json and swagger.json, now read with the JSON grammar but no node per key, and any YAML or JSON file whose root says openapi/swagger) is a Route, named by the same QN a route its handler registers in code gets (cbm_route_canon_path, moved into the extraction library); - its operationId is a doc link to the handler: a dotted one (Connexion, with x-openapi-router-controller) as a qualified code name, a bare one when exactly one function or method of the repository has that name. The eight new link families are held until their held-out audit passes. Read on real repositories, which shaped the rules: Excalidraw drawings put text over shapes and draw arrows up to them without binding, so loose text names the smallest box it sits in and a loose arrow end meets the smallest named box it touches (the drawing's frame only at its border); arrows bound to free text meet the text; D2 block strings and nested blocks; DOT line breaks and sentence-long ids; PlantUML kinds of classes inside packages and nested classes (`..+`, nested -> container, so they do not collide with the container's association). Connexion's 24 specs: 160 operationIds, all linked to their handlers. Signed-off-by: Martin Vogel --- Makefile.cbm | 6 +- internal/cbm/cbm.c | 31 + internal/cbm/cbm.h | 43 + internal/cbm/diagram.c | 3544 ++++++++++++++++++++++++ internal/cbm/doclink.h | 166 +- internal/cbm/lang_specs.c | 11 + internal/cbm/openapi.c | 239 ++ internal/cbm/result_compact.c | 9 + internal/cbm/route_path.c | 98 + src/discover/discover.c | 8 + src/discover/language.c | 35 +- src/discover/userconfig.c | 2 + src/pipeline/doc_links.c | 20 +- src/pipeline/doc_links.h | 2 +- src/pipeline/doc_links_adoc.c | 5 + src/pipeline/doc_links_md.c | 48 +- src/pipeline/doc_links_rst.c | 5 + src/pipeline/pass_definitions.c | 3 + src/pipeline/pass_diagrams.c | 68 + src/pipeline/pass_parallel.c | 4 + src/pipeline/pass_route_nodes.c | 89 - src/pipeline/pipeline.c | 2 + src/pipeline/pipeline_internal.h | 17 +- tests/repro/repro_call_node_manifest.c | 3 +- tests/repro/repro_language_registry.c | 8 +- tests/test_diagrams.c | 1065 +++++++ tests/test_doc_mentions_helpers.h | 47 +- tests/test_grammar_labels.c | 2 +- tests/test_main.c | 4 + tests/test_openapi.c | 242 ++ 30 files changed, 5621 insertions(+), 205 deletions(-) create mode 100644 internal/cbm/diagram.c create mode 100644 internal/cbm/openapi.c create mode 100644 internal/cbm/route_path.c create mode 100644 src/pipeline/pass_diagrams.c create mode 100644 tests/test_diagrams.c create mode 100644 tests/test_openapi.c diff --git a/Makefile.cbm b/Makefile.cbm index ffb0cb0e7..4a3e45d59 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -393,6 +393,9 @@ EXTRACTION_SRCS = \ $(CBM_DIR)/doclink_ts.c \ $(CBM_DIR)/doclink_php.c \ $(CBM_DIR)/doclink_c.c \ + $(CBM_DIR)/diagram.c \ + $(CBM_DIR)/route_path.c \ + $(CBM_DIR)/openapi.c \ $(CBM_DIR)/sql_values.c \ $(CBM_DIR)/helpers.c \ $(CBM_DIR)/cpp_branch_views.c \ @@ -526,6 +529,7 @@ PIPELINE_SRCS = \ src/pipeline/doc_links_ts.c \ src/pipeline/doc_links_php.c \ src/pipeline/doc_links_c.c \ + src/pipeline/pass_diagrams.c \ src/pipeline/doc_links_msbuild.c \ src/pipeline/pass_route_nodes.c \ src/pipeline/pass_enrichment.c \ @@ -788,7 +792,7 @@ TEST_DISCOVER_SRCS = \ TEST_GRAPH_BUFFER_SRCS = tests/test_graph_buffer.c -TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_pipeline_conventions.c tests/test_pipeline_frozen.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_doclinks.c tests/test_doc_mentions.c tests/test_doc_links_md.c tests/test_doc_links_pdf.c tests/test_doc_links_rst.c tests/test_doc_links_adoc.c tests/test_doc_mentions_go.c tests/test_doc_mentions_jvm.c tests/test_doc_mentions_rs.c tests/test_doc_mentions_perl.c tests/test_doc_mentions_ts.c tests/test_doc_mentions_php.c tests/test_doc_mentions_c.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c +TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_pipeline_conventions.c tests/test_pipeline_frozen.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_doclinks.c tests/test_doc_mentions.c tests/test_doc_links_md.c tests/test_doc_links_pdf.c tests/test_doc_links_rst.c tests/test_doc_links_adoc.c tests/test_doc_mentions_go.c tests/test_doc_mentions_jvm.c tests/test_doc_mentions_rs.c tests/test_doc_mentions_perl.c tests/test_doc_mentions_ts.c tests/test_doc_mentions_php.c tests/test_doc_mentions_c.c tests/test_diagrams.c tests/test_openapi.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c TEST_WATCHER_SRCS = tests/test_watcher.c diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index b9e3237b2..d60c9a9bd 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -277,6 +277,11 @@ void cbm_channels_push(CBMChannelArray *arr, CBMArena *a, CBMChannel ch) { arr->items[arr->count++] = ch; } +void cbm_diagram_arrows_push(CBMDiagramArrowArray *arr, CBMArena *a, CBMDiagramArrow da) { + GROW_ARRAY(arr, a); + arr->items[arr->count++] = da; +} + // --- String input reader (for parse_with_options) --- typedef struct { @@ -2837,6 +2842,26 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C .module_qn = result->module_qn, }; cbm_adoc_extract_document(&adoc_ctx); + cbm_extract_diagrams(&adoc_ctx); /* after the Sections they hang under */ + cbm_index_mark_done(rel_path); + return result; + } + /* Diagram files have no grammar either: each format is read from the text + * (diagram.c). */ + if (language == CBM_LANG_DIAGRAM) { + result->module_qn = cbm_fqn_module_source_lang(a, project, rel_path, language); + CBMExtractCtx diagram_ctx = { + .arena = a, + .scratch = scratch, + .result = result, + .source = source, + .source_len = source_len, + .language = language, + .project = project, + .rel_path = rel_path, + .module_qn = result->module_qn, + }; + cbm_extract_diagram_file(&diagram_ctx); cbm_index_mark_done(rel_path); return result; } @@ -3021,6 +3046,12 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C // Channel detection (Socket.IO / EventEmitter) — JS/TS only. cbm_extract_channels(&ctx); + // Diagrams: blocks of documents and diagram files (self-gated by language). + cbm_extract_diagrams(&ctx); + + // OpenAPI operations as Routes (self-gated: API description documents). + cbm_extract_openapi(&ctx); + // K8s / Kustomize semantic pass (additional structured extraction for YAML-based infra files). if (ctx.language == CBM_LANG_KUSTOMIZE || ctx.language == CBM_LANG_K8S) { cbm_extract_k8s(&ctx); diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index 6cc5d4743..7d3bf15a0 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -211,6 +211,10 @@ typedef enum { CBM_LANG_CHIALISP, // Chialisp (.clsp/.clib/.clinc — Chia smart-coin s-expression language) CBM_LANG_PDF, // PDF documents: the text layer, read without a grammar (doc_pdf.c) CBM_LANG_ASCIIDOC, // AsciiDoc documents, read without a grammar (doc_adoc.c) + CBM_LANG_OPENAPI, // openapi.json / swagger.json: the JSON grammar, no node per key; their + // operations are Routes (openapi.c) + CBM_LANG_DIAGRAM, // diagram files (PlantUML, DOT, D2, draw.io, Excalidraw), read + // without a grammar (diagram.c) CBM_LANG_COUNT } CBMLanguage; @@ -612,6 +616,23 @@ typedef struct { int cap; } CBMChannelArray; +/* One arrow of a diagram (diagram.c): from one DiagramElement definition of + * the file to another, written as a DIAGRAM_ARROW edge. */ +typedef struct { + const char *from_qn; + const char *to_qn; + const char *label; /* the arrow's text, NULL for none */ + const char *style; /* what the arrow means as its format writes it: "flow", + * "message", "inherits", "composition", ... */ + uint32_t line; +} CBMDiagramArrow; + +typedef struct { + CBMDiagramArrow *items; + int count; + int cap; +} CBMDiagramArrowArray; + typedef struct { CBMFieldType *items; int count; @@ -661,6 +682,7 @@ typedef struct CBMFileResult { CBMStringRefArray string_refs; // URL/config string literals from AST CBMInfraBindingArray infra_bindings; // topic→URL pairs from IaC configs CBMChannelArray channels; // Socket.IO / EventEmitter pub/sub participation + CBMDiagramArrowArray diagram_arrows; // arrows between the file's DiagramElement defs CBMFieldTypeArray field_types; // Python: annotated instance fields (#1277) const char *module_qn; // module qualified name @@ -1122,6 +1144,7 @@ void cbm_infrabinding_push(CBMInfraBindingArray *arr, CBMArena *a, CBMInfraBindi void cbm_impltrait_push(CBMImplTraitArray *arr, CBMArena *a, CBMImplTrait it); void cbm_resolvedcall_push(CBMResolvedCallArray *arr, CBMArena *a, CBMResolvedCall rc); void cbm_channels_push(CBMChannelArray *arr, CBMArena *a, CBMChannel ch); +void cbm_diagram_arrows_push(CBMDiagramArrowArray *arr, CBMArena *a, CBMDiagramArrow da); // --- Sub-extractor entry points --- @@ -1140,6 +1163,26 @@ void cbm_extract_type_refs(CBMExtractCtx *ctx); void cbm_extract_env_accesses(CBMExtractCtx *ctx); void cbm_extract_type_assigns(CBMExtractCtx *ctx); void cbm_extract_channels(CBMExtractCtx *ctx); +/* Diagrams (diagram.c): a Diagram definition per diagram -- a Markdown, reST + * or AsciiDoc block, or a diagram file -- with a DiagramElement definition per + * element and the arrows between them. */ +void cbm_extract_diagrams(CBMExtractCtx *ctx); +/* A diagram file (CBM_LANG_DIAGRAM): its format comes from its extension. */ +void cbm_extract_diagram_file(CBMExtractCtx *ctx); + +/* OpenAPI / Swagger documents (openapi.c): a Route definition per operation and + * a doc link per operationId. Self-gated: openapi.json / swagger.json, and YAML + * or JSON files whose root says `openapi` or `swagger`. */ +void cbm_extract_openapi(CBMExtractCtx *ctx); + +/* Canonicalize route-path parameter placeholders (":id", "{id}", "", + * "${...}") to a single "{}" token so that client call sites, server handlers + * and OpenAPI operations rendezvous on the same Route QN regardless of + * framework syntax. Parameter names are intentionally discarded ("/u/{id}" and + * "/u/{slug}" both canonicalize to "/u/{}"). The result never exceeds the + * input length, so out_sz >= strlen(in) + 1 always suffices. Returns out. + * (route_path.c) */ +const char *cbm_route_canon_path(const char *in, char *out, size_t out_sz); // Single-pass unified extraction (replaces the 7 calls above except defs+imports). void cbm_extract_unified(CBMExtractCtx *ctx); diff --git a/internal/cbm/diagram.c b/internal/cbm/diagram.c new file mode 100644 index 000000000..228c3ca67 --- /dev/null +++ b/internal/cbm/diagram.c @@ -0,0 +1,3544 @@ +/* + * diagram.c — diagrams as graph nodes: a Diagram definition per diagram and a + * DiagramElement definition per element, with the arrows between elements + * (CBMDiagramArrow, written as DIAGRAM_ARROW edges by pass_diagrams.c). + * + * Where diagrams come from: + * - a fenced block of a Markdown file whose info string names a diagram + * language (mermaid; plantuml, puml, uml; dot, graphviz; d2): the Diagram + * hangs under the Section the block is written in (the last heading above + * it), or under the file before any heading; + * - a Mermaid file (.mmd, .mermaid): one Diagram, under the file. + * + * Qualified names: the parent's (a Section's, else the file's module QN) + + * "#diagram-", k counting the diagrams of that parent from 1; an element's + * is the diagram's + "." + its id (dots and blanks as `_`). + * + * A Diagram's docstring is the text its elements and arrows carry (their + * labels), which is what a reader of the diagram reads; its properties hold + * the format and the diagram kind. An element's name is its label (its id + * when it has none); its properties hold the id and the element kind. An + * element named only by an arrow is an element too. + * + * Formats (one parser each, line by line, no grammar): Mermaid flowchart and + * graph, sequenceDiagram, classDiagram, erDiagram, stateDiagram, C4 (Context, + * Container, Component, Dynamic, Deployment) and architecture-beta. Every + * other Mermaid kind (gantt, pie, journey, mindmap, ...) is a Diagram without + * elements. + */ +#include "cbm.h" +#include "doclink.h" + +#include "arena.h" +#include "helpers.h" +#include "foundation/constants.h" +#include "foundation/hash_table.h" +#include "foundation/mem_core.h" +#include "tree_sitter/api.h" + +#include +#include + +#include +#include +#include +#include +#include +#include + +enum { + DG_DOC_MAX = 2048, /* a Diagram's docstring */ + DG_LABEL_MAX = 256, /* an element's or an arrow's label */ + DG_ID_MAX = 128, + DG_ID_HEAD = DG_ID_MAX - 18, /* a long id's head: `~` and 16 hex digits follow */ + DG_KIND_MAX = 32, + DG_ARGS_MAX = 8, +}; + +static const uint64_t DG_FNV_OFFSET = 0xcbf29ce484222325ULL; /* FNV-1a 64 */ +static const uint64_t DG_FNV_PRIME = 0x100000001b3ULL; +enum { DG_UTF8_CONT_MASK = 0xC0, DG_UTF8_CONT = 0x80 }; /* a continuation byte: 10xxxxxx */ + +/* ── The builder ─────────────────────────────────────────────────── */ + +typedef struct { + CBMExtractCtx *ctx; + const char *qn; /* the diagram's */ + const char *format; + char kind[DG_KIND_MAX]; + const char *element_kind; /* what an element named only by an arrow is */ + int def_index; /* the Diagram definition, in ctx->result->defs */ + uint32_t line0; /* the source line of the diagram's text */ + CBMHashTable *ids; /* element id -> its definition's index + 1 */ + char doc[DG_DOC_MAX]; /* the labels, for the docstring */ + size_t doc_len; + bool oom; +} dg_t; + +typedef struct { + const char *p; + size_t n; +} dg_str_t; + +static dg_str_t dg_s(const char *p, size_t n) { + return (dg_str_t){p, n}; +} + +static dg_str_t dg_trim(dg_str_t s) { + while (s.n && isspace((unsigned char)s.p[0])) { + s.p++; + s.n--; + } + while (s.n && isspace((unsigned char)s.p[s.n - SKIP_ONE])) { + s.n--; + } + return s; +} + +/* Quotes and Markdown-string backticks around a label dropped. */ +static dg_str_t dg_unquote(dg_str_t s) { + s = dg_trim(s); + if (s.n >= PAIR_LEN && (s.p[0] == '"' || s.p[0] == '\'' || s.p[0] == '`') && + s.p[s.n - SKIP_ONE] == s.p[0]) { + s.p++; + s.n -= PAIR_LEN; + } + if (s.n >= PAIR_LEN && s.p[0] == '`' && s.p[s.n - SKIP_ONE] == '`') { + s.p++; + s.n -= PAIR_LEN; + } + return dg_trim(s); +} + +static bool dg_eq(dg_str_t s, const char *lit) { + return strlen(lit) == s.n && memcmp(s.p, lit, s.n) == 0; +} + +static bool dg_starts(dg_str_t s, const char *lit) { + size_t l = strlen(lit); + return s.n >= l && memcmp(s.p, lit, l) == 0; +} + +/* A label's text: tags (`
`), the line breaks DOT, PlantUML and D2 write + * as `\n` (also `\l`, `\r`, `\t`) and blank runs as one space, at most cap-1 + * bytes. */ +static size_t dg_label_text(dg_str_t s, char *out, size_t cap) { + size_t w = 0; + bool space = false; + for (size_t i = 0; i < s.n && w + SKIP_ONE < cap; i++) { + char c = s.p[i]; + if (c == '\\' && i + SKIP_ONE < s.n && strchr("nlrt", s.p[i + SKIP_ONE]) && + s.p[i + SKIP_ONE]) { + i++; + space = true; + continue; + } + if (c == '<') { + const char *gt = memchr(s.p + i, '>', s.n - i); + if (gt) { + i = (size_t)(gt - s.p); + space = true; + continue; + } + } + if (isspace((unsigned char)c)) { + space = true; + continue; + } + if (space && w) { + out[w++] = ' '; + } + space = false; + out[w++] = c; + } + out[w] = '\0'; + return w; +} + +static void dg_doc_add(dg_t *dg, const char *text, size_t n) { + if (!n || dg->doc_len + n + PAIR_LEN >= sizeof(dg->doc)) { + return; + } + if (dg->doc_len) { + memcpy(dg->doc + dg->doc_len, "; ", PAIR_LEN); + dg->doc_len += PAIR_LEN; + } + memcpy(dg->doc + dg->doc_len, text, n); + dg->doc_len += n; + dg->doc[dg->doc_len] = '\0'; +} + +static const char *dg_dup(dg_t *dg, const char *s, size_t n) { + char *c = cbm_arena_strndup(dg->ctx->arena, s, n); + dg->oom = dg->oom || !c; + return c; +} + +static const char **dg_props(dg_t *dg, const char *k1, const char *v1, const char *k2, + const char *v2) { + const char **kv = (const char **)cbm_arena_alloc(dg->ctx->arena, CBM_SZ_5 * sizeof(char *)); + if (!kv) { + dg->oom = true; + return NULL; + } + kv[0] = k1; + kv[SKIP_ONE] = v1; + kv[PAIR_LEN] = k2; + kv[CBM_SZ_3] = v2; + kv[CBM_SZ_4] = NULL; + return kv; +} + +/* An element id as a QN segment: dots, blanks, backslashes, quotes and `#` + * (QN suffix syntax) as `_`. */ +static const char *dg_element_qn(dg_t *dg, dg_str_t id) { + char seg[DG_ID_MAX]; + size_t n = id.n < sizeof(seg) - SKIP_ONE ? id.n : sizeof(seg) - SKIP_ONE; + for (size_t i = 0; i < n; i++) { + char c = id.p[i]; + seg[i] = + (c == '.' || c == '\\' || c == '"' || c == '#' || isspace((unsigned char)c)) ? '_' : c; + } + seg[n] = '\0'; + char *qn = cbm_arena_sprintf(dg->ctx->arena, "%s.%s", dg->qn, seg); + dg->oom = dg->oom || !qn; + return qn; +} + +/* An element id as its table key and QN source. An id too long for one (a + * DOT node named by a sentence) keeps its head, cut at a character boundary, + * and the FNV-1a 64 of the whole: still one key per id. */ +static size_t dg_id_key(dg_str_t id, char *key) { + if (id.n < DG_ID_MAX) { + memcpy(key, id.p, id.n); + key[id.n] = '\0'; + return id.n; + } + uint64_t h = DG_FNV_OFFSET; + for (size_t i = 0; i < id.n; i++) { + h = (h ^ (unsigned char)id.p[i]) * DG_FNV_PRIME; + } + size_t head = DG_ID_HEAD; + while (head > 0 && ((unsigned char)id.p[head] & DG_UTF8_CONT_MASK) == DG_UTF8_CONT) { + head--; /* not inside a UTF-8 sequence */ + } + int n = snprintf(key, DG_ID_MAX, "%.*s~%016llx", (int)head, id.p, (unsigned long long)h); + return n > 0 ? (size_t)n : 0; +} + +/* An explicit link written on an element (or, for a PlantUML !include, on the + * diagram): a doc-link record the pipeline resolves as a Markdown link. An + * in-page anchor, a data URI or a script links nothing. */ +static void dg_link(dg_t *dg, const char *source_qn, dg_str_t raw, int syntax, uint32_t line) { + raw = dg_unquote(raw); + if (!source_qn || !raw.n || raw.p[0] == '#' || dg_starts(raw, "data:") || + dg_starts(raw, "javascript:")) { + return; + } + CBMDocLink link = { + .source_qn = source_qn, + .raw = dg_dup(dg, raw.p, raw.n), + .line = line, + .def_line = line, + .syntax = (uint16_t)syntax, + }; + if (link.raw) { + cbm_doclinks_push(&dg->ctx->result->doc_links, dg->ctx->arena, link); + } +} + +/* The element `id` (made on first sight), its label set when it has none. */ +static const char *dg_element(dg_t *dg, dg_str_t id, dg_str_t label, const char *kind, + uint32_t line) { + id = dg_trim(id); + if (!id.n || dg->oom) { + return NULL; + } + char key[DG_ID_MAX]; + size_t kn = dg_id_key(id, key); + char text[DG_LABEL_MAX]; + size_t tl = dg_label_text(dg_unquote(label), text, sizeof(text)); + CBMDefArray *defs = &dg->ctx->result->defs; + void *v = cbm_ht_get(dg->ids, key); + if (v) { + CBMDefinition *d = &defs->items[(uintptr_t)v - SKIP_ONE]; + if (tl && d->name == d->extra_props[SKIP_ONE]) { /* still named by its id */ + d->name = dg_dup(dg, text, tl); + dg_doc_add(dg, text, tl); + } + return d->qualified_name; + } + const char *qn = dg_element_qn(dg, dg_s(key, kn)); + const char *idc = dg_dup(dg, id.p, id.n); + if (!qn || !idc) { + return NULL; + } + CBMDefinition def; + memset(&def, 0, sizeof(def)); + def.name = tl ? dg_dup(dg, text, tl) : idc; + def.qualified_name = qn; + def.label = "DiagramElement"; + def.file_path = dg->ctx->rel_path; + def.start_line = line; + def.end_line = line; + def.parent_class = dg->qn; + /* the kind may live in a parser buffer (a JSON value, a draw.io cell) */ + const char *kindc = dg_dup(dg, kind, strlen(kind)); + def.extra_props = dg_props(dg, "element_id", idc, "element_kind", kindc); + if (dg->oom || !def.name) { + return NULL; + } + cbm_defs_push(defs, dg->ctx->arena, def); + const char *k = dg_dup(dg, key, kn); + if (k) { + cbm_ht_set(dg->ids, k, (void *)(uintptr_t)defs->count); + } + if (tl) { + dg_doc_add(dg, text, tl); + } + return qn; +} + +static void dg_arrow(dg_t *dg, dg_str_t from, dg_str_t to, dg_str_t label, const char *style, + uint32_t line) { + const char *kind = dg->element_kind ? dg->element_kind : "node"; + const char *a = dg_element(dg, from, dg_s(NULL, 0), kind, line); + const char *b = dg_element(dg, to, dg_s(NULL, 0), kind, line); + if (!a || !b) { + return; + } + char text[DG_LABEL_MAX]; + size_t tl = dg_label_text(dg_unquote(label), text, sizeof(text)); + CBMDiagramArrow arrow = {.from_qn = a, + .to_qn = b, + .label = tl ? dg_dup(dg, text, tl) : NULL, + .style = style, + .line = line}; + if (!dg->oom) { + cbm_diagram_arrows_push(&dg->ctx->result->diagram_arrows, dg->ctx->arena, arrow); + } + if (tl) { + dg_doc_add(dg, text, tl); + } +} + +/* ── Lines ───────────────────────────────────────────────────────── */ + +typedef struct { + const char *p; + const char *end; + uint32_t line; +} dg_lines_t; + +/* The next line (no newline), false at the end. */ +static bool dg_next_line(dg_lines_t *it, dg_str_t *out, uint32_t *line) { + if (it->p >= it->end) { + return false; + } + const char *nl = memchr(it->p, '\n', (size_t)(it->end - it->p)); + const char *le = nl ? nl : it->end; + *out = dg_s(it->p, (size_t)(le - it->p)); + if (out->n && out->p[out->n - SKIP_ONE] == '\r') { + out->n--; + } + *line = it->line++; + it->p = nl ? nl + SKIP_ONE : it->end; + return true; +} + +/* A Mermaid line without its `%%` comment. */ +static dg_str_t dg_mermaid_strip(dg_str_t s) { + for (size_t i = 0; i + SKIP_ONE < s.n; i++) { + if (s.p[i] == '%' && s.p[i + SKIP_ONE] == '%') { + s.n = i; + break; + } + } + return dg_trim(s); +} + +/* ── Mermaid: flowchart ──────────────────────────────────────────── */ + +static bool dg_id_char(char c) { + return isalnum((unsigned char)c) || c == '_' || c == '-' || (unsigned char)c >= 0x80; +} + +/* The closing bracket of a node shape opened at s[i]: the shape's text in + * *label, the position after it. */ +static size_t dg_shape(dg_str_t s, size_t i, dg_str_t *label) { + static const char *const opens[] = {"(((", "([", "[[", "[(", "((", "{{", "[/", + "[\\", "[", "(", "{", ">", NULL}; + static const char *const closes[] = {")))", "])", "]]", ")]", "))", "}}", "/]", + "\\]", "]", ")", "}", "]", NULL}; + for (int k = 0; opens[k]; k++) { + size_t ol = strlen(opens[k]); + if (i + ol > s.n || memcmp(s.p + i, opens[k], ol) != 0) { + continue; + } + size_t cl = strlen(closes[k]); + bool quoted = false; + for (size_t j = i + ol; j + cl <= s.n; j++) { + if (s.p[j] == '"') { + quoted = !quoted; + } + if (!quoted && memcmp(s.p + j, closes[k], cl) == 0) { + *label = dg_s(s.p + i + ol, j - i - ol); + return j + cl; + } + } + /* `[/text\]` and `[\text/]` close with the other slash */ + if (k == CBM_SZ_6 || k == CBM_SZ_7) { + const char *alt = k == CBM_SZ_6 ? "\\]" : "/]"; + for (size_t j = i + ol; j + PAIR_LEN <= s.n; j++) { + if (memcmp(s.p + j, alt, PAIR_LEN) == 0) { + *label = dg_s(s.p + i + ol, j - i - ol); + return j + PAIR_LEN; + } + } + } + return i; + } + return i; +} + +/* A node at s[i]: its id and label; the position after it (i when none). */ +static size_t dg_flow_node(dg_str_t s, size_t i, dg_str_t *id, dg_str_t *label) { + size_t j = i; + while (j < s.n && dg_id_char(s.p[j])) { + if (s.p[j] == '-' && j + SKIP_ONE < s.n && + (s.p[j + SKIP_ONE] == '-' || s.p[j + SKIP_ONE] == '.' || s.p[j + SKIP_ONE] == '>')) { + break; /* an edge starts */ + } + j++; + } + if (j == i) { + return i; + } + *id = dg_s(s.p + i, j - i); + *label = dg_s(NULL, 0); + if (j < s.n && s.p[j] == '@' && j + SKIP_ONE < s.n && s.p[j + SKIP_ONE] == '{') { + const char *close = memchr(s.p + j, '}', s.n - j); + if (close) { + dg_str_t body = dg_s(s.p + j + PAIR_LEN, (size_t)(close - s.p) - j - PAIR_LEN); + const char *lab = NULL; + for (size_t k = 0; k + CBM_SZ_6 <= body.n && !lab; k++) { + lab = memcmp(body.p + k, "label:", CBM_SZ_6) == 0 ? body.p + k + CBM_SZ_6 : NULL; + } + if (lab) { + const char *comma = memchr(lab, ',', (size_t)(body.p + body.n - lab)); + *label = dg_s(lab, (size_t)((comma ? comma : body.p + body.n) - lab)); + } + return (size_t)(close - s.p) + SKIP_ONE; + } + } + size_t k = dg_shape(s, j, label); + if (k > j) { + j = k; + } + if (j + CBM_SZ_3 <= s.n && memcmp(s.p + j, ":::", CBM_SZ_3) == 0) { + j += CBM_SZ_3; + while (j < s.n && dg_id_char(s.p[j])) { + j++; + } + } + return j; +} + +static size_t dg_skip_ws(dg_str_t s, size_t i) { + while (i < s.n && isspace((unsigned char)s.p[i])) { + i++; + } + return i; +} + +static bool dg_link_char(char c) { + return c == '-' || c == '=' || c == '.' || c == '>' || c == '<' || c == '~'; +} + +/* An edge at s[i]: its style and label; the position after it (i when none). */ +static size_t dg_flow_edge(dg_str_t s, size_t i, const char **style, dg_str_t *label) { + size_t j = i; + if (j < s.n && (s.p[j] == '<' || s.p[j] == 'x' || s.p[j] == 'o') && j + SKIP_ONE < s.n && + (s.p[j + SKIP_ONE] == '-' || s.p[j + SKIP_ONE] == '=')) { + j++; + } + size_t start = j; + while (j < s.n && dg_link_char(s.p[j])) { + j++; + } + size_t run = j - start; + if (run < PAIR_LEN) { + return i; + } + if (j < s.n && (s.p[j] == 'x' || s.p[j] == 'o') && + (j + SKIP_ONE >= s.n || !dg_id_char(s.p[j + SKIP_ONE]))) { + j++; /* an arrowhead x or o */ + } + dg_str_t op = dg_s(s.p + i, j - i); + *label = dg_s(NULL, 0); + bool head = + memchr(op.p, '>', op.n) || op.p[op.n - SKIP_ONE] == 'x' || op.p[op.n - SKIP_ONE] == 'o'; + if (!head && (dg_eq(op, "--") || dg_eq(op, "==") || dg_eq(op, "-."))) { + /* `-- text -->`, `== text ==>`, `-. text .->`: the label runs to the + * closing link, which opens with `--`, `==` or `.-` */ + char c0 = op.p[0] == '-' && op.p[SKIP_ONE] == '.' ? '.' : op.p[0]; + char c1 = op.p[0] == '-' && op.p[SKIP_ONE] == '.' ? '-' : op.p[SKIP_ONE]; + for (size_t k = j; k + SKIP_ONE < s.n; k++) { + if (s.p[k] == c0 && s.p[k + SKIP_ONE] == c1) { + size_t e = k; + while (e < s.n && dg_link_char(s.p[e])) { + e++; + } + *label = dg_s(s.p + j, k - j); + op = dg_s(s.p + i, e - i); + j = e; + break; + } + } + } + bool dotted = memchr(op.p, '.', op.n) != NULL; + bool thick = memchr(op.p, '=', op.n) != NULL; + bool both = op.p[0] == '<' && memchr(op.p + SKIP_ONE, '>', op.n - SKIP_ONE); + head = memchr(op.p, '>', op.n) != NULL; + *style = both ? "bidirectional" : dotted ? "dotted" : thick ? "thick" : head ? "arrow" : "line"; + j = dg_skip_ws(s, j); + if (j < s.n && s.p[j] == '|') { + const char *bar = memchr(s.p + j + SKIP_ONE, '|', s.n - j - SKIP_ONE); + if (bar) { + *label = dg_s(s.p + j + SKIP_ONE, (size_t)(bar - s.p) - j - SKIP_ONE); + j = (size_t)(bar - s.p) + SKIP_ONE; + } + } + return j; +} + +enum { DG_GROUP_MAX = 16 }; + +typedef struct { + dg_str_t ids[DG_GROUP_MAX]; + int n; +} dg_group_t; + +/* `A & B[label]`: the nodes of a group, made; the position after it. */ +static size_t dg_flow_group(dg_t *dg, dg_str_t s, size_t i, uint32_t line, dg_group_t *g) { + g->n = 0; + for (;;) { + i = dg_skip_ws(s, i); + dg_str_t id; + dg_str_t label; + size_t j = dg_flow_node(s, i, &id, &label); + if (j == i) { + return i; + } + dg_element(dg, id, label, "node", line); + if (g->n < DG_GROUP_MAX) { + g->ids[g->n++] = id; + } + i = dg_skip_ws(s, j); + if (i < s.n && s.p[i] == '&') { + i++; + continue; + } + return i; + } +} + +/* `click A "url"`, `click A href "url"` (a class diagram's `link A "url"`): + * the element's link; a callback (`click A call f()`) links nothing. */ +static void dg_mermaid_click(dg_t *dg, dg_str_t s, uint32_t line) { + size_t kw = dg_starts(s, "click ") ? CBM_SZ_6 : CBM_SZ_5; + dg_str_t rest = dg_trim(dg_s(s.p + kw, s.n - kw)); + size_t j = 0; + while (j < rest.n && !isspace((unsigned char)rest.p[j])) { + j++; + } + dg_str_t tail = dg_trim(dg_s(rest.p + j, rest.n - j)); + if (dg_starts(tail, "href ")) { + tail = dg_trim(dg_s(tail.p + CBM_SZ_5, tail.n - CBM_SZ_5)); + } + const char *close = + tail.n && tail.p[0] == '"' ? memchr(tail.p + SKIP_ONE, '"', tail.n - SKIP_ONE) : NULL; + if (!close) { + return; + } + const char *qn = dg_element(dg, dg_s(rest.p, j), dg_s(NULL, 0), + dg->element_kind ? dg->element_kind : "node", line); + dg_link(dg, qn, dg_s(tail.p + SKIP_ONE, (size_t)(close - tail.p) - SKIP_ONE), + CBM_DOCLINK_DG_MERMAID_CLICK, line); +} + +static void dg_flow_statement(dg_t *dg, dg_str_t s, uint32_t line) { + if (dg_starts(s, "click ")) { + dg_mermaid_click(dg, s, line); + return; + } + static const char *const skip[] = {"classDef ", "class ", "style ", "linkStyle ", + "click ", "direction ", "end", NULL}; + for (int k = 0; skip[k]; k++) { + if (dg_starts(s, skip[k])) { + return; + } + } + if (dg_starts(s, "subgraph")) { + /* `subgraph id [title]`, `subgraph id[title]`, or `subgraph Some title` + * (the title is its id too) */ + dg_str_t rest = dg_trim(dg_s(s.p + CBM_SZ_8, s.n - CBM_SZ_8)); + dg_str_t id; + dg_str_t label; + size_t j = dg_flow_node(rest, 0, &id, &label); + if (j == 0) { + return; + } + size_t k = dg_skip_ws(rest, j); + if (!label.n && k < rest.n && rest.p[k] == '[') { + dg_shape(rest, k, &label); + } else if (!label.n && k < rest.n) { + id = rest; /* a title of several words */ + label = rest; + } + dg_element(dg, id, label, "subgraph", line); + return; + } + dg_group_t left; + size_t i = dg_flow_group(dg, s, 0, line, &left); + while (i < s.n && left.n) { + const char *style = NULL; + dg_str_t label; + size_t j = dg_flow_edge(s, i, &style, &label); + if (j == i) { + return; + } + dg_group_t right; + i = dg_flow_group(dg, s, j, line, &right); + for (int a = 0; a < left.n; a++) { + for (int b = 0; b < right.n; b++) { + dg_arrow(dg, left.ids[a], right.ids[b], label, style, line); + } + } + left = right; + } +} + +static void dg_flowchart(dg_t *dg, dg_lines_t *it) { + dg_str_t ln; + uint32_t line = 0; + while (dg_next_line(it, &ln, &line)) { + ln = dg_mermaid_strip(ln); + size_t start = 0; + bool quoted = false; + int depth = 0; + for (size_t i = 0; i <= ln.n; i++) { + char c = i < ln.n ? ln.p[i] : ';'; + quoted = c == '"' ? !quoted : quoted; + depth += !quoted && (c == '[' || c == '(' || c == '{'); + depth -= !quoted && (c == ']' || c == ')' || c == '}') && depth > 0; + if (c == ';' && !quoted && depth == 0) { + dg_str_t st = dg_trim(dg_s(ln.p + start, i - start)); + if (st.n) { + dg_flow_statement(dg, st, line); + } + start = i + SKIP_ONE; + } + } + } +} + +/* ── Mermaid: sequence ───────────────────────────────────────────── */ + +/* A message line `A->>+B: text`: its arrow; false when it has none. */ +static bool dg_sequence_message(dg_t *dg, dg_str_t ln, uint32_t line) { + static const char *const arrows[] = {"<<-->>", "<<->>", "-->>", "->>", "--x", "-x", + "--)", "-)", "-->", "->", NULL}; + const char *colon = memchr(ln.p, ':', ln.n); + dg_str_t head = colon ? dg_s(ln.p, (size_t)(colon - ln.p)) : ln; + for (int k = 0; arrows[k]; k++) { + size_t al = strlen(arrows[k]); + for (size_t i = 0; i + al <= head.n; i++) { + if (memcmp(head.p + i, arrows[k], al) != 0) { + continue; + } + dg_str_t to = dg_trim(dg_s(head.p + i + al, head.n - i - al)); + while (to.n && (to.p[0] == '+' || to.p[0] == '-')) { + to.p++; + to.n--; + } + dg_str_t label = colon + ? dg_s(colon + SKIP_ONE, (size_t)(ln.p + ln.n - colon) - SKIP_ONE) + : dg_s(NULL, 0); + bool reply = arrows[k][0] == '-' && arrows[k][SKIP_ONE] == '-'; + dg_arrow(dg, dg_s(head.p, i), to, label, reply ? "reply" : "message", line); + return true; + } + } + return false; +} + +static void dg_sequence(dg_t *dg, dg_lines_t *it) { + static const char participant[] = "participant "; + dg_str_t ln; + uint32_t line = 0; + while (dg_next_line(it, &ln, &line)) { + ln = dg_mermaid_strip(ln); + if (dg_starts(ln, "create ")) { + ln = dg_trim(dg_s(ln.p + CBM_SZ_7, ln.n - CBM_SZ_7)); + } + bool actor = dg_starts(ln, "actor "); + if (actor || dg_starts(ln, participant)) { + size_t kl = actor ? CBM_SZ_6 : sizeof(participant) - SKIP_ONE; + dg_str_t rest = dg_trim(dg_s(ln.p + kl, ln.n - kl)); + dg_str_t id = rest; + dg_str_t label = dg_s(NULL, 0); + for (size_t i = 0; i + CBM_SZ_4 <= rest.n; i++) { + if (memcmp(rest.p + i, " as ", CBM_SZ_4) == 0) { + id = dg_s(rest.p, i); + label = dg_s(rest.p + i + CBM_SZ_4, rest.n - i - CBM_SZ_4); + break; + } + } + dg_element(dg, id, label, actor ? "actor" : "participant", line); + continue; + } + (void)dg_sequence_message(dg, ln, line); + } +} + +/* ── Mermaid: class and ER ───────────────────────────────────────── */ + +typedef struct { + const char *op; + const char *style; + bool to_left; /* the arrow points at the left side (child -> parent) */ +} dg_rel_t; + +static const dg_rel_t DG_CLASS_RELS[] = { + {"<|--|>", "inheritance", false}, + {"<|--", "inheritance", true}, + {"--|>", "inheritance", false}, + {"<|..", "realization", true}, + {"..|>", "realization", false}, + {"*--", "composition", false}, + {"--*", "composition", true}, + {"o--", "aggregation", false}, + {"--o", "aggregation", true}, + {"<--", "association", true}, + {"-->", "association", false}, + {"<..", "dependency", true}, + {"..>", "dependency", false}, + {"--", "link", false}, + {"..", "link", false}, + {NULL, NULL, false}, +}; + +/* `A "1" op "*" B : label`: the relation of a class line, false when none. */ +static bool dg_class_relation(dg_t *dg, dg_str_t ln, uint32_t line) { + const char *colon = NULL; + bool quoted = false; + for (size_t i = 0; i < ln.n && !colon; i++) { + quoted = ln.p[i] == '"' ? !quoted : quoted; + colon = !quoted && ln.p[i] == ':' ? ln.p + i : NULL; + } + dg_str_t head = colon ? dg_s(ln.p, (size_t)(colon - ln.p)) : ln; + for (int k = 0; DG_CLASS_RELS[k].op; k++) { + size_t ol = strlen(DG_CLASS_RELS[k].op); + for (size_t i = 0; i + ol <= head.n; i++) { + if (memcmp(head.p + i, DG_CLASS_RELS[k].op, ol) != 0) { + continue; + } + dg_str_t l = dg_trim(dg_s(head.p, i)); + dg_str_t r = dg_trim(dg_s(head.p + i + ol, head.n - i - ol)); + if (l.n && l.p[l.n - SKIP_ONE] == '"') { /* a cardinality */ + const char *q = memchr(l.p, '"', l.n - SKIP_ONE); + l = q ? dg_trim(dg_s(l.p, (size_t)(q - l.p))) : l; + } + if (r.n && r.p[0] == '"') { + const char *q = memchr(r.p + SKIP_ONE, '"', r.n - SKIP_ONE); + r = q ? dg_trim(dg_s(q + SKIP_ONE, (size_t)(r.p + r.n - q) - SKIP_ONE)) : r; + } + if (!l.n || !r.n) { + return false; + } + dg_str_t label = colon + ? dg_s(colon + SKIP_ONE, (size_t)(ln.p + ln.n - colon) - SKIP_ONE) + : dg_s(NULL, 0); + dg_element(dg, l, dg_s(NULL, 0), "class", line); + dg_element(dg, r, dg_s(NULL, 0), "class", line); + if (DG_CLASS_RELS[k].to_left) { + dg_arrow(dg, r, l, label, DG_CLASS_RELS[k].style, line); + } else { + dg_arrow(dg, l, r, label, DG_CLASS_RELS[k].style, line); + } + return true; + } + } + return false; +} + +/* The name of `class Name~T~["Label"] {`, its label in *label. */ +static dg_str_t dg_class_decl(dg_str_t rest, dg_str_t *label) { + size_t j = 0; + while (j < rest.n && (dg_id_char(rest.p[j]) || rest.p[j] == '.')) { + j++; + } + dg_str_t name = dg_s(rest.p, j); + *label = dg_s(NULL, 0); + const char *lb = memchr(rest.p + j, '[', rest.n - j); + const char *rb = lb ? memchr(lb, ']', (size_t)(rest.p + rest.n - lb)) : NULL; + if (lb && rb) { + *label = dg_s(lb + SKIP_ONE, (size_t)(rb - lb) - SKIP_ONE); + } + return name; +} + +static void dg_classdiagram(dg_t *dg, dg_lines_t *it) { + dg_str_t ln; + uint32_t line = 0; + int body = 0; + while (dg_next_line(it, &ln, &line)) { + ln = dg_mermaid_strip(ln); + if (body > 0) { + body -= ln.n && ln.p[0] == '}'; + continue; + } + if (dg_starts(ln, "class ")) { + dg_str_t label; + dg_str_t name = dg_class_decl(dg_trim(dg_s(ln.p + CBM_SZ_6, ln.n - CBM_SZ_6)), &label); + dg_element(dg, name, label, "class", line); + body += ln.n && ln.p[ln.n - SKIP_ONE] == '{'; + continue; + } + if (dg_starts(ln, "click ") || dg_starts(ln, "link ")) { + dg_mermaid_click(dg, ln, line); + continue; + } + if (dg_starts(ln, "namespace ") || dg_starts(ln, "note") || dg_starts(ln, "<<") || + dg_starts(ln, "style ") || dg_starts(ln, "cssClass") || dg_starts(ln, "classDef") || + dg_starts(ln, "direction") || dg_starts(ln, "link ") || dg_starts(ln, "callback ") || + dg_starts(ln, "click ") || dg_eq(ln, "}")) { + continue; + } + if (!dg_class_relation(dg, ln, line)) { + const char *colon = memchr(ln.p, ':', ln.n); + if (colon) { /* `Name : +member`: the class, no member */ + dg_element(dg, dg_s(ln.p, (size_t)(colon - ln.p)), dg_s(NULL, 0), "class", line); + } + } + } +} + +static bool dg_er_card(char c) { + return c == '|' || c == 'o' || c == '}' || c == '{'; +} + +static void dg_erdiagram(dg_t *dg, dg_lines_t *it) { + dg_str_t ln; + uint32_t line = 0; + int body = 0; + while (dg_next_line(it, &ln, &line)) { + ln = dg_mermaid_strip(ln); + if (body > 0) { + body -= ln.n && ln.p[0] == '}'; + continue; + } + const char *colon = memchr(ln.p, ':', ln.n); + dg_str_t head = colon ? dg_s(ln.p, (size_t)(colon - ln.p)) : ln; + bool done = false; + for (size_t i = 0; i + PAIR_LEN <= head.n && !done; i++) { + bool solid = head.p[i] == '-' && head.p[i + SKIP_ONE] == '-'; + bool dashed = head.p[i] == '.' && head.p[i + SKIP_ONE] == '.'; + if (!(solid || dashed) || i == 0 || !dg_er_card(head.p[i - SKIP_ONE])) { + continue; + } + size_t a = i; + while (a > 0 && dg_er_card(head.p[a - SKIP_ONE])) { + a--; + } + size_t b = i + PAIR_LEN; + while (b < head.n && dg_er_card(head.p[b])) { + b++; + } + dg_str_t l = dg_trim(dg_s(head.p, a)); + dg_str_t r = dg_trim(dg_s(head.p + b, head.n - b)); + dg_str_t label = colon + ? dg_s(colon + SKIP_ONE, (size_t)(ln.p + ln.n - colon) - SKIP_ONE) + : dg_s(NULL, 0); + dg_element(dg, l, dg_s(NULL, 0), "entity", line); + dg_element(dg, r, dg_s(NULL, 0), "entity", line); + dg_arrow(dg, l, r, label, solid ? "identifying" : "non_identifying", line); + done = true; + } + if (!done && ln.n && ln.p[ln.n - SKIP_ONE] == '{') { + dg_str_t name = dg_trim(dg_s(ln.p, ln.n - SKIP_ONE)); + dg_str_t label = dg_s(NULL, 0); + const char *lb = memchr(name.p, '[', name.n); + if (lb && name.p[name.n - SKIP_ONE] == ']') { + label = dg_s(lb + SKIP_ONE, (size_t)(name.p + name.n - lb) - PAIR_LEN); + name = dg_trim(dg_s(name.p, (size_t)(lb - name.p))); + } + dg_element(dg, name, label, "entity", line); + body++; + } + } +} + +/* ── Mermaid: state ──────────────────────────────────────────────── */ + +enum { DG_STATE_DEPTH = 8 }; + +/* A state of a transition: `[*]` is the start (left) or the end (right) of + * the composite state it is written in, `composite` (empty at the top). */ +static dg_str_t dg_state_id(dg_str_t s, bool left, dg_str_t composite, char *buf, size_t cap) { + s = dg_trim(s); + if (!dg_eq(s, "[*]")) { + return s; + } + const char *end = left ? "start" : "end"; + int n = composite.n ? snprintf(buf, cap, "%.*s_%s", (int)composite.n, composite.p, end) + : snprintf(buf, cap, "%s", end); + return n > 0 && (size_t)n < cap ? dg_s(buf, (size_t)n) : dg_s(end, strlen(end)); +} + +static void dg_statediagram(dg_t *dg, dg_lines_t *it) { + dg_str_t ln; + uint32_t line = 0; + dg_str_t stack[DG_STATE_DEPTH]; + int depth = 0; + while (dg_next_line(it, &ln, &line)) { + ln = dg_mermaid_strip(ln); + if (dg_eq(ln, "}")) { + depth -= depth > 0; + continue; + } + if (dg_starts(ln, "state ")) { + dg_str_t rest = dg_trim(dg_s(ln.p + CBM_SZ_6, ln.n - CBM_SZ_6)); + if (rest.n && rest.p[0] == '"') { + const char *q = memchr(rest.p + SKIP_ONE, '"', rest.n - SKIP_ONE); + if (q) { + dg_str_t label = dg_s(rest.p + SKIP_ONE, (size_t)(q - rest.p) - SKIP_ONE); + dg_str_t after = + dg_trim(dg_s(q + SKIP_ONE, (size_t)(rest.p + rest.n - q) - SKIP_ONE)); + if (dg_starts(after, "as ")) { + dg_element(dg, dg_trim(dg_s(after.p + CBM_SZ_3, after.n - CBM_SZ_3)), label, + "state", line); + } + } + continue; + } + size_t j = 0; + while (j < rest.n && dg_id_char(rest.p[j])) { + j++; + } + dg_element(dg, dg_s(rest.p, j), dg_s(NULL, 0), "state", line); + if (rest.n && rest.p[rest.n - SKIP_ONE] == '{') { + if (depth < DG_STATE_DEPTH) { + stack[depth] = dg_s(rest.p, j); + } + depth++; + } + continue; + } + const char *arrow = NULL; + for (size_t i = 0; i + CBM_SZ_3 <= ln.n && !arrow; i++) { + arrow = memcmp(ln.p + i, "-->", CBM_SZ_3) == 0 ? ln.p + i : NULL; + } + if (!arrow) { + continue; + } + dg_str_t rest = dg_s(arrow + CBM_SZ_3, (size_t)(ln.p + ln.n - arrow) - CBM_SZ_3); + const char *colon = memchr(rest.p, ':', rest.n); + dg_str_t to = colon ? dg_s(rest.p, (size_t)(colon - rest.p)) : rest; + dg_str_t label = colon + ? dg_s(colon + SKIP_ONE, (size_t)(rest.p + rest.n - colon) - SKIP_ONE) + : dg_s(NULL, 0); + dg_str_t composite = + depth > 0 && depth <= DG_STATE_DEPTH ? stack[depth - SKIP_ONE] : dg_s(NULL, 0); + char from_buf[DG_ID_MAX]; + char to_buf[DG_ID_MAX]; + dg_arrow(dg, + dg_state_id(dg_s(ln.p, (size_t)(arrow - ln.p)), true, composite, from_buf, + sizeof(from_buf)), + dg_state_id(to, false, composite, to_buf, sizeof(to_buf)), label, "transition", + line); + } +} + +/* ── Mermaid: C4 ─────────────────────────────────────────────────── */ + +/* The arguments of `Name(a, "b", c)`: up to DG_ARGS_MAX, quotes dropped. */ +static int dg_args(dg_str_t s, dg_str_t *args) { + const char *open = memchr(s.p, '(', s.n); + if (!open) { + return 0; + } + int n = 0; + bool quoted = false; + const char *start = open + SKIP_ONE; + for (const char *p = start; p < s.p + s.n && n < DG_ARGS_MAX; p++) { + quoted = *p == '"' ? !quoted : quoted; + if (!quoted && (*p == ',' || *p == ')')) { + args[n++] = dg_unquote(dg_s(start, (size_t)(p - start))); + start = p + SKIP_ONE; + if (*p == ')') { + break; + } + } + } + return n; +} + +/* A C4 macro line `Person(id, "Label", ...)` or `Rel(a, b, "label")`: false + * when the line is none. */ +static bool dg_c4_line(dg_t *dg, dg_str_t ln, uint32_t line) { + size_t j = 0; + while (j < ln.n && (isalnum((unsigned char)ln.p[j]) || ln.p[j] == '_')) { + j++; + } + if (j == 0 || j >= ln.n || ln.p[j] != '(') { + return false; + } + { + dg_str_t fn = dg_s(ln.p, j); + dg_str_t args[DG_ARGS_MAX]; + int na = dg_args(ln, args); + if (dg_starts(fn, "Rel") || dg_starts(fn, "BiRel")) { + int base = dg_eq(fn, "RelIndex") ? SKIP_ONE : 0; /* RelIndex(i, from, to, ...) */ + if (na >= base + PAIR_LEN) { + dg_str_t label = na > base + PAIR_LEN ? args[base + PAIR_LEN] : dg_s(NULL, 0); + dg_arrow(dg, args[base], args[base + SKIP_ONE], label, + dg_starts(fn, "BiRel") ? "bidirectional" : "relation", line); + } + return true; + } + if (dg_starts(fn, "Update") || dg_starts(fn, "Add") || dg_starts(fn, "Lay_") || + dg_eq(fn, "title") || na < SKIP_ONE) { + return true; + } + char kind[DG_KIND_MAX]; + size_t kl = fn.n < sizeof(kind) - SKIP_ONE ? fn.n : sizeof(kind) - SKIP_ONE; + for (size_t k = 0; k < kl; k++) { + kind[k] = (char)tolower((unsigned char)fn.p[k]); + } + kind[kl] = '\0'; + const char *kc = dg_dup(dg, kind, kl); + dg_element(dg, args[0], na > SKIP_ONE ? args[SKIP_ONE] : dg_s(NULL, 0), kc ? kc : "c4", + line); + } + return true; +} + +static void dg_c4(dg_t *dg, dg_lines_t *it) { + dg_str_t ln; + uint32_t line = 0; + while (dg_next_line(it, &ln, &line)) { + (void)dg_c4_line(dg, dg_mermaid_strip(ln), line); + } +} + +/* ── Mermaid: architecture ───────────────────────────────────────── */ + +/* `id(icon)[Label] in group`: the id; the label in *label. */ +static dg_str_t dg_arch_decl(dg_str_t rest, dg_str_t *label) { + size_t j = 0; + while (j < rest.n && dg_id_char(rest.p[j])) { + j++; + } + dg_str_t id = dg_s(rest.p, j); + *label = dg_s(NULL, 0); + const char *lb = memchr(rest.p + j, '[', rest.n - j); + const char *rb = lb ? memchr(lb, ']', (size_t)(rest.p + rest.n - lb)) : NULL; + if (lb && rb) { + *label = dg_s(lb + SKIP_ONE, (size_t)(rb - lb) - SKIP_ONE); + } + return id; +} + +/* `id{group}:R` or `R:id`: the id. */ +static dg_str_t dg_arch_end(dg_str_t s) { + s = dg_trim(s); + const char *colon = memchr(s.p, ':', s.n); + if (colon) { + dg_str_t a = dg_s(s.p, (size_t)(colon - s.p)); + dg_str_t b = dg_s(colon + SKIP_ONE, (size_t)(s.p + s.n - colon) - SKIP_ONE); + s = a.n > SKIP_ONE ? a : b; /* the side is one letter */ + } + const char *brace = memchr(s.p, '{', s.n); + return dg_trim(brace ? dg_s(s.p, (size_t)(brace - s.p)) : s); +} + +static void dg_architecture(dg_t *dg, dg_lines_t *it) { + static const char *const decls[] = {"group ", "service ", "junction ", NULL}; + dg_str_t ln; + uint32_t line = 0; + while (dg_next_line(it, &ln, &line)) { + ln = dg_mermaid_strip(ln); + bool decl = false; + for (int k = 0; decls[k] && !decl; k++) { + if (!dg_starts(ln, decls[k])) { + continue; + } + size_t kl = strlen(decls[k]); + dg_str_t label; + dg_str_t id = dg_arch_decl(dg_trim(dg_s(ln.p + kl, ln.n - kl)), &label); + char kind[DG_KIND_MAX]; + snprintf(kind, sizeof(kind), "%.*s", (int)(kl - SKIP_ONE), decls[k]); + dg_element(dg, id, label, dg_dup(dg, kind, kl - SKIP_ONE), line); + decl = true; + } + if (decl) { + continue; + } + static const char *const ops[] = {"<-->", "-->", "<--", "--", NULL}; + for (int k = 0; ops[k]; k++) { + size_t ol = strlen(ops[k]); + const char *at = NULL; + for (size_t i = 0; i + ol <= ln.n && !at; i++) { + at = memcmp(ln.p + i, ops[k], ol) == 0 ? ln.p + i : NULL; + } + if (!at) { + continue; + } + dg_str_t l = dg_arch_end(dg_s(ln.p, (size_t)(at - ln.p))); + dg_str_t r = dg_arch_end(dg_s(at + ol, (size_t)(ln.p + ln.n - at) - ol)); + if (k == PAIR_LEN) { /* <-- points left */ + dg_arrow(dg, r, l, dg_s(NULL, 0), "arrow", line); + } else { + dg_arrow(dg, l, r, dg_s(NULL, 0), + k == 0 ? "bidirectional" + : k == 1 ? "arrow" + : "line", + line); + } + break; + } + } +} + +/* ── Mermaid: the header ─────────────────────────────────────────── */ + +typedef void (*dg_parser_fn)(dg_t *dg, dg_lines_t *it); + +typedef struct { + const char *keyword; + const char *kind; + const char *element_kind; /* of an element named only by an arrow */ + dg_parser_fn parse; +} dg_kind_t; + +static const dg_kind_t DG_MERMAID_KINDS[] = { + {"flowchart", "flowchart", "node", dg_flowchart}, + {"graph", "flowchart", "node", dg_flowchart}, + {"sequenceDiagram", "sequence", "participant", dg_sequence}, + {"classDiagram", "class", "class", dg_classdiagram}, + {"erDiagram", "er", "entity", dg_erdiagram}, + {"stateDiagram", "state", "state", dg_statediagram}, + {"C4Context", "c4_context", "c4", dg_c4}, + {"C4Container", "c4_container", "c4", dg_c4}, + {"C4Component", "c4_component", "c4", dg_c4}, + {"C4Dynamic", "c4_dynamic", "c4", dg_c4}, + {"C4Deployment", "c4_deployment", "c4", dg_c4}, + {"architecture", "architecture", "service", dg_architecture}, + {NULL, NULL, NULL, NULL}, +}; + +/* A `title Text` line of the diagram (or `pie title Text` in its header), + * when the front matter named none. */ +static dg_str_t dg_title_line(dg_lines_t it, dg_str_t head) { + for (size_t i = 0; i + CBM_SZ_7 <= head.n; i++) { + if (memcmp(head.p + i, " title ", CBM_SZ_7) == 0) { + return dg_unquote(dg_s(head.p + i + CBM_SZ_7, head.n - i - CBM_SZ_7)); + } + } + dg_str_t ln; + uint32_t line = 0; + while (dg_next_line(&it, &ln, &line)) { + ln = dg_mermaid_strip(ln); + if (dg_starts(ln, "title ")) { + return dg_unquote(dg_s(ln.p + CBM_SZ_6, ln.n - CBM_SZ_6)); + } + } + return dg_s(NULL, 0); +} + +/* The first line that says what the diagram is: front matter (`---` ... + * `---`, its `title:` kept) and `%%` lines skipped. A title of the diagram's + * own replaces one given from outside (a caption); none keeps it. */ +static dg_str_t dg_mermaid_header(dg_lines_t *it, dg_str_t *title) { + dg_str_t ln; + uint32_t line = 0; + bool front = false; + while (dg_next_line(it, &ln, &line)) { + dg_str_t t = dg_trim(ln); + if (dg_eq(t, "---")) { + front = !front; + continue; + } + if (front) { + if (dg_starts(t, "title:")) { + *title = dg_unquote(dg_s(t.p + CBM_SZ_6, t.n - CBM_SZ_6)); + } + continue; + } + t = dg_mermaid_strip(t); + if (t.n) { + return t; + } + } + return dg_s(NULL, 0); +} + +static void dg_mermaid(dg_t *dg, dg_lines_t *it, dg_str_t *title) { + dg_str_t head = dg_mermaid_header(it, title); + dg_str_t own = dg_title_line(*it, head); + *title = own.n ? own : *title; + size_t j = 0; + while (j < head.n && !isspace((unsigned char)head.p[j]) && head.p[j] != ';') { + j++; + } + dg_str_t word = dg_s(head.p, j); + for (int k = 0; DG_MERMAID_KINDS[k].keyword; k++) { + if (dg_starts(word, DG_MERMAID_KINDS[k].keyword)) { + snprintf(dg->kind, sizeof(dg->kind), "%s", DG_MERMAID_KINDS[k].kind); + dg->element_kind = DG_MERMAID_KINDS[k].element_kind; + DG_MERMAID_KINDS[k].parse(dg, it); + return; + } + } + snprintf(dg->kind, sizeof(dg->kind), "%.*s", + (int)(word.n < DG_KIND_MAX - SKIP_ONE ? word.n : DG_KIND_MAX - SKIP_ONE), word.p); +} + +/* ── PlantUML ────────────────────────────────────────────────────── */ + +/* Declarations whose `{ ... }` body holds members, not elements. */ +static const char *const DG_PUML_CLASSLIKE[] = { + "abstract class", "abstract", "class", "interface", "enum", "annotation", "protocol", + "struct", "exception", "object", "map", "metaclass", NULL}; +/* Declarations whose `{ ... }` body holds further elements. */ +static const char *const DG_PUML_ELEMENTS[] = { + "participant", "actor", "boundary", "control", "database", "collections", "queue", + "usecase", "component", "node", "cloud", "frame", "folder", "package", + "rectangle", "artifact", "file", "storage", "card", "agent", "stack", + "hexagon", "person", "process", "state", "entity", "namespace", NULL}; + +static bool dg_word_at(dg_str_t s, const char *word) { + size_t l = strlen(word); + return s.n > l && memcmp(s.p, word, l) == 0 && (s.p[l] == ' ' || s.p[l] == '\t'); +} + +/* An operand of a PlantUML line: `[Comp]`, `(use case)`, `:actor:`, `"text"`, + * an id, or `[*]`; its id in *id; the position after it (i when none). */ +static size_t dg_puml_operand(dg_str_t s, size_t i, dg_str_t *id, dg_str_t *label) { + i = dg_skip_ws(s, i); + *label = dg_s(NULL, 0); + if (i + CBM_SZ_3 <= s.n && memcmp(s.p + i, "[*]", CBM_SZ_3) == 0) { + *id = dg_s(s.p + i, CBM_SZ_3); + return i + CBM_SZ_3; + } + static const char opens[] = "[(:\""; + static const char closes[] = "]):\""; + const char *o = i < s.n ? strchr(opens, s.p[i]) : NULL; + if (o && s.p[i]) { + char close = closes[o - opens]; + const char *c = memchr(s.p + i + SKIP_ONE, close, s.n - i - SKIP_ONE); + if (!c) { + return i; + } + *id = dg_s(s.p + i + SKIP_ONE, (size_t)(c - s.p) - i - SKIP_ONE); + *label = *id; + return (size_t)(c - s.p) + SKIP_ONE; + } + size_t j = i; + while (j < s.n && (dg_id_char(s.p[j]) || s.p[j] == '.' || s.p[j] == ':' || s.p[j] == '$')) { + if (s.p[j] == '-' || s.p[j] == ':') { /* an arrow or a label starts */ + break; + } + j++; + } + *id = dg_s(s.p + i, j - i); + return j; +} + +/* What a PlantUML arrow means, and whether it points at its left side. */ +static const char *dg_puml_arrow_style(dg_str_t op, bool sequence, bool *to_left) { + *to_left = false; + bool lt = memchr(op.p, '<', op.n) != NULL; + bool gt = memchr(op.p, '>', op.n) != NULL; + bool dotted = memchr(op.p, '.', op.n) != NULL; + bool inherit = false; + for (size_t i = 0; i + SKIP_ONE < op.n; i++) { + if (op.p[i] == '<' && op.p[i + SKIP_ONE] == '|') { + inherit = true; + *to_left = true; + } + if (op.p[i] == '|' && op.p[i + SKIP_ONE] == '>') { + inherit = true; + } + } + if (inherit) { + return dotted ? "realization" : "inheritance"; + } + if (op.p[0] == '+' || op.p[op.n - SKIP_ONE] == '+') { /* an inner class */ + *to_left = op.p[0] == '+'; /* nested -> its container, as `A ..+ B` reads */ + return "nesting"; + } + if (op.p[0] == '*' || op.p[op.n - SKIP_ONE] == '*') { + *to_left = op.p[op.n - SKIP_ONE] == '*'; + return "composition"; + } + if (op.p[0] == 'o' || op.p[op.n - SKIP_ONE] == 'o') { + *to_left = op.p[op.n - SKIP_ONE] == 'o'; + return "aggregation"; + } + if (lt && gt) { + return "bidirectional"; + } + *to_left = lt; + if (sequence) { + return memchr(op.p, '-', op.n) && op.n >= PAIR_LEN && op.p[SKIP_ONE] == '-' ? "reply" + : "message"; + } + return dotted ? (gt || lt ? "dependency" : "dotted") : (gt || lt ? "arrow" : "line"); +} + +static bool dg_puml_arrow_char(char c) { + return c == '-' || c == '.' || c == '<' || c == '>' || c == '|' || c == '*' || c == 'o' || + c == '#' || c == '+' || c == '^' || c == '/' || c == '\\' || c == 'x' || c == '[' || + c == ']' || c == ',' || c == '=' || isalpha((unsigned char)c); +} + +/* `A "1" -up-> "n" B : label`: false when the line holds no relation. */ +static bool dg_puml_relation(dg_t *dg, dg_str_t ln, uint32_t line, bool sequence) { + dg_str_t lid; + dg_str_t llab; + size_t i = dg_puml_operand(ln, 0, &lid, &llab); + if (!lid.n) { + return false; + } + i = dg_skip_ws(ln, i); + if (i < ln.n && ln.p[i] == '"') { /* a cardinality */ + const char *q = memchr(ln.p + i + SKIP_ONE, '"', ln.n - i - SKIP_ONE); + i = q ? dg_skip_ws(ln, (size_t)(q - ln.p) + SKIP_ONE) : i; + } + size_t a = i; + while (i < ln.n && !isspace((unsigned char)ln.p[i]) && dg_puml_arrow_char(ln.p[i])) { + i++; + } + dg_str_t op = dg_s(ln.p + a, i - a); + bool arrow = false; + for (size_t k = 0; k + SKIP_ONE < op.n && !arrow; k++) { + arrow = (op.p[k] == '-' || op.p[k] == '.') && + (op.p[k + SKIP_ONE] == '-' || op.p[k + SKIP_ONE] == '.' || + op.p[k + SKIP_ONE] == '>' || op.p[k + SKIP_ONE] == '|' || + op.p[k + SKIP_ONE] == '[' || isalpha((unsigned char)op.p[k + SKIP_ONE])); + arrow = + arrow || (op.p[k] == '<' && (op.p[k + SKIP_ONE] == '-' || op.p[k + SKIP_ONE] == '.')); + } + if (!arrow) { + return false; + } + i = dg_skip_ws(ln, i); + if (i < ln.n && ln.p[i] == '"') { + const char *q = memchr(ln.p + i + SKIP_ONE, '"', ln.n - i - SKIP_ONE); + i = q ? (size_t)(q - ln.p) + SKIP_ONE : i; + } + dg_str_t rid; + dg_str_t rlab; + i = dg_puml_operand(ln, i, &rid, &rlab); + if (!rid.n) { + return false; + } + i = dg_skip_ws(ln, i); + dg_str_t label = + i < ln.n && ln.p[i] == ':' ? dg_s(ln.p + i + SKIP_ONE, ln.n - i - SKIP_ONE) : dg_s(NULL, 0); + dg_element(dg, lid, llab, dg->element_kind, line); + dg_element(dg, rid, rlab, dg->element_kind, line); + bool to_left = false; + const char *style = dg_puml_arrow_style(op, sequence, &to_left); + if (to_left) { + dg_arrow(dg, rid, lid, label, style, line); + } else { + dg_arrow(dg, lid, rid, label, style, line); + } + return true; +} + +/* `[[url]]`, `[[url label]]`, `[[url{tooltip}]]` in a PlantUML line: the URL. */ +static dg_str_t dg_puml_url(dg_str_t ln) { + for (size_t i = 0; i + SKIP_ONE < ln.n; i++) { + if (ln.p[i] == '[' && ln.p[i + SKIP_ONE] == '[') { + size_t e = i + PAIR_LEN; + while (e < ln.n && ln.p[e] != ']' && ln.p[e] != '{' && + !isspace((unsigned char)ln.p[e])) { + e++; + } + return dg_s(ln.p + i + PAIR_LEN, e - i - PAIR_LEN); + } + } + return dg_s(NULL, 0); +} + +/* `!include path` (also _many, _once, url): the diagram's link to the file it + * takes in; `` names none of the repo's. `url of X is [[url]]`: X's. */ +static bool dg_puml_link_line(dg_t *dg, dg_str_t ln, uint32_t line) { + if (dg_starts(ln, "!include")) { + size_t i = CBM_SZ_8; + while (i < ln.n && !isspace((unsigned char)ln.p[i])) { + i++; + } + dg_str_t arg = dg_trim(dg_s(ln.p + i, ln.n - i)); + const char *bang = memchr(arg.p, '!', arg.n); /* `file!2`: a block of it */ + arg = bang ? dg_trim(dg_s(arg.p, (size_t)(bang - arg.p))) : arg; + if (arg.n && arg.p[0] != '<') { + dg_link(dg, dg->qn, arg, CBM_DOCLINK_DG_PLANTUML_INCLUDE, line); + } + return true; + } + if (dg_starts(ln, "url of ") || dg_starts(ln, "url for ")) { + size_t kw = dg_starts(ln, "url of ") ? CBM_SZ_7 : CBM_SZ_8; + dg_str_t rest = dg_s(ln.p + kw, ln.n - kw); + size_t j = 0; + while (j + CBM_SZ_4 <= rest.n && memcmp(rest.p + j, " is ", CBM_SZ_4) != 0) { + j++; + } + if (j + CBM_SZ_4 <= rest.n) { + const char *qn = + dg_element(dg, dg_unquote(dg_s(rest.p, j)), dg_s(NULL, 0), dg->element_kind, line); + dg_link(dg, qn, dg_puml_url(rest), CBM_DOCLINK_DG_PLANTUML_URL, line); + } + return true; + } + return false; +} + +/* `kw "Label" as Id`, `kw Id as "Label"`, `kw Id`, `kw [Label] as Id`: an + * element. *body is set when its `{` opens a member body. */ +static bool dg_puml_decl(dg_t *dg, dg_str_t ln, uint32_t line, bool *members) { + const char *kw = NULL; + bool classlike = false; + for (int k = 0; DG_PUML_CLASSLIKE[k] && !kw; k++) { + if (dg_word_at(ln, DG_PUML_CLASSLIKE[k])) { + kw = DG_PUML_CLASSLIKE[k]; + classlike = true; + } + } + for (int k = 0; DG_PUML_ELEMENTS[k] && !kw; k++) { + kw = dg_word_at(ln, DG_PUML_ELEMENTS[k]) ? DG_PUML_ELEMENTS[k] : NULL; + } + if (!kw) { + return false; + } + dg_str_t rest = dg_trim(dg_s(ln.p + strlen(kw), ln.n - strlen(kw))); + dg_str_t id; + dg_str_t label; + size_t j = dg_puml_operand(rest, 0, &id, &label); + dg_str_t after = dg_trim(dg_s(rest.p + j, rest.n - j)); + if (dg_starts(after, "as ")) { + dg_str_t aid; + dg_str_t alab; + dg_puml_operand(dg_trim(dg_s(after.p + CBM_SZ_3, after.n - CBM_SZ_3)), 0, &aid, &alab); + if (label.n || !alab.n) { + label = label.n ? label : id; + id = aid; + } else { + label = alab; /* `kw Id as "Label"` */ + } + } + *members = classlike && rest.n && rest.p[rest.n - SKIP_ONE] == '{'; + const char *kind = strcmp(kw, "abstract class") == 0 ? "abstract" : kw; + const char *qn = dg_element(dg, id, label, kind, line); + dg_link(dg, qn, dg_puml_url(after), CBM_DOCLINK_DG_PLANTUML_URL, line); + return true; +} + +/* What a PlantUML diagram is: the strongest kind any of its lines tells (the + * packages around classes make no component diagram). */ +enum { + PK_C4, + PK_CLASS, + PK_SEQUENCE, + PK_USECASE, + PK_ACTOR, /* an actor alone: a sequence, unless use cases say otherwise */ + PK_STATE, + PK_ACTIVITY, + PK_COMPONENT, + PK_COUNT +}; + +static const char *dg_puml_kind(dg_lines_t it) { + static const char *const names[PK_COUNT] = {"c4", "class", "sequence", "usecase", + "sequence", "state", "activity", "component"}; + bool seen[PK_COUNT] = {false}; + dg_str_t ln; + uint32_t line = 0; + while (dg_next_line(&it, &ln, &line)) { + ln = dg_trim(ln); + seen[PK_C4] |= dg_starts(ln, "Person(") || dg_starts(ln, "System(") || + dg_starts(ln, "Container(") || dg_starts(ln, "Component(") || + dg_starts(ln, "Rel("); + for (int k = 0; DG_PUML_CLASSLIKE[k]; k++) { + seen[PK_CLASS] |= dg_word_at(ln, DG_PUML_CLASSLIKE[k]); + } + seen[PK_SEQUENCE] |= dg_word_at(ln, "participant") || dg_word_at(ln, "boundary") || + dg_word_at(ln, "control") || dg_word_at(ln, "collections"); + seen[PK_USECASE] |= dg_word_at(ln, "usecase") || (ln.n && ln.p[0] == '('); + seen[PK_ACTOR] |= dg_word_at(ln, "actor"); + seen[PK_STATE] |= dg_word_at(ln, "state") || dg_starts(ln, "[*]"); + seen[PK_ACTIVITY] |= + dg_eq(ln, "start") || (ln.n && ln.p[0] == ':' && ln.p[ln.n - SKIP_ONE] == ';'); + seen[PK_COMPONENT] |= dg_word_at(ln, "component") || dg_word_at(ln, "node") || + dg_word_at(ln, "package") || dg_word_at(ln, "database") || + (ln.n && ln.p[0] == '['); + } + for (int k = 0; k < PK_COUNT; k++) { + if (seen[k]) { + return names[k]; + } + } + return "uml"; +} + +static bool dg_c4_line(dg_t *dg, dg_str_t ln, uint32_t line); + +static void dg_plantuml(dg_t *dg, dg_lines_t *it, dg_str_t *title) { + const char *kind = dg_puml_kind(*it); + snprintf(dg->kind, sizeof(dg->kind), "%s", kind); + bool sequence = strcmp(kind, "sequence") == 0; + dg->element_kind = sequence ? "participant" + : strcmp(kind, "class") == 0 ? "class" + : strcmp(kind, "usecase") == 0 ? "usecase" + : strcmp(kind, "state") == 0 ? "state" + : strcmp(kind, "component") == 0 ? "component" + : "node"; + if (strcmp(kind, "activity") == 0) { + return; /* an activity's steps are prose: the Diagram alone */ + } + dg_str_t ln; + uint32_t line = 0; + int members = 0; + bool comment = false; + while (dg_next_line(it, &ln, &line)) { + ln = dg_trim(ln); + if (comment) { + comment = !(ln.n >= PAIR_LEN && ln.p[ln.n - PAIR_LEN] == '\'' && + ln.p[ln.n - SKIP_ONE] == '/'); + continue; + } + if (dg_starts(ln, "/'")) { + comment = !(ln.n >= CBM_SZ_4 && ln.p[ln.n - PAIR_LEN] == '\'' && + ln.p[ln.n - SKIP_ONE] == '/'); + continue; + } + if (members > 0) { + members += ln.n && ln.p[ln.n - SKIP_ONE] == '{'; + members -= ln.n && ln.p[0] == '}'; + continue; + } + if (dg_puml_link_line(dg, ln, line)) { + continue; + } + if (!ln.n || ln.p[0] == '\'' || ln.p[0] == '@' || ln.p[0] == '!' || dg_eq(ln, "}") || + dg_starts(ln, "skinparam") || dg_starts(ln, "hide ") || dg_starts(ln, "show ") || + dg_starts(ln, "note ") || dg_starts(ln, "end note") || dg_starts(ln, "legend") || + dg_starts(ln, "header") || dg_starts(ln, "footer") || dg_starts(ln, "left to right") || + dg_starts(ln, "top to bottom") || dg_starts(ln, "autonumber") || + dg_starts(ln, "activate") || dg_starts(ln, "deactivate")) { + continue; + } + if (dg_starts(ln, "title ")) { + if (!title->n) { + *title = dg_unquote(dg_s(ln.p + CBM_SZ_6, ln.n - CBM_SZ_6)); + } + continue; + } + if (strcmp(kind, "c4") == 0 && dg_c4_line(dg, ln, line)) { + continue; + } + bool body = false; + if (dg_puml_decl(dg, ln, line, &body)) { + members += body; + continue; + } + (void)dg_puml_relation(dg, ln, line, sequence); + } +} + +/* ── Graphviz DOT ────────────────────────────────────────────────── */ + +typedef enum { + DT_END = 0, + DT_ID, + DT_LBRACE, + DT_RBRACE, + DT_LBRACK, + DT_RBRACK, + DT_SEMI, + DT_COMMA, + DT_EQ, + DT_COLON, + DT_EDGE, +} dt_kind_t; + +typedef struct { + dt_kind_t k; + dg_str_t s; + uint32_t line; +} dt_tok_t; + +typedef struct { + const char *p; + const char *end; + uint32_t line; + dt_tok_t tok; /* the current token */ +} dt_lex_t; + +static void dt_skip(dt_lex_t *lx) { + for (;;) { + while (lx->p < lx->end && isspace((unsigned char)*lx->p)) { + lx->line += *lx->p == '\n'; + lx->p++; + } + if (lx->p + SKIP_ONE < lx->end && lx->p[0] == '/' && lx->p[SKIP_ONE] == '/') { + while (lx->p < lx->end && *lx->p != '\n') { + lx->p++; + } + continue; + } + if (lx->p < lx->end && lx->p[0] == '#') { + while (lx->p < lx->end && *lx->p != '\n') { + lx->p++; + } + continue; + } + if (lx->p + SKIP_ONE < lx->end && lx->p[0] == '/' && lx->p[SKIP_ONE] == '*') { + lx->p += PAIR_LEN; + while (lx->p + SKIP_ONE < lx->end && !(lx->p[0] == '*' && lx->p[SKIP_ONE] == '/')) { + lx->line += *lx->p == '\n'; + lx->p++; + } + lx->p = lx->p + PAIR_LEN <= lx->end ? lx->p + PAIR_LEN : lx->end; + continue; + } + return; + } +} + +static void dt_next(dt_lex_t *lx) { + dt_skip(lx); + lx->tok = (dt_tok_t){DT_END, dg_s(NULL, 0), lx->line}; + if (lx->p >= lx->end) { + return; + } + char c = *lx->p; + static const char single[] = "{}[];,=:"; + static const dt_kind_t kinds[] = {DT_LBRACE, DT_RBRACE, DT_LBRACK, DT_RBRACK, + DT_SEMI, DT_COMMA, DT_EQ, DT_COLON}; + const char *hit = strchr(single, c); + if (c && hit) { + lx->tok.k = kinds[hit - single]; + lx->p++; + return; + } + if (c == '-' && lx->p + SKIP_ONE < lx->end && + (lx->p[SKIP_ONE] == '>' || lx->p[SKIP_ONE] == '-')) { + lx->tok.k = DT_EDGE; + lx->tok.s = dg_s(lx->p, PAIR_LEN); + lx->p += PAIR_LEN; + return; + } + if (c == '"') { + const char *q = lx->p + SKIP_ONE; + while (q < lx->end && *q != '"') { + lx->line += *q == '\n'; + q += *q == '\\' && q + SKIP_ONE < lx->end ? PAIR_LEN : SKIP_ONE; + } + lx->tok.k = DT_ID; + lx->tok.s = dg_s(lx->p + SKIP_ONE, (size_t)(q - lx->p) - SKIP_ONE); + lx->p = q < lx->end ? q + SKIP_ONE : lx->end; + return; + } + if (c == '<') { /* an HTML string */ + int depth = 0; + const char *q = lx->p; + for (; q < lx->end; q++) { + depth += *q == '<'; + depth -= *q == '>'; + lx->line += *q == '\n'; + if (depth == 0) { + break; + } + } + lx->tok.k = DT_ID; + lx->tok.s = dg_s(lx->p + SKIP_ONE, (size_t)(q - lx->p) - (q > lx->p ? SKIP_ONE : 0)); + lx->p = q < lx->end ? q + SKIP_ONE : lx->end; + return; + } + const char *q = lx->p; + while (q < lx->end && (isalnum((unsigned char)*q) || *q == '_' || *q == '.' || + (unsigned char)*q >= 0x80 || (*q == '-' && q == lx->p))) { + q++; + } + if (q == lx->p) { + lx->p++; /* a stray byte */ + dt_next(lx); + return; + } + lx->tok.k = DT_ID; + lx->tok.s = dg_s(lx->p, (size_t)(q - lx->p)); + lx->p = q; +} + +/* `[a=b, c=d]` lists: the value of `label`, when one is set; url (when given) + * gets the value of `URL` or `href`. */ +static dg_str_t dt_attrs(dt_lex_t *lx, dg_str_t *url) { + dg_str_t label = dg_s(NULL, 0); + while (lx->tok.k == DT_LBRACK) { + dt_next(lx); + while (lx->tok.k != DT_RBRACK && lx->tok.k != DT_END) { + if (lx->tok.k == DT_ID) { + bool is_label = dg_eq(lx->tok.s, "label"); + bool is_url = dg_eq(lx->tok.s, "URL") || dg_eq(lx->tok.s, "href"); + dt_next(lx); + if (lx->tok.k == DT_EQ) { + dt_next(lx); + if (lx->tok.k == DT_ID && is_label) { + label = lx->tok.s; + } else if (lx->tok.k == DT_ID && is_url && url) { + *url = lx->tok.s; + } + } + } + dt_next(lx); + } + dt_next(lx); + } + return label; +} + +enum { DT_SET_MAX = 64, DT_DEPTH_MAX = 32 }; + +typedef struct { + dg_str_t ids[DT_SET_MAX]; + int n; +} dt_set_t; + +static void dt_stmts(dg_t *dg, dt_lex_t *lx, bool directed, dt_set_t *all, int depth, + dg_str_t *title); + +/* An edge operand: a node id (its port dropped) or a subgraph's nodes. */ +static bool dt_operand(dg_t *dg, dt_lex_t *lx, bool directed, dt_set_t *out, int depth, + dg_str_t *title) { + out->n = 0; + if (lx->tok.k == DT_ID && dg_eq(lx->tok.s, "subgraph")) { + dt_next(lx); + if (lx->tok.k == DT_ID) { + dt_next(lx); + } + } + if (lx->tok.k == DT_LBRACE) { + dt_next(lx); + dt_stmts(dg, lx, directed, out, depth + SKIP_ONE, title); + if (lx->tok.k == DT_RBRACE) { + dt_next(lx); + } + return true; + } + if (lx->tok.k != DT_ID) { + return false; + } + out->ids[out->n++] = lx->tok.s; + dt_next(lx); + while (lx->tok.k == DT_COLON) { /* a port, a compass point */ + dt_next(lx); + if (lx->tok.k == DT_ID) { + dt_next(lx); + } + } + return true; +} + +static void dt_stmt(dg_t *dg, dt_lex_t *lx, bool directed, dt_set_t *all, int depth, + dg_str_t *title) { + uint32_t line = lx->tok.line; + if (lx->tok.k == DT_ID && + (dg_eq(lx->tok.s, "graph") || dg_eq(lx->tok.s, "node") || dg_eq(lx->tok.s, "edge"))) { + dt_next(lx); + (void)dt_attrs(lx, NULL); + return; + } + dt_set_t left; + if (!dt_operand(dg, lx, directed, &left, depth, title)) { + dt_next(lx); + return; + } + if (lx->tok.k == DT_EQ) { /* a graph attribute */ + dt_next(lx); + if (lx->tok.k == DT_ID && left.n && dg_eq(left.ids[0], "label") && depth == 0 && + !title->n) { + *title = lx->tok.s; + } + dt_next(lx); + return; + } + if (lx->tok.k != DT_EDGE) { + dg_str_t url = dg_s(NULL, 0); + dg_str_t label = dt_attrs(lx, &url); + for (int i = 0; i < left.n; i++) { + dg_link(dg, dg_element(dg, left.ids[i], label, "node", line), url, + CBM_DOCLINK_DG_DOT_URL, line); + if (all->n < DT_SET_MAX) { + all->ids[all->n++] = left.ids[i]; + } + } + return; + } + /* an edge chain: a -> b -> {c d} [label=x] */ + dt_set_t chain[DT_DEPTH_MAX]; + int nc = 0; + chain[nc++] = left; + while (lx->tok.k == DT_EDGE && nc < DT_DEPTH_MAX) { + dt_next(lx); + if (!dt_operand(dg, lx, directed, &chain[nc], depth, title)) { + break; + } + nc++; + } + dg_str_t label = dt_attrs(lx, NULL); + for (int k = 0; k < nc; k++) { + for (int i = 0; i < chain[k].n; i++) { + dg_element(dg, chain[k].ids[i], dg_s(NULL, 0), "node", line); + if (all->n < DT_SET_MAX) { + all->ids[all->n++] = chain[k].ids[i]; + } + } + } + for (int k = 0; k + SKIP_ONE < nc; k++) { + for (int i = 0; i < chain[k].n; i++) { + for (int j = 0; j < chain[k + SKIP_ONE].n; j++) { + dg_arrow(dg, chain[k].ids[i], chain[k + SKIP_ONE].ids[j], label, + directed ? "arrow" : "line", line); + } + } + } +} + +static void dt_stmts(dg_t *dg, dt_lex_t *lx, bool directed, dt_set_t *all, int depth, + dg_str_t *title) { + if (depth > DT_DEPTH_MAX) { + return; + } + while (lx->tok.k != DT_RBRACE && lx->tok.k != DT_END && !dg->oom) { + if (lx->tok.k == DT_SEMI || lx->tok.k == DT_COMMA) { + dt_next(lx); + continue; + } + dt_stmt(dg, lx, directed, all, depth, title); + } +} + +static void dg_dot(dg_t *dg, dg_lines_t *it, dg_str_t *title) { + dt_lex_t lx = {it->p, it->end, it->line, {DT_END, {NULL, 0}, 0}}; + dt_next(&lx); + if (lx.tok.k == DT_ID && dg_eq(lx.tok.s, "strict")) { + dt_next(&lx); + } + bool directed = lx.tok.k == DT_ID && dg_eq(lx.tok.s, "digraph"); + snprintf(dg->kind, sizeof(dg->kind), "%s", directed ? "digraph" : "graph"); + dg->element_kind = "node"; + while (lx.tok.k != DT_LBRACE && lx.tok.k != DT_END) { + dt_next(&lx); + } + if (lx.tok.k == DT_END) { + return; + } + dt_next(&lx); + dt_set_t all = {.n = 0}; + dt_stmts(dg, &lx, directed, &all, 0, title); +} + +/* ── D2 ──────────────────────────────────────────────────────────── */ + +enum { D2_DEPTH_MAX = 16, D2_PATH_MAX = 512, D2_DELIM_MAX = 8 }; + +/* Keys that set an attribute of the shape they are written in. */ +static bool d2_reserved(dg_str_t key) { + static const char *const words[] = { + "shape", "style", "icon", "near", "direction", "tooltip", + "link", "width", "height", "class", "classes", "vars", + "label", "constraint", "top", "left", "grid-rows", "grid-columns", + "grid-gap", "layers", "scenarios", "steps", "source-arrowhead", "target-arrowhead", + "filled", "font-size", NULL}; + for (int k = 0; words[k]; k++) { + if (dg_eq(key, words[k])) { + return true; + } + } + return false; +} + +/* The containers a line is written in: the path of ids, and how far it + * reached at each enclosing level (restored by the closing brace). */ +typedef struct { + char path[D2_PATH_MAX]; + size_t lens[D2_DEPTH_MAX]; + int depth; + int skip_depth; /* inside a style, vars or connection block; -1 for none */ +} d2_scope_t; + +/* One level deeper: into the container `id`, or (no id) into a block that + * keeps the path. */ +static void d2_push(d2_scope_t *sc, dg_str_t id) { + if (sc->depth < D2_DEPTH_MAX) { + sc->lens[sc->depth] = strlen(sc->path); + if (id.n) { + char next[D2_PATH_MAX]; + snprintf(next, sizeof(next), "%.*s", (int)id.n, id.p); + memcpy(sc->path, next, sizeof(next)); + } + } + sc->depth++; +} + +static void d2_pop(d2_scope_t *sc) { + if (sc->depth == 0) { + return; + } + if (sc->skip_depth == sc->depth) { + sc->skip_depth = -1; + } + sc->depth--; + if (sc->depth < D2_DEPTH_MAX) { + sc->path[sc->lens[sc->depth]] = '\0'; + } +} + +/* The full id of a reference written inside the containers `path`. */ +static dg_str_t d2_id(dg_str_t ref, const char *path, char *buf, size_t cap) { + ref = dg_unquote(ref); + if (!path[0]) { + return ref; + } + int n = snprintf(buf, cap, "%s.%.*s", path, (int)ref.n, ref.p); + return n > 0 && (size_t)n < cap ? dg_s(buf, (size_t)n) : ref; +} + +/* A line without its `#` comment; a block string (`|`) ends the scan, its + * text is no comment. */ +static dg_str_t d2_strip_comment(dg_str_t ln) { + bool quoted = false; + for (size_t i = 0; i < ln.n; i++) { + quoted = ln.p[i] == '"' ? !quoted : quoted; + if (quoted) { + continue; + } + if (ln.p[i] == '|') { + break; + } + if (ln.p[i] == '#') { + return dg_trim(dg_s(ln.p, i)); + } + } + return ln; +} + +/* The first `:` outside quotes, NULL for none. */ +static const char *d2_colon(dg_str_t ln) { + bool quoted = false; + for (size_t i = 0; i < ln.n; i++) { + quoted = ln.p[i] == '"' ? !quoted : quoted; + if (!quoted && ln.p[i] == ':') { + return ln.p + i; + } + } + return NULL; +} + +/* A block string value (`|md ...|`, `||...||`, `|||...|||`, |`...`|): its + * text, read on through the lines it spans, and what follows its closing + * delimiter on that line. False when value opens none. */ +static bool d2_block(dg_lines_t *it, dg_str_t value, dg_str_t *text, dg_str_t *after) { + size_t pipes = 0; + while (pipes < value.n && value.p[pipes] == '|') { + pipes++; + } + if (pipes == 0 || pipes + PAIR_LEN > D2_DELIM_MAX) { + return false; + } + char close[D2_DELIM_MAX]; + size_t cl = 0; + size_t open = pipes; + if (open < value.n && value.p[open] == '`') { + close[cl++] = '`'; + open++; + } + for (size_t i = 0; i < pipes; i++) { + close[cl++] = '|'; + } + while (open < value.n && !isspace((unsigned char)value.p[open])) { /* the language tag */ + open++; + } + const char *from = value.p + open; + const char *at = NULL; + for (const char *q = from; q + cl <= it->end && !at; q++) { + at = memcmp(q, close, cl) == 0 ? q : NULL; + } + at = at ? at : it->end; + *text = dg_s(from, (size_t)(at - from)); + const char *rest = at + (at < it->end ? cl : 0); + const char *eol = memchr(rest, '\n', (size_t)(it->end - rest)); + eol = eol ? eol : it->end; + *after = dg_trim(dg_s(rest, (size_t)(eol - rest))); + dg_str_t skipped; + uint32_t l = 0; + while (it->p <= at && dg_next_line(it, &skipped, &l)) { /* the lines the block spans */ + } + return true; +} + +/* A block string's label: its first line of text, tags and heading marks + * dropped. */ +static size_t d2_block_label(dg_str_t text, char *out, size_t cap) { + dg_lines_t li = {text.p, text.p + text.n, 0}; + dg_str_t ln; + uint32_t l = 0; + while (dg_next_line(&li, &ln, &l)) { + size_t n = dg_label_text(ln, out, cap); + size_t k = 0; + while (k < n && (out[k] == '#' || out[k] == ' ')) { + k++; + } + if (k < n) { + memmove(out, out + k, n - k + SKIP_ONE); + return n - k; + } + } + out[0] = '\0'; + return 0; +} + +/* A value that is a block string read whole: the label it gives in buf, its + * text added to the diagram's docstring, and whether a `{` follows it. */ +static dg_str_t d2_value(dg_t *dg, dg_lines_t *it, dg_str_t value, char *buf, size_t cap, + bool *opens) { + dg_str_t text; + dg_str_t after; + if (!value.n || value.p[0] != '|' || !d2_block(it, value, &text, &after)) { + return value; + } + *opens = after.n && after.p[after.n - SKIP_ONE] == '{'; + size_t n = d2_block_label(text, buf, cap); + char all[DG_LABEL_MAX]; + size_t an = dg_label_text(text, all, sizeof(all)); + if (an > n) { + dg_doc_add(dg, all, an); + } + return dg_s(buf, n); +} + +/* A connection chain `a -> b <- c: label`: true when the line is one. */ +static bool d2_connection(dg_t *dg, dg_lines_t *it, dg_str_t ln, const char *path, uint32_t line, + bool *opens) { + static const char *const ops[] = {"<->", "->", "<-", "--", NULL}; + dg_str_t parts[DT_SET_MAX]; + const char *styles[DT_SET_MAX]; + int np = 0; + const char *colon = NULL; + bool quoted = false; + size_t start = 0; + for (size_t i = 0; i < ln.n && np + SKIP_ONE < DT_SET_MAX; i++) { + quoted = ln.p[i] == '"' ? !quoted : quoted; + if (quoted) { + continue; + } + if (ln.p[i] == ':') { + colon = ln.p + i; + break; + } + for (int k = 0; ops[k]; k++) { + size_t l = strlen(ops[k]); + if (i + l <= ln.n && memcmp(ln.p + i, ops[k], l) == 0) { + parts[np] = dg_trim(dg_s(ln.p + start, i - start)); + styles[np++] = k == 0 ? "bidirectional" + : k == 1 ? "arrow" + : k == PAIR_LEN ? "<" + : "line"; + start = i + l; + i += l - SKIP_ONE; + break; + } + } + } + if (np == 0) { + return false; + } + size_t end = colon ? (size_t)(colon - ln.p) : ln.n; + parts[np] = dg_trim(dg_s(ln.p + start, end - start)); + dg_str_t label = colon + ? dg_trim(dg_s(colon + SKIP_ONE, (size_t)(ln.p + ln.n - colon) - SKIP_ONE)) + : dg_s(NULL, 0); + char buf[DG_LABEL_MAX]; + label = d2_value(dg, it, label, buf, sizeof(buf), opens); + if (label.n && label.p[label.n - SKIP_ONE] == '{') { + label = dg_trim(dg_s(label.p, label.n - SKIP_ONE)); + } + for (int k = 0; k < np; k++) { + char ba[D2_PATH_MAX]; + char bb[D2_PATH_MAX]; + dg_str_t a = d2_id(parts[k], path, ba, sizeof(ba)); + dg_str_t b = d2_id(parts[k + SKIP_ONE], path, bb, sizeof(bb)); + if (!a.n || !b.n) { + continue; + } + if (strcmp(styles[k], "<") == 0) { + dg_arrow(dg, b, a, label, "arrow", line); + } else { + dg_arrow(dg, a, b, label, styles[k], line); + } + } + return true; +} + +/* The first reserved segment of a dotted key: its index, -1 for none. */ +static int d2_reserved_segment(dg_str_t key, size_t *seg_start, size_t *seg_len) { + int index = 0; + size_t s = 0; + bool quoted = false; + for (size_t i = 0; i <= key.n; i++) { + quoted = i < key.n && key.p[i] == '"' ? !quoted : quoted; + if (i < key.n && (quoted || key.p[i] != '.')) { + continue; + } + if (d2_reserved(dg_trim(dg_s(key.p + s, i - s)))) { + *seg_start = s; + *seg_len = i - s; + return index; + } + index++; + s = i + SKIP_ONE; + } + return -1; +} + +/* `key: value` (or `key: value {`) that is no connection. */ +static void d2_key(dg_t *dg, d2_scope_t *sc, dg_lines_t *it, dg_str_t ln, uint32_t line, bool opens, + dg_str_t *title) { + const char *colon = d2_colon(ln); + dg_str_t key = dg_trim(colon ? dg_s(ln.p, (size_t)(colon - ln.p)) + : dg_s(ln.p, ln.n - (opens ? SKIP_ONE : 0))); + dg_str_t value = colon + ? dg_trim(dg_s(colon + SKIP_ONE, (size_t)(ln.p + ln.n - colon) - SKIP_ONE)) + : dg_s(NULL, 0); + char vbuf[DG_LABEL_MAX]; + value = d2_value(dg, it, value, vbuf, sizeof(vbuf), &opens); + if (value.n && value.p[value.n - SKIP_ONE] == '{') { + value = dg_trim(dg_s(value.p, value.n - SKIP_ONE)); + } + size_t rs = 0; + size_t rl = 0; + int r = d2_reserved_segment(key, &rs, &rl); + if (r >= 0) { + /* an attribute: `label` names the shape it is set on */ + bool label = dg_eq(dg_trim(dg_s(key.p + rs, rl)), "label") && rs + rl == key.n; + char buf[D2_PATH_MAX]; + dg_str_t owner = r == 0 ? dg_s(sc->path, strlen(sc->path)) + : d2_id(dg_s(key.p, rs - SKIP_ONE), sc->path, buf, sizeof(buf)); + bool link = dg_eq(dg_trim(dg_s(key.p + rs, rl)), "link") && rs + rl == key.n; + if (label && owner.n && value.n) { + dg_element(dg, owner, value, "shape", line); + } else if (link && owner.n && value.n) { + dg_link(dg, dg_element(dg, owner, dg_s(NULL, 0), "shape", line), value, + CBM_DOCLINK_DG_D2_LINK, line); + } else if (label && value.n && r == 0) { /* the board's own label */ + const char *t = dg_dup(dg, value.p, value.n); + *title = t ? dg_s(t, value.n) : *title; + } + if (opens) { + d2_push(sc, dg_s(NULL, 0)); + sc->skip_depth = sc->depth; + } + return; + } + char buf[D2_PATH_MAX]; + dg_str_t id = d2_id(key, sc->path, buf, sizeof(buf)); + dg_element(dg, id, value, opens ? "container" : "shape", line); + if (opens) { + d2_push(sc, id); + } +} + +static void dg_d2(dg_t *dg, dg_lines_t *it, dg_str_t *title) { + snprintf(dg->kind, sizeof(dg->kind), "%s", "d2"); + dg->element_kind = "shape"; + d2_scope_t sc; + memset(&sc, 0, sizeof(sc)); + sc.skip_depth = -1; + dg_str_t ln; + uint32_t line = 0; + while (dg_next_line(it, &ln, &line)) { + ln = d2_strip_comment(dg_trim(ln)); + if (!ln.n) { + continue; + } + if (dg_eq(ln, "}")) { + d2_pop(&sc); + continue; + } + bool opens = ln.p[ln.n - SKIP_ONE] == '{'; + if (sc.skip_depth >= 0 || ln.p[0] == '(' || ln.p[0] == '|' || ln.p[0] == '*' || + ln.p[0] == '.' || ln.p[0] == '@') { + if (opens) { + d2_push(&sc, dg_s(NULL, 0)); + } + continue; + } + if (d2_connection(dg, it, ln, sc.path, line, &opens)) { + if (opens) { + d2_push(&sc, dg_s(NULL, 0)); + sc.skip_depth = sc.depth; /* a connection's attribute block */ + } + continue; + } + d2_key(dg, &sc, it, ln, line, opens, title); + } +} + +/* ── XML attributes (draw.io) ────────────────────────────────────── */ + +/* The value of attribute `name` in the tag text [p, end), entities + * decoded, into out; false when the tag has none. */ +static bool dg_xml_attr(const char *p, const char *end, const char *name, char *out, size_t cap) { + size_t nl = strlen(name); + for (const char *q = p; q + nl + PAIR_LEN < end; q++) { + if (memcmp(q, name, nl) != 0 || q[nl] != '=' || + (q[nl + SKIP_ONE] != '"' && q[nl + SKIP_ONE] != '\'') || + (q > p && !isspace((unsigned char)q[-SKIP_ONE]))) { + continue; + } + char quote = q[nl + SKIP_ONE]; + const char *v = q + nl + PAIR_LEN; + size_t w = 0; + for (; v < end && *v != quote && w + SKIP_ONE < cap; v++) { + if (*v != '&') { + out[w++] = *v; + continue; + } + static const struct { + const char *ent; + char c; + } ents[] = {{"<", '<'}, {">", '>'}, {"&", '&'}, {""", '"'}, + {"'", '\''}, {" ", ' '}, {" ", ' '}, {" ", ' '}, + {" ", ' '}, {NULL, 0}}; + bool hit = false; + for (int k = 0; ents[k].ent && !hit; k++) { + size_t el = strlen(ents[k].ent); + if (v + el <= end && memcmp(v, ents[k].ent, el) == 0) { + out[w++] = ents[k].c; + v += el - SKIP_ONE; + hit = true; + } + } + if (!hit) { + out[w++] = '&'; + } + } + out[w] = '\0'; + return true; + } + out[0] = '\0'; + return false; +} + +enum { DG_DRAWIO_CELLS = 4096, DG_ATTR_MAX = 4096 }; + +/* The text of a draw.io label: its HTML entities once more (an HTML label is + * escaped twice), its tags dropped. */ +static size_t dg_drawio_text(const char *html, char *out, size_t cap) { + char tmp[DG_ATTR_MAX]; + size_t w = 0; + for (const char *v = html; *v && w + SKIP_ONE < sizeof(tmp); v++) { + static const struct { + const char *ent; + char c; + } ents[] = {{"<", '<'}, {">", '>'}, {"&", '&'}, + {""", '"'}, {" ", ' '}, {NULL, 0}}; + bool hit = false; + for (int k = 0; ents[k].ent && !hit && *v == '&'; k++) { + size_t el = strlen(ents[k].ent); + if (strncmp(v, ents[k].ent, el) == 0) { + tmp[w++] = ents[k].c; + v += el - SKIP_ONE; + hit = true; + } + } + if (!hit) { + tmp[w++] = *v; + } + } + tmp[w] = '\0'; + /* a label cut short can end inside a tag: drop it, not show it */ + char *lt = strrchr(tmp, '<'); + if (lt && !strchr(lt, '>') && + (isalpha((unsigned char)lt[SKIP_ONE]) || lt[SKIP_ONE] == '/' || lt[SKIP_ONE] == '!')) { + *lt = '\0'; + w = (size_t)(lt - tmp); + } + return dg_label_text(dg_s(tmp, w), out, cap); +} + +typedef struct { + char id[DG_ID_MAX]; + char label[DG_LABEL_MAX]; + char source[DG_ID_MAX]; + char target[DG_ID_MAX]; + char parent[DG_ID_MAX]; + bool vertex; + bool edge; + bool text; /* a free text box */ + bool edge_label; /* the label of the edge `parent` */ + bool dashed; + bool no_arrow; + char kind[DG_KIND_MAX]; + const char *link; /* the cell's `link` attribute, NULL for none */ +} dg_cell_t; + +/* The cell a `` tag (inside an `` wrapper: `wrap`) says. */ +static void dg_drawio_cell(CBMArena *a, const char *tag, const char *tag_end, const char *wrap, + const char *wrap_end, dg_cell_t *c) { + char buf[DG_ATTR_MAX]; + memset(c, 0, sizeof(*c)); + if (dg_xml_attr(wrap ? wrap : tag, wrap ? wrap_end : tag_end, "link", buf, sizeof(buf)) && + buf[0]) { + c->link = cbm_arena_strdup(a, buf); + } + const char *idp = wrap ? wrap : tag; + const char *ide = wrap ? wrap_end : tag_end; + dg_xml_attr(idp, ide, "id", c->id, sizeof(c->id)); + if (dg_xml_attr(wrap ? wrap : tag, wrap ? wrap_end : tag_end, wrap ? "label" : "value", buf, + sizeof(buf))) { + dg_drawio_text(buf, c->label, sizeof(c->label)); + } + c->vertex = dg_xml_attr(tag, tag_end, "vertex", buf, sizeof(buf)) && buf[0] == '1'; + c->edge = dg_xml_attr(tag, tag_end, "edge", buf, sizeof(buf)) && buf[0] == '1'; + dg_xml_attr(tag, tag_end, "source", c->source, sizeof(c->source)); + dg_xml_attr(tag, tag_end, "target", c->target, sizeof(c->target)); + dg_xml_attr(tag, tag_end, "parent", c->parent, sizeof(c->parent)); + if (dg_xml_attr(tag, tag_end, "style", buf, sizeof(buf))) { + c->text = strncmp(buf, "text;", CBM_SZ_5) == 0 || strncmp(buf, "text", CBM_SZ_4) == 0; + c->edge_label = strstr(buf, "edgeLabel") != NULL; + c->dashed = strstr(buf, "dashed=1") != NULL; + c->no_arrow = strstr(buf, "endArrow=none") != NULL; + const char *shape = strstr(buf, "shape="); + const char *k = shape ? shape + CBM_SZ_6 : buf; + size_t kl = 0; + while (k[kl] && k[kl] != ';' && k[kl] != '=' && kl + SKIP_ONE < sizeof(c->kind)) { + kl++; + } + if (k[kl] == '=' && !shape) { + kl = 0; /* the first entry is a setting, not a shape */ + } + memcpy(c->kind, k, kl); + c->kind[kl] = '\0'; + } + if (!c->kind[0]) { + snprintf(c->kind, sizeof(c->kind), "%s", "shape"); + } +} + +/* The elements and arrows of one draw.io page's mxGraphModel. */ +static void dg_drawio(dg_t *dg, dg_lines_t *it, dg_str_t *title) { + (void)title; + snprintf(dg->kind, sizeof(dg->kind), "%s", "drawio"); + dg->element_kind = "shape"; + const char *p = it->p; + const char *end = it->end; + dg_cell_t *cells = + (dg_cell_t *)cbm_arena_alloc(dg->ctx->arena, DG_DRAWIO_CELLS * sizeof(dg_cell_t)); + if (!cells) { + dg->oom = true; + return; + } + int n = 0; + for (const char *q = p; q < end && n < DG_DRAWIO_CELLS;) { + const char *lt = memchr(q, '<', (size_t)(end - q)); + if (!lt) { + break; + } + const char *gt = memchr(lt, '>', (size_t)(end - lt)); + if (!gt) { + break; + } + bool object = (size_t)(end - lt) > CBM_SZ_8 && (memcmp(lt, "', (size_t)(end - cell)); + if (cend) { + dg_drawio_cell(dg->ctx->arena, cell, cend, lt, gt, &cells[n++]); + q = cend + SKIP_ONE; + continue; + } + } + } else if ((size_t)(end - lt) > CBM_SZ_7 && memcmp(lt, "ctx->arena, lt, gt, NULL, NULL, &cells[n++]); + } + q = gt + SKIP_ONE; + } + uint32_t line = dg->line0; + for (int i = 0; i < n; i++) { /* the shapes with text */ + dg_cell_t *c = &cells[i]; + if (c->vertex && !c->edge_label && c->label[0]) { + if (c->text) { + dg_doc_add(dg, c->label, strlen(c->label)); + } else { + const char *qn = dg_element(dg, dg_s(c->id, strlen(c->id)), + dg_s(c->label, strlen(c->label)), c->kind, line); + dg_link(dg, qn, dg_s(c->link, c->link ? strlen(c->link) : 0), + CBM_DOCLINK_DG_DRAWIO_LINK, line); + } + } + } + for (int i = 0; i < n; i++) { /* the connections */ + dg_cell_t *c = &cells[i]; + if (!c->edge || !c->source[0] || !c->target[0]) { + continue; + } + const char *label = c->label; + for (int k = 0; k < n && !label[0]; k++) { + if (cells[k].edge_label && strcmp(cells[k].parent, c->id) == 0) { + label = cells[k].label; + } + } + if (!cbm_ht_get(dg->ids, c->source) || !cbm_ht_get(dg->ids, c->target)) { + continue; /* an end without text names nothing */ + } + dg_arrow(dg, dg_s(c->source, strlen(c->source)), dg_s(c->target, strlen(c->target)), + dg_s(label, strlen(label)), + c->dashed ? "dotted" + : c->no_arrow ? "line" + : "arrow", + line); + } +} + +/* ── Excalidraw ──────────────────────────────────────────────────── */ + +enum { DG_EX_MARGIN = 8 }; /* px: how near a loose arrow end comes to the shape it meets */ + +/* A shape of the scene: its box, and the label it gets. */ +typedef struct { + const char *id; + const char *type; + double x0; + double y0; + double x1; + double y1; + double area; + const char *label; + double label_x; /* a loose label's place: the topmost text inside wins */ + double label_y; + const char *link; /* the shape's `link`, NULL for none */ + bool fixed; /* bound text or a frame's name: no loose text replaces it */ + bool region; /* a box that can hold loose text and meet loose arrow ends */ + bool frame; /* a region around every other shape, the drawing's frame: loose + * arrow ends meet it at its border only */ +} ex_shape_t; + +static const char *dg_json_str(yyjson_val *obj, const char *key) { + yyjson_val *v = yyjson_obj_get(obj, key); + return yyjson_is_str(v) ? yyjson_get_str(v) : NULL; +} + +static double ex_num(yyjson_val *obj, const char *key) { + yyjson_val *v = yyjson_obj_get(obj, key); + return yyjson_is_num(v) ? yyjson_get_num(v) : 0.0; +} + +static bool ex_live(yyjson_val *e) { + return !yyjson_get_bool(yyjson_obj_get(e, "isDeleted")); +} + +static bool ex_is(const char *type, const char *const *kinds) { + for (int k = 0; kinds[k]; k++) { + if (strcmp(type, kinds[k]) == 0) { + return true; + } + } + return false; +} + +static const char *const EX_REGIONS[] = {"rectangle", "ellipse", "diamond", "frame", + "magicframe", "embeddable", "iframe", NULL}; +static const char *const EX_NOT_SHAPES[] = {"text", "arrow", "line", "freedraw", "selection", NULL}; + +static int ex_by_area(const void *a, const void *b) { + const ex_shape_t *x = (const ex_shape_t *)a; + const ex_shape_t *y = (const ex_shape_t *)b; + if (x->area != y->area) { + return x->area < y->area ? -1 : SKIP_ONE; + } + return strcmp(x->id, y->id); +} + +/* The smallest region whose box, grown by margin, holds (x, y): -1 for none. + * The shapes are sorted by area, so the first hit is the smallest. An arrow + * end (labelled) deep inside the drawing's frame points at something in the + * drawing, not at the frame: none. */ +static int ex_region_at(const ex_shape_t *s, int n, double x, double y, double margin, + bool labelled) { + for (int i = 0; i < n; i++) { + if (!s[i].region || (labelled && !s[i].label)) { + continue; + } + if (x >= s[i].x0 - margin && x <= s[i].x1 + margin && y >= s[i].y0 - margin && + y <= s[i].y1 + margin) { + bool deep = x > s[i].x0 + margin && x < s[i].x1 - margin && y > s[i].y0 + margin && + y < s[i].y1 - margin; + return labelled && s[i].frame && deep ? -1 : i; + } + } + return -1; +} + +/* A loose text: the label of the smallest region it sits in (the topmost + * text of several), else the diagram's docstring. */ +static void ex_loose_text(dg_t *dg, ex_shape_t *s, int n, yyjson_val *e, const char *text) { + double x = ex_num(e, "x"); + double y = ex_num(e, "y"); + int i = ex_region_at(s, n, x + ex_num(e, "width") / PAIR_LEN, + y + ex_num(e, "height") / PAIR_LEN, 0.0, false); + if (i < 0 || s[i].fixed) { + dg_doc_add(dg, text, strlen(text)); + return; + } + if (s[i].label && (s[i].label_y < y || (s[i].label_y == y && s[i].label_x <= x))) { + dg_doc_add(dg, text, strlen(text)); + return; + } + if (s[i].label) { + dg_doc_add(dg, s[i].label, strlen(s[i].label)); + } + s[i].label = text; + s[i].label_x = x; + s[i].label_y = y; +} + +/* The end bound to free text: the shape that text names, else the text as an + * element of its own. */ +static const char *ex_text_end(dg_t *dg, const ex_shape_t *s, int n, CBMHashTable *texts, + const char *id) { + const char *text = (const char *)cbm_ht_get(texts, id); + if (!text) { + return NULL; + } + for (int i = 0; i < n; i++) { + if (s[i].label == text) { + return s[i].id; + } + } + return dg_element(dg, dg_s(id, strlen(id)), dg_s(text, strlen(text)), "text", dg->line0) ? id + : NULL; +} + +/* An arrow end: the shape (or free text) it is bound to, else (arrows only) + * the smallest labelled region the point meets. */ +static const char *ex_end(dg_t *dg, const ex_shape_t *s, int n, CBMHashTable *texts, yyjson_val *e, + const char *binding, bool first, bool loose_ok) { + const char *bound = dg_json_str(yyjson_obj_get(e, binding), "elementId"); + if (bound) { + return cbm_ht_get(dg->ids, bound) ? bound : ex_text_end(dg, s, n, texts, bound); + } + yyjson_val *pts = yyjson_obj_get(e, "points"); + yyjson_val *pt = first ? yyjson_arr_get_first(pts) : yyjson_arr_get_last(pts); + if (!loose_ok || !yyjson_is_arr(pt)) { + return NULL; + } + double x = ex_num(e, "x") + yyjson_get_num(yyjson_arr_get(pt, 0)); + double y = ex_num(e, "y") + yyjson_get_num(yyjson_arr_get(pt, SKIP_ONE)); + int i = ex_region_at(s, n, x, y, DG_EX_MARGIN, true); + return i >= 0 ? s[i].id : NULL; +} + +static void ex_arrows(dg_t *dg, const ex_shape_t *s, int n, yyjson_val *els, CBMHashTable *labels, + CBMHashTable *texts) { + size_t i; + size_t max; + yyjson_val *e; + yyjson_arr_foreach(els, i, max, e) { + const char *type = dg_json_str(e, "type"); + const char *id = dg_json_str(e, "id"); + if (!type || !id || !ex_live(e) || + (strcmp(type, "arrow") != 0 && strcmp(type, "line") != 0)) { + continue; + } + bool arrow = strcmp(type, "arrow") == 0; + const char *from = ex_end(dg, s, n, texts, e, "startBinding", true, arrow); + const char *to = ex_end(dg, s, n, texts, e, "endBinding", false, arrow); + bool loose = !dg_json_str(yyjson_obj_get(e, "startBinding"), "elementId") || + !dg_json_str(yyjson_obj_get(e, "endBinding"), "elementId"); + if (!from || !to || (loose && strcmp(from, to) == 0)) { + continue; /* unbound, bound to a shape without text, or a loose loop in one box */ + } + const char *label = (const char *)cbm_ht_get(labels, id); + dg_arrow(dg, dg_s(from, strlen(from)), dg_s(to, strlen(to)), + dg_s(label ? label : "", label ? strlen(label) : 0), arrow ? "arrow" : "line", + dg->line0); + } +} + +static void dg_excalidraw(dg_t *dg, dg_lines_t *it, dg_str_t *title) { + (void)title; + snprintf(dg->kind, sizeof(dg->kind), "%s", "excalidraw"); + dg->element_kind = "shape"; + yyjson_doc *doc = yyjson_read(it->p, (size_t)(it->end - it->p), 0); + yyjson_val *els = doc ? yyjson_obj_get(yyjson_doc_get_root(doc), "elements") : NULL; + size_t total = yyjson_arr_size(els); + CBMHashTable *labels = + yyjson_is_arr(els) ? cbm_ht_create_in(CBM_MEM_CLASS_OTHER, CBM_SZ_64) : NULL; + ex_shape_t *s = labels && total + ? (ex_shape_t *)cbm_arena_alloc(dg->ctx->arena, total * sizeof(ex_shape_t)) + : NULL; + CBMHashTable *texts = s ? cbm_ht_create_in(CBM_MEM_CLASS_OTHER, CBM_SZ_64) : NULL; + if (!texts) { + cbm_ht_free(labels); + yyjson_doc_free(doc); + return; + } + size_t i; + size_t max; + yyjson_val *e; + yyjson_arr_foreach(els, i, max, e) { /* bound text: the label of its container */ + const char *text = dg_json_str(e, "originalText"); + text = text ? text : dg_json_str(e, "text"); + const char *type = dg_json_str(e, "type"); + const char *container = dg_json_str(e, "containerId"); + const char *id = dg_json_str(e, "id"); + if (ex_live(e) && type && strcmp(type, "text") == 0 && text && container) { + cbm_ht_set(labels, container, (void *)text); + } else if (ex_live(e) && type && strcmp(type, "text") == 0 && text && id) { + cbm_ht_set(texts, id, (void *)text); /* free text an arrow may be bound to */ + } + } + int n = 0; + yyjson_arr_foreach(els, i, max, e) { /* the shapes */ + const char *type = dg_json_str(e, "type"); + const char *id = dg_json_str(e, "id"); + if (!type || !id || !ex_live(e) || ex_is(type, EX_NOT_SHAPES)) { + continue; + } + ex_shape_t *sh = &s[n++]; + memset(sh, 0, sizeof(*sh)); + sh->id = id; + sh->type = type; + sh->x0 = ex_num(e, "x"); + sh->y0 = ex_num(e, "y"); + sh->x1 = sh->x0 + ex_num(e, "width"); + sh->y1 = sh->y0 + ex_num(e, "height"); + sh->area = (sh->x1 - sh->x0) * (sh->y1 - sh->y0); + sh->region = ex_is(type, EX_REGIONS); + sh->label = (const char *)cbm_ht_get(labels, id); + sh->label = sh->label ? sh->label : dg_json_str(e, "name"); /* a frame's name */ + sh->fixed = sh->label != NULL; + sh->link = dg_json_str(e, "link"); + } + qsort(s, (size_t)n, sizeof(ex_shape_t), ex_by_area); + yyjson_arr_foreach(els, i, max, e) { /* loose text: in a shape, or the docstring */ + const char *type = dg_json_str(e, "type"); + const char *text = dg_json_str(e, "originalText"); + text = text ? text : dg_json_str(e, "text"); + if (ex_live(e) && type && strcmp(type, "text") == 0 && text && + !dg_json_str(e, "containerId")) { + ex_loose_text(dg, s, n, e, text); + } + } + if (n > SKIP_ONE && s[n - SKIP_ONE].region) { /* only the largest box can frame the rest */ + ex_shape_t *f = &s[n - SKIP_ONE]; + bool all = true; + for (int j = 0; j < n - SKIP_ONE && all; j++) { + all = s[j].x0 >= f->x0 && s[j].x1 <= f->x1 && s[j].y0 >= f->y0 && s[j].y1 <= f->y1; + } + f->frame = all; + } + for (int k = 0; k < n; k++) { + if (s[k].label) { + const char *qn = dg_element(dg, dg_s(s[k].id, strlen(s[k].id)), + dg_s(s[k].label, strlen(s[k].label)), s[k].type, dg->line0); + dg_link(dg, qn, dg_s(s[k].link, s[k].link ? strlen(s[k].link) : 0), + CBM_DOCLINK_DG_EXCALIDRAW_LINK, dg->line0); + } + } + ex_arrows(dg, s, n, els, labels, texts); + cbm_ht_free(texts); + cbm_ht_free(labels); + yyjson_doc_free(doc); +} + +/* ── One diagram ─────────────────────────────────────────────────── */ + +typedef struct { + const char *format; + const char *title; /* a draw.io page's name, NULL for none */ + const char *fallback; /* the name when the text gives none: a diagram file's */ + const char *parent_qn; /* the Section's, NULL: the file */ + int ordinal; + const char *text; + size_t len; + uint32_t first_line; /* the text's first line in the file */ + uint32_t start_line; /* the block's lines */ + uint32_t end_line; +} dg_source_t; + +static void dg_build(CBMExtractCtx *ctx, const dg_source_t *src) { + dg_t dg; + memset(&dg, 0, sizeof(dg)); + dg.ctx = ctx; + dg.format = src->format; + const char *parent = src->parent_qn ? src->parent_qn : ctx->module_qn; + if (!parent) { + return; + } + dg.qn = cbm_arena_sprintf(ctx->arena, "%s#diagram-%d", parent, src->ordinal); + dg.ids = cbm_ht_create_in(CBM_MEM_CLASS_OTHER, CBM_SZ_32); + if (!dg.qn || !dg.ids) { + cbm_ht_free(dg.ids); + return; + } + CBMDefinition def; + memset(&def, 0, sizeof(def)); + def.label = "Diagram"; + def.qualified_name = dg.qn; + def.file_path = ctx->rel_path; + def.start_line = src->start_line; + def.end_line = src->end_line; + def.parent_class = src->parent_qn; + def.name = src->format; + cbm_defs_push(&ctx->result->defs, ctx->arena, def); + dg.def_index = ctx->result->defs.count - SKIP_ONE; + dg.line0 = src->first_line; + dg_lines_t it = {src->text, src->text + src->len, src->first_line}; + dg_str_t title = src->title ? dg_s(src->title, strlen(src->title)) : dg_s(NULL, 0); + if (strcmp(src->format, "mermaid") == 0) { + dg_mermaid(&dg, &it, &title); + } else if (strcmp(src->format, "plantuml") == 0) { + dg_plantuml(&dg, &it, &title); + } else if (strcmp(src->format, "dot") == 0) { + dg_dot(&dg, &it, &title); + } else if (strcmp(src->format, "d2") == 0) { + dg_d2(&dg, &it, &title); + } else if (strcmp(src->format, "drawio") == 0) { + dg_drawio(&dg, &it, &title); + } else if (strcmp(src->format, "excalidraw") == 0) { + dg_excalidraw(&dg, &it, &title); + } + cbm_ht_free(dg.ids); + CBMDefinition *d = &ctx->result->defs.items[dg.def_index]; + char tbuf[DG_LABEL_MAX]; + size_t tl = dg_label_text(title, tbuf, sizeof(tbuf)); + d->name = tl ? cbm_arena_strndup(ctx->arena, tbuf, tl) + : src->fallback ? src->fallback + : cbm_arena_sprintf(ctx->arena, "%s %s", src->format, + dg.kind[0] ? dg.kind : "diagram"); + d->docstring = dg.doc_len ? cbm_arena_strndup(ctx->arena, dg.doc, dg.doc_len) : NULL; + const char *kind = dg.kind[0] ? cbm_arena_strdup(ctx->arena, dg.kind) : "unknown"; + const char **kv = (const char **)cbm_arena_alloc(ctx->arena, CBM_SZ_5 * sizeof(char *)); + if (kv) { + kv[0] = "format"; + kv[SKIP_ONE] = src->format; + kv[PAIR_LEN] = "diagram_kind"; + kv[CBM_SZ_3] = kind; + kv[CBM_SZ_4] = NULL; + d->extra_props = kv; + } +} + +/* ── Sources ─────────────────────────────────────────────────────── */ + +/* The diagram format an info string names, NULL for none. */ +static const char *dg_format_of(const char *s, size_t n) { + static const struct { + const char *word; + const char *format; + } names[] = { + {"mermaid", "mermaid"}, {"plantuml", "plantuml"}, {"puml", "plantuml"}, {"uml", "plantuml"}, + {"dot", "dot"}, {"graphviz", "dot"}, {"d2", "d2"}, {NULL, NULL}}; + for (int k = 0; names[k].word; k++) { + size_t l = strlen(names[k].word); + bool same = n == l; + for (size_t i = 0; same && i < l; i++) { + same = tolower((unsigned char)s[i]) == names[k].word[i]; + } + if (same) { + return names[k].format; + } + } + return NULL; +} + +/* The Section a line of the file is written under: the last heading at or + * before it, NULL above the first. */ +static const char *dg_section_at(const CBMExtractCtx *ctx, uint32_t line) { + const char *qn = NULL; + uint32_t best = 0; + const CBMDefArray *defs = &ctx->result->defs; + for (int i = 0; i < defs->count; i++) { + const CBMDefinition *d = &defs->items[i]; + if (d->label && strcmp(d->label, "Section") == 0 && d->start_line <= line && + d->start_line >= best) { + best = d->start_line; + qn = d->qualified_name; + } + } + return qn; +} + +typedef struct { + const char *parent; + int count; +} dg_ordinal_t; + +enum { DG_PARENTS_MAX = 512 }; + +/* The next ordinal of a parent's diagrams. */ +static int dg_next_ordinal(dg_ordinal_t *ords, int *n, const char *parent) { + for (int i = 0; i < *n; i++) { + if (ords[i].parent == parent) { + return ++ords[i].count; + } + } + if (*n < DG_PARENTS_MAX) { + ords[(*n)++] = (dg_ordinal_t){parent, SKIP_ONE}; + } + return SKIP_ONE; +} + +static bool dg_kind(TSNode n, const char *k) { + return !ts_node_is_null(n) && strcmp(ts_node_type(n), k) == 0; +} + +/* A fenced block of a Markdown file: a diagram when its info string says so. */ +static void dg_markdown_fence(CBMExtractCtx *ctx, TSNode block, dg_ordinal_t *ords, int *nords) { + TSNode info = {0}; + TSNode content = {0}; + uint32_t nc = ts_node_named_child_count(block); + for (uint32_t i = 0; i < nc; i++) { + TSNode c = ts_node_named_child(block, i); + info = dg_kind(c, "info_string") ? c : info; + content = dg_kind(c, "code_fence_content") ? c : content; + } + if (ts_node_is_null(info) || ts_node_is_null(content)) { + return; + } + TSNode lang = ts_node_named_child(info, 0); + TSNode word = dg_kind(lang, "language") ? lang : info; + uint32_t wb = ts_node_start_byte(word); + uint32_t we = ts_node_end_byte(word); + const char *format = dg_format_of(ctx->source + wb, we - wb); + if (!format) { + return; + } + uint32_t start = ts_node_start_point(block).row + SKIP_ONE; + const char *parent = dg_section_at(ctx, start); + uint32_t cb = ts_node_start_byte(content); + dg_source_t src = { + .format = format, + .parent_qn = parent, + .ordinal = dg_next_ordinal(ords, nords, parent), + .text = ctx->source + cb, + .len = ts_node_end_byte(content) - cb, + .first_line = ts_node_start_point(content).row + SKIP_ONE, + .start_line = start, + .end_line = ts_node_end_point(block).row + SKIP_ONE, + }; + dg_build(ctx, &src); +} + +static void dg_markdown(CBMExtractCtx *ctx) { + dg_ordinal_t *ords = + (dg_ordinal_t *)cbm_arena_alloc(ctx->arena, DG_PARENTS_MAX * sizeof(dg_ordinal_t)); + if (!ords) { + return; + } + int nords = 0; + TSTreeCursor cur = ts_tree_cursor_new(ctx->root); + for (bool more = true; more;) { + TSNode n = ts_tree_cursor_current_node(&cur); + if (dg_kind(n, "fenced_code_block")) { + dg_markdown_fence(ctx, n, ords, &nords); + } else if (ts_tree_cursor_goto_first_child(&cur)) { + continue; + } + while (!ts_tree_cursor_goto_next_sibling(&cur)) { + if (!ts_tree_cursor_goto_parent(&cur)) { + more = false; + break; + } + } + } + ts_tree_cursor_delete(&cur); +} + +/* ── RST directives and AsciiDoc blocks ──────────────────────────── */ + +/* The diagram format a directive (RST) or block style (AsciiDoc) names; for + * Sphinx's `digraph` / `graph` also the graph keyword its body goes inside. */ +static const char *dg_block_format(dg_str_t word, const char **wrap) { + static const struct { + const char *word; + const char *format; + const char *wrap; + } names[] = { + {"mermaid", "mermaid", NULL}, {"uml", "plantuml", NULL}, {"plantuml", "plantuml", NULL}, + {"graphviz", "dot", NULL}, {"dot", "dot", NULL}, {"digraph", "dot", "digraph"}, + {"graph", "dot", "graph"}, {"d2", "d2", NULL}, {NULL, NULL, NULL}}; + *wrap = NULL; + for (int k = 0; names[k].word; k++) { + if (dg_eq(word, names[k].word)) { + *wrap = names[k].wrap; + return names[k].format; + } + } + return NULL; +} + +static dg_str_t dg_node_text(const CBMExtractCtx *ctx, TSNode n) { + uint32_t b = ts_node_start_byte(n); + return dg_s(ctx->source + b, ts_node_end_byte(n) - b); +} + +/* The first descendant of kind k, a null node for none. */ +static TSNode dg_find(TSNode n, const char *k) { + uint32_t nc = ts_node_named_child_count(n); + for (uint32_t i = 0; i < nc; i++) { + TSNode c = ts_node_named_child(n, i); + if (dg_kind(c, k)) { + return c; + } + TSNode d = dg_find(c, k); + if (!ts_node_is_null(d)) { + return d; + } + } + return (TSNode){0}; +} + +/* A directive's `:caption:` option, NULL for none. */ +static const char *dg_rst_caption(const CBMExtractCtx *ctx, TSNode options) { + uint32_t nc = ts_node_is_null(options) ? 0 : ts_node_named_child_count(options); + for (uint32_t i = 0; i < nc; i++) { + TSNode f = ts_node_named_child(options, i); + TSNode name = dg_find(f, "field_name"); + TSNode body = dg_find(f, "field_body"); + dg_str_t key = ts_node_is_null(name) ? dg_s("", 0) : dg_trim(dg_node_text(ctx, name)); + while (key.n && key.p[0] == ':') { /* `:caption:` or `caption` */ + key = dg_s(key.p + SKIP_ONE, key.n - SKIP_ONE); + } + while (key.n && key.p[key.n - SKIP_ONE] == ':') { + key.n--; + } + if (!ts_node_is_null(body) && dg_eq(key, "caption")) { + dg_str_t v = dg_trim(dg_node_text(ctx, body)); + return cbm_arena_strndup(ctx->arena, v.p, v.n); + } + } + return NULL; +} + +/* `.. mermaid::`, `.. uml::`, `.. graphviz::`, `.. digraph:: name`: a + * diagram when the directive holds its text (a file argument is the file, + * indexed as itself). */ +static void dg_rst_directive(CBMExtractCtx *ctx, TSNode dir, dg_ordinal_t *ords, int *nords) { + TSNode type = dg_find(dir, "type"); + const char *wrap = NULL; + const char *format = + ts_node_is_null(type) ? NULL : dg_block_format(dg_trim(dg_node_text(ctx, type)), &wrap); + TSNode content = format ? dg_find(dir, "content") : (TSNode){0}; + if (ts_node_is_null(content)) { + return; + } + dg_str_t text = dg_node_text(ctx, content); + uint32_t first = ts_node_start_point(content).row + SKIP_ONE; + if (wrap) { /* Sphinx writes the statements only */ + TSNode args = dg_find(dir, "arguments"); + dg_str_t name = ts_node_is_null(args) ? dg_s("", 0) : dg_trim(dg_node_text(ctx, args)); + char *w = cbm_arena_sprintf(ctx->arena, "%s \"%.*s\" {\n%.*s\n}\n", wrap, (int)name.n, + name.p, (int)text.n, text.p); + if (!w) { + return; + } + text = dg_s(w, strlen(w)); + first--; + } + uint32_t start = ts_node_start_point(dir).row + SKIP_ONE; + const char *parent = dg_section_at(ctx, start); + dg_source_t src = { + .format = format, + .title = dg_rst_caption(ctx, dg_find(dir, "options")), + .parent_qn = parent, + .ordinal = dg_next_ordinal(ords, nords, parent), + .text = text.p, + .len = text.n, + .first_line = first, + .start_line = start, + .end_line = ts_node_end_point(dir).row + SKIP_ONE, + }; + dg_build(ctx, &src); +} + +static void dg_rst(CBMExtractCtx *ctx) { + dg_ordinal_t *ords = + (dg_ordinal_t *)cbm_arena_alloc(ctx->arena, DG_PARENTS_MAX * sizeof(dg_ordinal_t)); + if (!ords) { + return; + } + int nords = 0; + TSTreeCursor cur = ts_tree_cursor_new(ctx->root); + for (bool more = true; more;) { + TSNode n = ts_tree_cursor_current_node(&cur); + if (dg_kind(n, "directive")) { + dg_rst_directive(ctx, n, ords, &nords); + } else if (ts_tree_cursor_goto_first_child(&cur)) { + continue; + } + while (!ts_tree_cursor_goto_next_sibling(&cur)) { + if (!ts_tree_cursor_goto_parent(&cur)) { + more = false; + break; + } + } + } + ts_tree_cursor_delete(&cur); +} + +/* An AsciiDoc block attribute line `[plantuml, target=x]` or + * `[source,mermaid]`: the diagram format it names, NULL for none. */ +static const char *dg_adoc_style(dg_str_t ln) { + if (ln.n < CBM_SZ_3 || ln.p[0] != '[' || ln.p[ln.n - SKIP_ONE] != ']' || + ln.p[SKIP_ONE] == '[') { + return NULL; + } + dg_str_t attrs = dg_s(ln.p + SKIP_ONE, ln.n - PAIR_LEN); + const char *comma = memchr(attrs.p, ',', attrs.n); + dg_str_t style = dg_trim(dg_s(attrs.p, comma ? (size_t)(comma - attrs.p) : attrs.n)); + if (dg_eq(style, "source") && comma) { /* a listing in a diagram language */ + dg_str_t rest = dg_s(comma + SKIP_ONE, (size_t)(attrs.p + attrs.n - comma) - SKIP_ONE); + const char *c2 = memchr(rest.p, ',', rest.n); + style = dg_trim(dg_s(rest.p, c2 ? (size_t)(c2 - rest.p) : rest.n)); + } + const char *wrap = NULL; + const char *format = dg_block_format(style, &wrap); + return wrap ? NULL : format; +} + +/* `----` or `....` (four or more): a listing or literal block delimiter. */ +static bool dg_adoc_delimiter(dg_str_t ln) { + if (ln.n < CBM_SZ_4 || (ln.p[0] != '-' && ln.p[0] != '.')) { + return false; + } + for (size_t i = SKIP_ONE; i < ln.n; i++) { + if (ln.p[i] != ln.p[0]) { + return false; + } + } + return true; +} + +/* AsciiDoc diagram blocks (Asciidoctor Diagram): the delimited block after a + * `[mermaid]`, `[plantuml]`, `[graphviz]` or `[d2]` line, titled by the + * `.Title` line before it. */ +static void dg_asciidoc(CBMExtractCtx *ctx) { + dg_ordinal_t *ords = + (dg_ordinal_t *)cbm_arena_alloc(ctx->arena, DG_PARENTS_MAX * sizeof(dg_ordinal_t)); + if (!ords) { + return; + } + int nords = 0; + dg_lines_t it = {ctx->source, ctx->source + ctx->source_len, SKIP_ONE}; + dg_str_t title = dg_s(NULL, 0); + dg_str_t ln; + uint32_t line = 0; + while (dg_next_line(&it, &ln, &line)) { + dg_str_t t = dg_trim(ln); + if (t.n > SKIP_ONE && t.p[0] == '.' && t.p[SKIP_ONE] != '.' && + !isspace((unsigned char)t.p[SKIP_ONE])) { + title = dg_s(t.p + SKIP_ONE, t.n - SKIP_ONE); + continue; + } + const char *format = dg_adoc_style(t); + dg_str_t delim; + uint32_t attr_line = line; + if (!format || !dg_next_line(&it, &delim, &line) || !dg_adoc_delimiter(dg_trim(delim))) { + title = t.n && t.p[0] == '[' ? title : dg_s(NULL, 0); /* attribute lines stack */ + continue; + } + delim = dg_trim(delim); + const char *body = it.p; + uint32_t first = it.line; + const char *stop = it.end; + uint32_t last = line; + while (dg_next_line(&it, &ln, &line)) { + dg_str_t c = dg_trim(ln); + if (c.n == delim.n && memcmp(c.p, delim.p, c.n) == 0) { + stop = ln.p; + last = line; + break; + } + } + const char *parent = dg_section_at(ctx, attr_line); + dg_source_t src = { + .format = format, + .title = title.n ? cbm_arena_strndup(ctx->arena, title.p, title.n) : NULL, + .parent_qn = parent, + .ordinal = dg_next_ordinal(ords, &nords, parent), + .text = body, + .len = (size_t)(stop - body), + .first_line = first, + .start_line = attr_line, + .end_line = last, + }; + dg_build(ctx, &src); + title = dg_s(NULL, 0); + } +} + +/* ── Diagram files ───────────────────────────────────────────────── */ + +enum { + DG_INFLATE_MAX = 64 * 1024 * 1024, /* a draw.io page; beyond it the page is skipped */ + DG_STARTUML_LEN = 9, /* @startuml */ + DG_DIAGRAM_CLOSE_LEN = 10, /* */ + DG_CONTENT_ATTR_LEN = 9, /* content=\" */ + DG_PAYLOAD_START_LEN = 22, /* */ + DG_PAYLOAD_VERSION_LEN = 21, /* "); + const char *stop = start ? strstr(start, "") : NULL; + if (!stop) { + return NULL; + } + const char *b = start + DG_PAYLOAD_START_LEN; + size_t bn = (size_t)(stop - b); + unsigned char *raw = (unsigned char *)cbm_arena_alloc(ctx->arena, bn + SKIP_ONE); + size_t rn = raw ? dg_base64(b, bn, raw, bn) : 0; + if (!rn) { + return NULL; + } + raw[rn] = '\0'; + const char *ver = strstr(ctx->source, "|calls| B(API)\n" /* 12 */ + " B --> C[(DB)] & D{Cache}\n" /* 13 */ + " B -. retries .-> E\n" /* 14 */ + " subgraph S [Backend]\n" /* 15 */ + " end\n" /* 16 */ + "```\n" /* 17 */ + "\n" /* 18 */ + "```mermaid\n" /* 19 */ + "sequenceDiagram\n" /* 20 */ + " participant U as User\n" /* 21 */ + " U->>API: login\n" /* 22 */ + " API-->>U: token\n" /* 23 */ + "```\n" /* 24 */ + "\n" /* 25 */ + "## Model\n" /* 26 */ + "\n" /* 27 */ + "```mermaid\n" /* 28 */ + "classDiagram\n" /* 29 */ + " Animal <|-- Dog : is a\n" /* 30 */ + " class Shop[\"Pet shop\"]\n" /* 31 */ + " Shop *-- Dog\n" /* 32 */ + "```\n"; /* 33 */ + +static const CBMDefinition *dg_def(const CBMFileResult *r, const char *qn_suffix) { + size_t sl = strlen(qn_suffix); + for (int i = 0; i < r->defs.count; i++) { + const char *qn = r->defs.items[i].qualified_name; + size_t ql = qn ? strlen(qn) : 0; + if (ql >= sl && strcmp(qn + ql - sl, qn_suffix) == 0) { + return &r->defs.items[i]; + } + } + printf(" no definition *%s\n", qn_suffix); + return NULL; +} + +static const char *dg_prop(const CBMDefinition *d, const char *key) { + for (const char **kv = d->extra_props; kv && kv[0]; kv += 2) { + if (strcmp(kv[0], key) == 0) { + return kv[1]; + } + } + return ""; +} + +static int dg_arrows(const CBMFileResult *r, const char *from_suffix, const char *to_suffix, + const char *style, const char *label) { + int n = 0; + size_t fl = strlen(from_suffix); + size_t tl = strlen(to_suffix); + for (int i = 0; i < r->diagram_arrows.count; i++) { + const CBMDiagramArrow *a = &r->diagram_arrows.items[i]; + size_t al = strlen(a->from_qn); + size_t bl = strlen(a->to_qn); + if (al >= fl && strcmp(a->from_qn + al - fl, from_suffix) == 0 && bl >= tl && + strcmp(a->to_qn + bl - tl, to_suffix) == 0 && strcmp(a->style, style) == 0 && + strcmp(a->label ? a->label : "", label) == 0) { + n++; + } + } + if (n != 1) { + printf(" arrow %s -> %s [%s \"%s\"]: %d found\n", from_suffix, to_suffix, style, label, n); + } + return n; +} + +TEST(diagrams_markdown_mermaid) { + CBMFileResult *r = dm_extract(DG_DOC, CBM_LANG_MARKDOWN, "docs/arch.md"); + ASSERT_NOT_NULL(r); + /* a flowchart under its section, titled by its front matter */ + const CBMDefinition *d = dg_def(r, ".Arch#diagram-1"); + ASSERT_NOT_NULL(d); + ASSERT_STR_EQ(d->label, "Diagram"); + ASSERT_STR_EQ(d->name, "System overview"); + ASSERT_STR_EQ(dg_prop(d, "format"), "mermaid"); + ASSERT_STR_EQ(dg_prop(d, "diagram_kind"), "flowchart"); + ASSERT_NOT_NULL(d->parent_class); + ASSERT(strstr(d->parent_class, ".Arch") != NULL); + ASSERT(d->docstring && strstr(d->docstring, "Web app") && strstr(d->docstring, "calls")); + ASSERT_EQ(d->start_line, 7); + const CBMDefinition *a = dg_def(r, ".Arch#diagram-1.A"); + ASSERT_NOT_NULL(a); + ASSERT_STR_EQ(a->label, "DiagramElement"); + ASSERT_STR_EQ(a->name, "Web app"); + ASSERT_STR_EQ(a->parent_class, d->qualified_name); + ASSERT_STR_EQ(dg_prop(a, "element_id"), "A"); + ASSERT_STR_EQ(dg_def(r, ".Arch#diagram-1.C")->name, "DB"); + ASSERT_STR_EQ(dg_def(r, ".Arch#diagram-1.D")->name, "Cache"); + ASSERT_STR_EQ(dg_def(r, ".Arch#diagram-1.E")->name, "E"); /* named only by an arrow */ + ASSERT_STR_EQ(dg_def(r, ".Arch#diagram-1.S")->name, "Backend"); + ASSERT_EQ(dg_arrows(r, "#diagram-1.A", "#diagram-1.B", "arrow", "calls"), 1); + ASSERT_EQ(dg_arrows(r, "#diagram-1.B", "#diagram-1.C", "arrow", ""), 1); + ASSERT_EQ(dg_arrows(r, "#diagram-1.B", "#diagram-1.D", "arrow", ""), 1); + ASSERT_EQ(dg_arrows(r, "#diagram-1.B", "#diagram-1.E", "dotted", "retries"), 1); + /* a second diagram of the section: a sequence */ + ASSERT_STR_EQ(dg_prop(dg_def(r, ".Arch#diagram-2"), "diagram_kind"), "sequence"); + ASSERT_STR_EQ(dg_def(r, ".Arch#diagram-2.U")->name, "User"); + ASSERT_EQ(dg_arrows(r, "#diagram-2.U", "#diagram-2.API", "message", "login"), 1); + ASSERT_EQ(dg_arrows(r, "#diagram-2.API", "#diagram-2.U", "reply", "token"), 1); + /* a class diagram under the next section: inheritance points at the parent */ + ASSERT_STR_EQ(dg_def(r, ".Model#diagram-1.Shop")->name, "Pet shop"); + ASSERT_EQ(dg_arrows(r, "#diagram-1.Dog", "#diagram-1.Animal", "inheritance", "is a"), 1); + ASSERT_EQ(dg_arrows(r, "#diagram-1.Shop", "#diagram-1.Dog", "composition", ""), 1); + cbm_free_result(r); + PASS(); +} + +TEST(diagrams_mermaid_kinds) { + CBMFileResult *r = dm_extract("erDiagram\n" + " CUSTOMER ||--o{ ORDER : places\n" + " ORDER ||..|{ ITEM : contains\n" + " CUSTOMER {\n" + " string name\n" + " }\n", + CBM_LANG_MERMAID, "diagrams/er.mmd"); + ASSERT_NOT_NULL(r); + ASSERT_STR_EQ(dg_prop(dg_def(r, "er#diagram-1"), "diagram_kind"), "er"); + ASSERT_EQ(dg_arrows(r, ".CUSTOMER", ".ORDER", "identifying", "places"), 1); + ASSERT_EQ(dg_arrows(r, ".ORDER", ".ITEM", "non_identifying", "contains"), 1); + cbm_free_result(r); + + r = dm_extract("stateDiagram-v2\n" + " [*] --> Idle\n" + " Idle --> Busy : start\n" + " state \"Working hard\" as Busy\n" + " Busy --> [*]\n" + " state Busy {\n" + " [*] --> Step\n" + " }\n", + CBM_LANG_MERMAID, "diagrams/state.mmd"); + ASSERT_NOT_NULL(r); + ASSERT_EQ(dg_arrows(r, ".start", ".Idle", "transition", ""), 1); + ASSERT_EQ(dg_arrows(r, ".Idle", ".Busy", "transition", "start"), 1); + ASSERT_EQ(dg_arrows(r, ".Busy", ".end", "transition", ""), 1); + /* a composite state's own start; states named by arrows are states */ + ASSERT_EQ(dg_arrows(r, ".Busy_start", ".Step", "transition", ""), 1); + ASSERT_STR_EQ(dg_prop(dg_def(r, "state#diagram-1.Idle"), "element_kind"), "state"); + ASSERT_STR_EQ(dg_def(r, "state#diagram-1.Busy")->name, "Working hard"); + cbm_free_result(r); + + r = dm_extract("C4Context\n" + " title Bank\n" + " Person(cust, \"Customer\", \"A customer\")\n" + " System_Ext(mail, \"Mail system\")\n" + " Rel(cust, mail, \"Reads\", \"SMTP\")\n", + CBM_LANG_MERMAID, "diagrams/c4.mmd"); + ASSERT_NOT_NULL(r); + ASSERT_STR_EQ(dg_prop(dg_def(r, "c4#diagram-1"), "diagram_kind"), "c4_context"); + ASSERT_STR_EQ(dg_def(r, "c4#diagram-1")->name, "Bank"); /* its title line */ + const CBMDefinition *cust = dg_def(r, "c4#diagram-1.cust"); + ASSERT_NOT_NULL(cust); + ASSERT_STR_EQ(cust->name, "Customer"); + ASSERT_STR_EQ(dg_prop(cust, "element_kind"), "person"); + ASSERT_STR_EQ(dg_prop(dg_def(r, "c4#diagram-1.mail"), "element_kind"), "system_ext"); + ASSERT_EQ(dg_arrows(r, ".cust", ".mail", "relation", "Reads"), 1); + cbm_free_result(r); + + r = dm_extract("architecture-beta\n" + " group api(cloud)[API]\n" + " service db(database)[Database] in api\n" + " service server(server)[Server] in api\n" + " db:L -- R:server\n" + " server:T --> B:db\n", + CBM_LANG_MERMAID, "diagrams/arch.mmd"); + ASSERT_NOT_NULL(r); + ASSERT_STR_EQ(dg_def(r, "arch#diagram-1.db")->name, "Database"); + ASSERT_STR_EQ(dg_prop(dg_def(r, "arch#diagram-1.api"), "element_kind"), "group"); + ASSERT_EQ(dg_arrows(r, ".db", ".server", "line", ""), 1); + ASSERT_EQ(dg_arrows(r, ".server", ".db", "arrow", ""), 1); + cbm_free_result(r); + + /* a kind without elements is a Diagram all the same */ + r = dm_extract("pie title Pets\n \"Dogs\" : 386\n", CBM_LANG_MERMAID, "diagrams/pie.mmd"); + ASSERT_NOT_NULL(r); + ASSERT_STR_EQ(dg_prop(dg_def(r, "pie#diagram-1"), "diagram_kind"), "pie"); + ASSERT_STR_EQ(dg_def(r, "pie#diagram-1")->name, "Pets"); + ASSERT_EQ(r->diagram_arrows.count, 0); + cbm_free_result(r); + PASS(); +} + +/* A fenced block of another language is no diagram. */ +TEST(diagrams_other_fences) { + CBMFileResult *r = dm_extract("# T\n\n```js\nA --> B\n```\n", CBM_LANG_MARKDOWN, "a.md"); + ASSERT_NOT_NULL(r); + for (int i = 0; i < r->defs.count; i++) { + ASSERT(strcmp(r->defs.items[i].label, "Diagram") != 0); + } + ASSERT_EQ(r->diagram_arrows.count, 0); + cbm_free_result(r); + PASS(); +} + +/* ── the other formats ───────────────────────────────────────────── */ + +static CBMFileResult *dg_file(const char *src, const char *rel) { + return dm_extract(src, CBM_LANG_DIAGRAM, rel); +} + +TEST(diagrams_plantuml) { + CBMFileResult *r = dg_file("@startuml Components\n" + "[Web] --> [API] : REST\n" + "component \"Billing\" as B\n" + "API ..> B\n" + "@enduml\n" + "@startuml\n" + "class Animal\n" + "class Dog {\n" + " +bark()\n" + "}\n" + "Animal <|-- Dog\n" + "Dog \"1\" *-- \"many\" Leg : has\n" + "Dog +-- Tail\n" + "@enduml\n", + "docs/model.puml"); + ASSERT_NOT_NULL(r); + const CBMDefinition *d = dg_def(r, "model#diagram-1"); + ASSERT_NOT_NULL(d); + ASSERT_STR_EQ(d->name, "Components"); + ASSERT_STR_EQ(dg_prop(d, "format"), "plantuml"); + ASSERT_STR_EQ(dg_prop(d, "diagram_kind"), "component"); + ASSERT_STR_EQ(dg_def(r, "model#diagram-1.B")->name, "Billing"); + ASSERT_EQ(dg_arrows(r, "#diagram-1.Web", "#diagram-1.API", "arrow", "REST"), 1); + ASSERT_EQ(dg_arrows(r, "#diagram-1.API", "#diagram-1.B", "dependency", ""), 1); + ASSERT_STR_EQ(dg_prop(dg_def(r, "model#diagram-2"), "diagram_kind"), "class"); + /* classes in packages: a class diagram, whatever line comes first */ + CBMFileResult *pk = dg_file("@startuml\npackage app {\n class A\n interface B\n}\n" + "A ..|> B\n@enduml\n", + "docs/pk.puml"); + ASSERT_STR_EQ(dg_prop(dg_def(pk, "pk#diagram-1"), "diagram_kind"), "class"); + ASSERT_STR_EQ(dg_def(pk, "pk#diagram-1")->name, "pk"); /* untitled: the file */ + cbm_free_result(pk); + ASSERT_EQ(dg_arrows(r, "#diagram-2.Dog", "#diagram-2.Animal", "inheritance", ""), 1); + ASSERT_EQ(dg_arrows(r, "#diagram-2.Dog", "#diagram-2.Leg", "composition", "has"), 1); + ASSERT_EQ(dg_arrows(r, "#diagram-2.Tail", "#diagram-2.Dog", "nesting", ""), 1); + cbm_free_result(r); + + /* a sequence in a Markdown block */ + r = dm_extract("# Flow\n\n```plantuml\n@startuml\nparticipant Alice\nactor Bob as B\n" + "Alice -> B : hello\nB --> Alice : hi\n@enduml\n```\n", + CBM_LANG_MARKDOWN, "docs/flow.md"); + ASSERT_NOT_NULL(r); + ASSERT_EQ(dg_arrows(r, "#diagram-1.Alice", "#diagram-1.B", "message", "hello"), 1); + ASSERT_EQ(dg_arrows(r, "#diagram-1.B", "#diagram-1.Alice", "reply", "hi"), 1); + ASSERT_STR_EQ(dg_def(r, "Flow#diagram-1.B")->name, "Bob"); + cbm_free_result(r); + PASS(); +} + +TEST(diagrams_dot) { + CBMFileResult *r = + dg_file("digraph G {\n" + " label=\"Pipeline\";\n" + " // a comment -> nowhere\n" + " a [label=\"Parse\"];\n" + " b [label=\"Resolve\"];\n" + " a -> b -> c [label=\"next\"];\n" + " subgraph cluster_x { d; e; }\n" + " c -> {d e};\n" + " \"Read every verification output in full before claiming that the step passed, " + "then record the result in the ledger with its timestamp\" -> b;\n" + " y [label=\"two\\nlines\"];\n" + "}\n", + "docs/pipe.dot"); + ASSERT_NOT_NULL(r); + const CBMDefinition *d = dg_def(r, "pipe#diagram-1"); + ASSERT_NOT_NULL(d); + ASSERT_STR_EQ(d->name, "Pipeline"); + ASSERT_STR_EQ(dg_prop(d, "diagram_kind"), "digraph"); + ASSERT_STR_EQ(dg_def(r, "pipe#diagram-1.a")->name, "Parse"); + ASSERT_EQ(dg_arrows(r, "#diagram-1.a", "#diagram-1.b", "arrow", "next"), 1); + ASSERT_EQ(dg_arrows(r, "#diagram-1.b", "#diagram-1.c", "arrow", "next"), 1); + ASSERT_EQ(dg_arrows(r, "#diagram-1.c", "#diagram-1.d", "arrow", ""), 1); + ASSERT_EQ(dg_arrows(r, "#diagram-1.c", "#diagram-1.e", "arrow", ""), 1); + ASSERT_EQ(r->diagram_arrows.count, 5); + /* a node named by a sentence too long for an id: kept, under a hashed key */ + int longs = 0; + for (int i = 0; i < r->diagram_arrows.count; i++) { + const CBMDiagramArrow *a = &r->diagram_arrows.items[i]; + longs += strstr(a->from_qn, "#diagram-1.Read_every_verification") && + strchr(a->from_qn, '~') && strstr(a->to_qn, "#diagram-1.b"); + } + ASSERT_EQ(longs, 1); + ASSERT_STR_EQ(dg_def(r, "pipe#diagram-1.y")->name, "two lines"); + cbm_free_result(r); + PASS(); +} + +TEST(diagrams_d2) { + CBMFileResult *r = dg_file("server: Web Server\n" + "db: Database {\n" + " shape: cylinder\n" + "}\n" + "server -> db: queries\n" + "cloud: {\n" + " api -> worker: jobs\n" + "}\n", + "docs/sys.d2"); + ASSERT_NOT_NULL(r); + ASSERT_STR_EQ(dg_def(r, "sys#diagram-1.server")->name, "Web Server"); + ASSERT_STR_EQ(dg_def(r, "sys#diagram-1.db")->name, "Database"); + ASSERT_EQ(dg_arrows(r, "#diagram-1.server", "#diagram-1.db", "arrow", "queries"), 1); + ASSERT_EQ(dg_arrows(r, "#diagram-1.cloud_api", "#diagram-1.cloud_worker", "arrow", "jobs"), 1); + ASSERT_EQ(r->diagram_arrows.count, 2); + cbm_free_result(r); + PASS(); +} + +static const char *const DG_DRAWIO_PAGE = + "" + "" + "" + "" + "" + "" + ""; + +static char *dg_b64(const unsigned char *in, size_t dn) { + static const char b64[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + char *out = malloc(dn * 4 / 3 + 8); + size_t o = 0; + for (size_t i = 0; i < dn; i += 3) { + uint32_t v = (uint32_t)in[i] << 16; + v |= i + 1 < dn ? (uint32_t)in[i + 1] << 8 : 0; + v |= i + 2 < dn ? in[i + 2] : 0; + out[o++] = b64[(v >> 18) & 63]; + out[o++] = b64[(v >> 12) & 63]; + out[o++] = i + 1 < dn ? b64[(v >> 6) & 63] : '='; + out[o++] = i + 2 < dn ? b64[v & 63] : '='; + } + out[o] = '\0'; + return out; +} + +/* A draw.io page as the app stores it compressed: URI-encoded, raw deflate, + * base64. */ +static char *dg_drawio_compress(const char *xml) { + size_t n = strlen(xml); + char *enc = malloc(n * 3 + 1); + size_t w = 0; + for (size_t i = 0; i < n; i++) { + unsigned char c = (unsigned char)xml[i]; + if (isalnum(c) || strchr("-_.!~*'()", c)) { + enc[w++] = (char)c; + } else { + w += (size_t)snprintf(enc + w, n * 3 + 1 - w, "%%%02X", c); + } + } + unsigned char *def = malloc(w + 64); + z_stream z; + memset(&z, 0, sizeof(z)); + deflateInit2(&z, Z_DEFAULT_COMPRESSION, Z_DEFLATED, -MAX_WBITS, 8, Z_DEFAULT_STRATEGY); + z.next_in = (Bytef *)enc; + z.avail_in = (uInt)w; + z.next_out = def; + z.avail_out = (uInt)(w + 64); + deflate(&z, Z_FINISH); + size_t dn = z.total_out; + deflateEnd(&z); + char *out = dg_b64(def, dn); + free(enc); + free(def); + return out; +} + +TEST(diagrams_drawio) { + char file[4096]; + snprintf(file, sizeof(file), + "%s", DG_DRAWIO_PAGE); + CBMFileResult *r = dg_file(file, "docs/sys.drawio"); + ASSERT_NOT_NULL(r); + const CBMDefinition *d = dg_def(r, "sys#diagram-1"); + ASSERT_NOT_NULL(d); + ASSERT_STR_EQ(d->name, "Overview"); + ASSERT_STR_EQ(dg_def(r, "sys#diagram-1.2")->name, "Frontend"); + const CBMDefinition *b = dg_def(r, "sys#diagram-1.3"); + ASSERT_STR_EQ(b->name, "Backend"); /* its HTML dropped */ + ASSERT_STR_EQ(dg_prop(b, "element_kind"), "cylinder3"); + ASSERT_EQ(dg_arrows(r, "#diagram-1.2", "#diagram-1.3", "arrow", "calls"), 1); + ASSERT_EQ(r->diagram_arrows.count, 1); /* the edge to a shape without text is none */ + cbm_free_result(r); + + /* the compressed form, two pages */ + char *packed = dg_drawio_compress(DG_DRAWIO_PAGE); + size_t big_cap = strlen(packed) * 2 + 256; + char *big = malloc(big_cap); + snprintf(big, big_cap, + "%s%s", + packed, packed); + r = dg_file(big, "docs/two.drawio"); + ASSERT_NOT_NULL(r); + ASSERT_STR_EQ(dg_def(r, "two#diagram-1")->name, "A"); + ASSERT_STR_EQ(dg_def(r, "two#diagram-2")->name, "B"); + ASSERT_EQ(dg_arrows(r, "#diagram-2.2", "#diagram-2.3", "arrow", "calls"), 1); + cbm_free_result(r); + free(big); + free(packed); + + /* draw.io's editable SVG keeps the file in its content attribute */ + r = dg_file("", + "docs/edit.drawio.svg"); + ASSERT_NOT_NULL(r); + ASSERT_STR_EQ(dg_def(r, "#diagram-1")->name, "S"); + ASSERT_EQ(dg_arrows(r, "#diagram-1.2", "#diagram-1.3", "arrow", ""), 1); + cbm_free_result(r); + PASS(); +} + +TEST(diagrams_excalidraw) { + CBMFileResult *r = + dg_file("{\"type\":\"excalidraw\",\"elements\":[" + "{\"id\":\"r1\",\"type\":\"rectangle\",\"isDeleted\":false}," + "{\"id\":\"t1\",\"type\":\"text\",\"text\":\"User\",\"containerId\":\"r1\"}," + "{\"id\":\"r2\",\"type\":\"ellipse\"}," + "{\"id\":\"t2\",\"type\":\"text\",\"text\":\"Service\",\"containerId\":\"r2\"}," + "{\"id\":\"a1\",\"type\":\"arrow\",\"startBinding\":{\"elementId\":\"r1\"}," + "\"endBinding\":{\"elementId\":\"r2\"}}," + "{\"id\":\"t3\",\"type\":\"text\",\"text\":\"calls\",\"containerId\":\"a1\"}," + "{\"id\":\"r3\",\"type\":\"rectangle\",\"isDeleted\":true}," + "{\"id\":\"t5\",\"type\":\"text\",\"text\":\"Gone\",\"containerId\":\"r3\"," + "\"isDeleted\":true}," + "{\"id\":\"t4\",\"type\":\"text\",\"text\":\"Note here\"}]}", + "docs/flow.excalidraw"); + ASSERT_NOT_NULL(r); + const CBMDefinition *d = dg_def(r, "flow#diagram-1"); + ASSERT_NOT_NULL(d); + ASSERT(d->docstring && strstr(d->docstring, "Note here")); + ASSERT_STR_EQ(dg_def(r, "flow#diagram-1.r1")->name, "User"); + ASSERT_STR_EQ(dg_prop(dg_def(r, "flow#diagram-1.r2"), "element_kind"), "ellipse"); + ASSERT_EQ(dg_arrows(r, "#diagram-1.r1", "#diagram-1.r2", "arrow", "calls"), 1); + for (int i = 0; i < r->defs.count; i++) { + ASSERT(!strstr(r->defs.items[i].qualified_name, ".r3")); + } + cbm_free_result(r); + PASS(); +} + +TEST(diagrams_d2_blocks) { + /* class and style blocks around containers (their braces once restored + * the path from an unset level), block-string labels, dotted attributes */ + CBMFileResult *r = dg_file("direction: down\n" + "classes: {\n" + " card: {\n" + " shape: text\n" + " }\n" + "}\n" + "label: Hierarchy\n" + "consumers: \"\" {\n" + " style: {\n" + " stroke-dash: 4\n" + " }\n" + " app: ||md\n" + " App\n" + " Cross-platform: everywhere\n" + " || {\n" + " class: card\n" + " }\n" + " cli: |md # CLI|\n" + "}\n" + "core.style.fill: red\n" + "core.label: Core\n" + "consumers.app -> core: |md uses|\n" + "consumers.cli -- core\n", + "docs/deps.d2"); + ASSERT_NOT_NULL(r); + ASSERT_STR_EQ(dg_def(r, "deps#diagram-1")->name, "Hierarchy"); + ASSERT_STR_EQ(dg_def(r, "deps#diagram-1.consumers_app")->name, "App"); + ASSERT_STR_EQ(dg_def(r, "deps#diagram-1.consumers_cli")->name, "CLI"); + ASSERT_STR_EQ(dg_def(r, "deps#diagram-1.core")->name, "Core"); + const char *doc = dg_def(r, "deps#diagram-1")->docstring; + ASSERT(doc && strstr(doc, "Cross-platform: everywhere")); + ASSERT_EQ(dg_arrows(r, "#diagram-1.consumers_app", "#diagram-1.core", "arrow", "uses"), 1); + ASSERT_EQ(dg_arrows(r, "#diagram-1.consumers_cli", "#diagram-1.core", "line", ""), 1); + ASSERT_EQ(r->diagram_arrows.count, 2); + for (int i = 0; i < r->defs.count; i++) { /* no attribute or block text became a shape */ + const char *qn = r->defs.items[i].qualified_name; + ASSERT(!strstr(qn, "style") && !strstr(qn, "card") && !strstr(qn, "class") && + !strstr(qn, "Cross")); + } + cbm_free_result(r); + PASS(); +} + +TEST(diagrams_excalidraw_layout) { + /* text laid over shapes and arrows drawn up to them, nothing bound */ + CBMFileResult *r = dg_file( + "{\"type\":\"excalidraw\",\"elements\":[" + "{\"id\":\"f1\",\"type\":\"rectangle\",\"x\":0,\"y\":0,\"width\":900,\"height\":400}," + "{\"id\":\"t0\",\"type\":\"text\",\"text\":\"System\",\"x\":10,\"y\":5,\"width\":60," + "\"height\":20}," + "{\"id\":\"r1\",\"type\":\"rectangle\",\"x\":20,\"y\":50,\"width\":100,\"height\":60}," + "{\"id\":\"t2\",\"type\":\"text\",\"text\":\"browser\",\"x\":30,\"y\":95,\"width\":40," + "\"height\":10}," + "{\"id\":\"t1\",\"type\":\"text\",\"text\":\"Client\",\"x\":40,\"y\":70,\"width\":60," + "\"height\":20}," + "{\"id\":\"r2\",\"type\":\"ellipse\",\"x\":250,\"y\":50,\"width\":100,\"height\":60}," + "{\"id\":\"t3\",\"type\":\"text\",\"text\":\"Server\",\"x\":270,\"y\":70,\"width\":60," + "\"height\":20}," + "{\"id\":\"a1\",\"type\":\"arrow\",\"x\":122,\"y\":80,\"points\":[[0,0],[126,0]]}," + "{\"id\":\"l1\",\"type\":\"line\",\"x\":122,\"y\":90,\"points\":[[0,0],[126,0]]}," + "{\"id\":\"a2\",\"type\":\"arrow\",\"x\":30,\"y\":60,\"points\":[[0,0],[70,40]]}," + "{\"id\":\"a3\",\"type\":\"arrow\",\"x\":122,\"y\":200,\"points\":[[0,0],[500,400]]}," + /* from r2 deep into the frame around it all: at nothing named */ + "{\"id\":\"a4\",\"type\":\"arrow\",\"x\":300,\"y\":112,\"points\":[[0,0],[0,90]]}," + /* from r1 to the frame's border: at the frame */ + "{\"id\":\"a5\",\"type\":\"arrow\",\"x\":70,\"y\":112,\"points\":[[0,0],[0,284]]}," + /* two lanes, each with a named badge: an end deep in one is at the lane */ + "{\"id\":\"L1\",\"type\":\"rectangle\",\"x\":500,\"y\":0,\"width\":100,\"height\":300}," + "{\"id\":\"tl1\",\"type\":\"text\",\"text\":\"Lane A\",\"x\":520,\"y\":10,\"width\":60," + "\"height\":20}," + "{\"id\":\"b1\",\"type\":\"ellipse\",\"x\":540,\"y\":200,\"width\":20,\"height\":20}," + "{\"id\":\"tb1\",\"type\":\"text\",\"text\":\"1\",\"containerId\":\"b1\"}," + "{\"id\":\"L2\",\"type\":\"rectangle\",\"x\":700,\"y\":0,\"width\":100,\"height\":300}," + "{\"id\":\"tl2\",\"type\":\"text\",\"text\":\"Lane B\",\"x\":720,\"y\":10,\"width\":60," + "\"height\":20}," + "{\"id\":\"b2\",\"type\":\"rectangle\",\"x\":745,\"y\":150,\"width\":10,\"height\":40}," + "{\"id\":\"a6\",\"type\":\"arrow\",\"x\":555,\"y\":120,\"points\":[[0,0],[195,0]]}," + /* bound to the text that names r1: at r1 */ + "{\"id\":\"a7\",\"type\":\"arrow\",\"startBinding\":{\"elementId\":\"r2\"}," + "\"endBinding\":{\"elementId\":\"t1\"}}," + /* bound to free text that names no shape: at the text */ + "{\"id\":\"t9\",\"type\":\"text\",\"text\":\"Note\",\"x\":440,\"y\":350,\"width\":40," + "\"height\":20}," + "{\"id\":\"a8\",\"type\":\"arrow\",\"startBinding\":{\"elementId\":\"L2\"}," + "\"endBinding\":{\"elementId\":\"t9\"}}]}", + "docs/layout.excalidraw"); + ASSERT_NOT_NULL(r); + ASSERT_STR_EQ(dg_def(r, "layout#diagram-1.f1")->name, "System"); + ASSERT_STR_EQ(dg_def(r, "layout#diagram-1.r1")->name, + "Client"); /* the top text, not the frame */ + ASSERT_STR_EQ(dg_def(r, "layout#diagram-1.r2")->name, "Server"); + const char *doc = dg_def(r, "layout#diagram-1")->docstring; + ASSERT(doc && strstr(doc, "browser")); + ASSERT_EQ(dg_arrows(r, "#diagram-1.r1", "#diagram-1.r2", "arrow", ""), 1); + ASSERT_EQ(dg_arrows(r, "#diagram-1.r1", "#diagram-1.f1", "arrow", ""), 1); + /* the loose line, the loop inside r1, the dangling end and the end deep in + * the frame are no arrows */ + ASSERT_EQ(dg_arrows(r, "#diagram-1.L1", "#diagram-1.L2", "arrow", ""), 1); + ASSERT_EQ(dg_arrows(r, "#diagram-1.r2", "#diagram-1.r1", "arrow", ""), 1); + ASSERT_EQ(dg_arrows(r, "#diagram-1.L2", "#diagram-1.t9", "arrow", ""), 1); + ASSERT_STR_EQ(dg_prop(dg_def(r, "layout#diagram-1.t9"), "element_kind"), "text"); + ASSERT_EQ(r->diagram_arrows.count, 5); + ASSERT_STR_EQ(dg_def(r, "layout#diagram-1")->name, "layout"); /* untitled: the file */ + cbm_free_result(r); + PASS(); +} + +/* An Excalidraw SVG export as the app writes it: the scene zlib-compressed, + * wrapped in JSON as a byte string, base64 in a payload comment. */ +static char *dg_excalidraw_svg(const char *scene) { + uLongf zn = compressBound((uLong)strlen(scene)); + unsigned char *z = malloc(zn); + compress2(z, &zn, (const Bytef *)scene, (uLong)strlen(scene), Z_DEFAULT_COMPRESSION); + size_t cap = zn * 6 + 128; + char *json = malloc(cap); + size_t w = (size_t)snprintf(json, cap, + "{\"version\":\"1\",\"encoding\":\"bstring\",\"compressed\":true," + "\"encoded\":\""); + for (uLongf i = 0; i < zn; i++) { + unsigned char b = z[i]; + if (b == '"' || b == '\\') { + json[w++] = '\\'; + json[w++] = (char)b; + } else if (b < 0x20) { + w += (size_t)snprintf(json + w, cap - w, "\\u%04x", b); + } else { + json[w++] = (char)b; /* a byte as one character: the wrapper is no UTF-8 */ + } + } + w += (size_t)snprintf(json + w, cap - w, "\"}"); + char *b64 = dg_b64((const unsigned char *)json, w); + size_t sc = strlen(b64) + 512; + char *svg = malloc(sc); + snprintf(svg, sc, + "\n" + " \n" + " " + "%s\n" + " \n\n", + b64); + free(b64); + free(json); + free(z); + return svg; +} + +TEST(diagrams_excalidraw_svg) { + char *svg = dg_excalidraw_svg( + "{\"type\":\"excalidraw\",\"elements\":[" + "{\"id\":\"r1\",\"type\":\"rectangle\"}," + "{\"id\":\"t1\",\"type\":\"text\",\"text\":\"Queue \xc3\xa9v\xc3\xa9nements\"," + "\"containerId\":\"r1\"}," + "{\"id\":\"r2\",\"type\":\"rectangle\"}," + "{\"id\":\"t2\",\"type\":\"text\",\"text\":\"Worker\",\"containerId\":\"r2\"}," + "{\"id\":\"a1\",\"type\":\"arrow\",\"startBinding\":{\"elementId\":\"r1\"}," + "\"endBinding\":{\"elementId\":\"r2\"}}]}"); + CBMFileResult *r = dg_file(svg, "docs/flow.excalidraw.svg"); + ASSERT_NOT_NULL(r); + ASSERT_STR_EQ(dg_def(r, "#diagram-1.r1")->name, "Queue \xc3\xa9v\xc3\xa9nements"); + ASSERT_EQ(dg_arrows(r, "#diagram-1.r1", "#diagram-1.r2", "arrow", ""), 1); + cbm_free_result(r); + free(svg); + PASS(); +} + +TEST(diagrams_rst) { + CBMFileResult *r = dm_extract("Guide\n" + "=====\n" + "\n" + ".. mermaid::\n" + " :caption: Request flow\n" + "\n" + " flowchart LR\n" + " A --> B\n" + "\n" + ".. digraph:: deps\n" + "\n" + " core -> util;\n" + " app -> core;\n" + "\n" + ".. graphviz:: other.dot\n" + "\n" + ".. uml::\n" + "\n" + " @startuml\n" + " [Web] --> [API]\n" + " @enduml\n", + CBM_LANG_RST, "docs/guide.rst"); + ASSERT_NOT_NULL(r); + const CBMDefinition *m = dg_def(r, "guide#diagram-1"); + ASSERT_NOT_NULL(m); + ASSERT_STR_EQ(m->name, "Request flow"); + ASSERT_STR_EQ(dg_prop(m, "format"), "mermaid"); + ASSERT_EQ(dg_arrows(r, "guide#diagram-1.A", "guide#diagram-1.B", "arrow", ""), 1); + /* Sphinx's digraph: the statements inside the graph it names */ + ASSERT_STR_EQ(dg_prop(dg_def(r, "guide#diagram-2"), "format"), "dot"); + ASSERT_EQ(dg_arrows(r, "guide#diagram-2.core", "guide#diagram-2.util", "arrow", ""), 1); + ASSERT_EQ(dg_arrows(r, "guide#diagram-2.app", "guide#diagram-2.core", "arrow", ""), 1); + /* a graphviz file argument is no diagram here (the file is one itself) */ + ASSERT_STR_EQ(dg_prop(dg_def(r, "guide#diagram-3"), "format"), "plantuml"); + ASSERT_EQ(dg_arrows(r, "guide#diagram-3.Web", "guide#diagram-3.API", "arrow", ""), 1); + ASSERT_NULL(dg_def(r, "guide#diagram-4")); + cbm_free_result(r); + PASS(); +} + +/* The Diagram named `name`, NULL for none. */ +static const CBMDefinition *dg_named(const CBMFileResult *r, const char *name) { + for (int i = 0; i < r->defs.count; i++) { + const CBMDefinition *d = &r->defs.items[i]; + if (d->label && strcmp(d->label, "Diagram") == 0 && d->name && strcmp(d->name, name) == 0) { + return d; + } + } + return NULL; +} + +TEST(diagrams_asciidoc) { + CBMFileResult *r = dm_extract("= Guide\n" + "\n" + "== Flow\n" + "\n" + ".Login sequence\n" + "[plantuml, target=login, format=svg]\n" + "----\n" + "participant Alice\n" + "Alice -> Bob : hello\n" + "----\n" + "\n" + ".Steps\n" + "[source,mermaid]\n" + "....\n" + "flowchart TD\n" + " X --> Y\n" + "....\n" + "\n" + "[source,python]\n" + "----\n" + "a -> b\n" + "----\n" + "\n" + "[graphviz]\n" + "----\n" + "digraph { a -> b }\n" + "----\n", + CBM_LANG_ASCIIDOC, "docs/guide.adoc"); + ASSERT_NOT_NULL(r); + const CBMDefinition *p = dg_named(r, "Login sequence"); + ASSERT_NOT_NULL(p); + ASSERT_NOT_NULL(p->parent_class); /* under its Section */ + ASSERT(strncmp(p->qualified_name, p->parent_class, strlen(p->parent_class)) == 0); + ASSERT_STR_EQ(dg_prop(p, "diagram_kind"), "sequence"); + ASSERT_EQ(dg_arrows(r, "#diagram-1.Alice", "#diagram-1.Bob", "message", "hello"), 1); + ASSERT_STR_EQ(dg_prop(dg_named(r, "Steps"), "format"), "mermaid"); + ASSERT_EQ(dg_arrows(r, "#diagram-2.X", "#diagram-2.Y", "arrow", ""), 1); + ASSERT_EQ(dg_arrows(r, "#diagram-3.a", "#diagram-3.b", "arrow", ""), 1); /* not python */ + ASSERT_EQ(r->diagram_arrows.count, 3); + cbm_free_result(r); + PASS(); +} + +/* The doc-link records of one syntax whose source ends in `src` and whose + * reference is `raw`. */ +static int dg_links(const CBMFileResult *r, int syntax, const char *src, const char *raw) { + int n = 0; + size_t sl = strlen(src); + for (int i = 0; i < r->doc_links.count; i++) { + const CBMDocLink *l = &r->doc_links.items[i]; + size_t ql = strlen(l->source_qn); + n += l->syntax == syntax && ql >= sl && strcmp(l->source_qn + ql - sl, src) == 0 && + strcmp(l->raw, raw) == 0; + } + if (n != 1) { + printf(" link %d %s -> %s: %d found\n", syntax, src, raw, n); + } + return n; +} + +TEST(diagrams_links) { + CBMFileResult *r = + dm_extract("# Flow\n\n```mermaid\nflowchart LR\n A[Api] --> B\n" + " click A \"src/api.py\" \"the API\"\n click B call cb()\n```\n", + CBM_LANG_MARKDOWN, "docs/flow.md"); + ASSERT_NOT_NULL(r); + ASSERT_EQ(dg_links(r, CBM_DOCLINK_DG_MERMAID_CLICK, "#diagram-1.A", "src/api.py"), 1); + ASSERT_EQ(r->doc_links.count, 1); /* a callback links nothing */ + cbm_free_result(r); + + r = dg_file("@startuml\n!include common.iuml\n!include \n" + "class Foo [[src/foo.py]]\nclass Bar\nurl of Bar is [[docs/bar.md{the bar}]]\n" + "Foo --> Bar\n@enduml\n", + "docs/m.puml"); + ASSERT_NOT_NULL(r); + ASSERT_EQ(dg_links(r, CBM_DOCLINK_DG_PLANTUML_INCLUDE, "m#diagram-1", "common.iuml"), 1); + ASSERT_EQ(dg_links(r, CBM_DOCLINK_DG_PLANTUML_URL, "#diagram-1.Foo", "src/foo.py"), 1); + ASSERT_EQ(dg_links(r, CBM_DOCLINK_DG_PLANTUML_URL, "#diagram-1.Bar", "docs/bar.md"), 1); + ASSERT_EQ(r->doc_links.count, 3); /* the standard library is none of the repo's */ + cbm_free_result(r); + + r = dg_file("digraph { a [label=\"A\", URL=\"src/a.c\"]; b [href=\"#top\"]; a -> b; }\n", + "docs/g.dot"); + ASSERT_NOT_NULL(r); + ASSERT_EQ(dg_links(r, CBM_DOCLINK_DG_DOT_URL, "#diagram-1.a", "src/a.c"), 1); + ASSERT_EQ(r->doc_links.count, 1); /* an anchor links nothing */ + cbm_free_result(r); + + r = dg_file("x: X\nx.link: src/x.go\ny: Y {\n link: https://example.com/y\n}\n", "docs/d.d2"); + ASSERT_NOT_NULL(r); + ASSERT_EQ(dg_links(r, CBM_DOCLINK_DG_D2_LINK, "#diagram-1.x", "src/x.go"), 1); + ASSERT_EQ(dg_links(r, CBM_DOCLINK_DG_D2_LINK, "#diagram-1.y", "https://example.com/y"), 1); + cbm_free_result(r); + + r = dg_file("" + "" + "" + "", + "docs/s.drawio"); + ASSERT_NOT_NULL(r); + ASSERT_EQ(dg_links(r, CBM_DOCLINK_DG_DRAWIO_LINK, "#diagram-1.7", "src/svc.py"), 1); + ASSERT_EQ(r->doc_links.count, 1); /* a page link links nothing of the repo */ + cbm_free_result(r); + + r = dg_file("{\"type\":\"excalidraw\",\"elements\":[" + "{\"id\":\"r1\",\"type\":\"rectangle\",\"link\":\"src/e.ts\"}," + "{\"id\":\"t1\",\"type\":\"text\",\"text\":\"E\",\"containerId\":\"r1\"}," + "{\"id\":\"r2\",\"type\":\"rectangle\",\"link\":null}]}", + "docs/e.excalidraw"); + ASSERT_NOT_NULL(r); + ASSERT_EQ(dg_links(r, CBM_DOCLINK_DG_EXCALIDRAW_LINK, "#diagram-1.r1", "src/e.ts"), 1); + ASSERT_EQ(r->doc_links.count, 1); + cbm_free_result(r); + PASS(); +} + +static int dg_db_edges(const char *db, const char *type, const char *src_suffix, + const char *tgt_suffix, const char *props_part); + +TEST(diagrams_links_graph) { + /* the families are held until their audit: shipped here */ + dm_ship_held_families(); + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dglinks_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[512]; + char db[512]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + snprintf(db, sizeof(db), "%s/g.db", tmp); + th_write_file(TH_PATH(repo, "src/api.py"), "def handle():\n return 1\n"); + th_write_file(TH_PATH(repo, "diagrams/a.puml"), + "@startuml\nclass Api [[../src/api.py]]\nclass Db\nApi --> Db\n@enduml\n"); + ASSERT_EQ(dm_index(repo, db, NULL), 0); + /* the element names the file it links to */ + ASSERT_EQ(dg_db_edges(db, "MENTIONS", "a#diagram-1.Api", "", NULL), 1); + th_cleanup(tmp); + cbm_doclink_test_reset_ships(); + PASS(); +} + +/* ── the graph ───────────────────────────────────────────────────── */ + +static int dg_db_edges(const char *db, const char *type, const char *src_suffix, + const char *tgt_suffix, const char *props_part) { + sqlite3 *h = NULL; + int n = 0; + if (sqlite3_open_v2(db, &h, SQLITE_OPEN_READONLY, NULL) != SQLITE_OK) { + sqlite3_close(h); + return -1; + } + sqlite3_stmt *st = NULL; + const char *sql = "SELECT e.properties FROM edges e JOIN nodes s ON s.id = e.source_id " + "JOIN nodes t ON t.id = e.target_id WHERE e.type = ?1 " + "AND s.qualified_name LIKE '%' || ?2 AND t.qualified_name LIKE '%' || ?3"; + if (sqlite3_prepare_v2(h, sql, -1, &st, NULL) == SQLITE_OK) { + sqlite3_bind_text(st, 1, type, -1, SQLITE_TRANSIENT); + sqlite3_bind_text(st, 2, src_suffix, -1, SQLITE_TRANSIENT); + sqlite3_bind_text(st, 3, tgt_suffix, -1, SQLITE_TRANSIENT); + while (sqlite3_step(st) == SQLITE_ROW) { + const char *p = (const char *)sqlite3_column_text(st, 0); + n += !props_part || (p && strstr(p, props_part)); + } + } + sqlite3_finalize(st); + sqlite3_close(h); + if (n != 1) { + printf(" %s %s -> %s (%s): %d found\n", type, src_suffix, tgt_suffix, + props_part ? props_part : "", n); + } + return n; +} + +TEST(diagrams_graph) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dg_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[512]; + char db[512]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + snprintf(db, sizeof(db), "%s/g.db", tmp); + th_write_file(TH_PATH(repo, "docs/arch.md"), DG_DOC); + th_write_file(TH_PATH(repo, "diagrams/flow.mmd"), "flowchart TD\n X[Start] --> Y[Stop]\n"); + th_write_file(TH_PATH(repo, "diagrams/c.puml"), "@startuml\n[A] --> [B]\n@enduml\n"); + /* an image by its last extension, a diagram by its name */ + th_write_file(TH_PATH(repo, "diagrams/e.drawio.svg"), + "\n"); + ASSERT_EQ(dm_index(repo, db, NULL), 0); + /* containment: the section holds the diagram, the diagram its elements */ + ASSERT_EQ(dg_db_edges(db, "DEFINES", "docs.arch.Arch", "docs.arch.Arch#diagram-1", NULL), 1); + ASSERT_EQ(dg_db_edges(db, "DEFINES", "Arch#diagram-1", "Arch#diagram-1.A", NULL), 1); + /* the arrows, with their text */ + ASSERT_EQ(dg_db_edges(db, "DIAGRAM_ARROW", "Arch#diagram-1.A", "Arch#diagram-1.B", + "\"label\":\"calls\""), + 1); + ASSERT_EQ(dg_db_edges(db, "DIAGRAM_ARROW", "Model#diagram-1.Dog", "Model#diagram-1.Animal", + "\"style\":\"inheritance\""), + 1); + /* a diagram file: its diagram under the file */ + ASSERT_EQ(dg_db_edges(db, "DIAGRAM_ARROW", "flow#diagram-1.X", "flow#diagram-1.Y", NULL), 1); + /* a diagram file of another format: discovered, its arrows in the graph */ + ASSERT_EQ(dg_db_edges(db, "DIAGRAM_ARROW", "c#diagram-1.A", "c#diagram-1.B", NULL), 1); + ASSERT_EQ(dg_db_edges(db, "DIAGRAM_ARROW", "#diagram-1.p", "#diagram-1.q", NULL), 1); + th_cleanup(tmp); + PASS(); +} + +/* ── incremental == full ─────────────────────────────────────────── */ + +/* The diagram layer of a graph as sorted lines: its nodes, their containment, + * the arrows, and the edges the elements' links resolved to. */ +static char *dg_state(const char *db) { + sqlite3 *h = NULL; + if (sqlite3_open_v2(db, &h, SQLITE_OPEN_READONLY, NULL) != SQLITE_OK) { + sqlite3_close(h); + return NULL; + } + const char *sql = + "SELECT 'N|' || label || '|' || qualified_name || '|' || name || '|' || " + "COALESCE(properties, '') AS l FROM nodes WHERE label IN ('Diagram', 'DiagramElement') " + "UNION ALL SELECT 'E|' || e.type || '|' || s.qualified_name || '|' || t.qualified_name || " + "'|' || COALESCE(e.properties, '') FROM edges e JOIN nodes s ON s.id = e.source_id " + "JOIN nodes t ON t.id = e.target_id WHERE e.type = 'DIAGRAM_ARROW' OR (e.type = " + "'DEFINES' AND t.label IN ('Diagram', 'DiagramElement')) OR (e.type = 'MENTIONS' AND " + "s.label IN ('Diagram', 'DiagramElement')) ORDER BY 1"; + sqlite3_stmt *st = NULL; + size_t cap = 4096; + size_t len = 0; + char *buf = malloc(cap); + buf[0] = '\0'; + if (sqlite3_prepare_v2(h, sql, -1, &st, NULL) == SQLITE_OK) { + while (sqlite3_step(st) == SQLITE_ROW) { + const char *line = (const char *)sqlite3_column_text(st, 0); + size_t l = strlen(line); + if (len + l + 2 > cap) { + cap = (len + l + 2) * 2; + buf = realloc(buf, cap); + } + memcpy(buf + len, line, l); + len += l; + buf[len++] = '\n'; + buf[len] = '\0'; + } + } + sqlite3_finalize(st); + sqlite3_close(h); + return buf; +} + +/* An incremental index of `repo` into inc_db, a full one into a fresh + * full_db: the same diagram layer. 0, or -1 after printing the difference. */ +static int dg_step(const char *repo, const char *inc_db, const char *full_db, const char *what) { + if (dm_index(repo, inc_db, NULL) != 0) { + printf(" %s: incremental index failed\n", what); + return -1; + } + dm_unlink_db(full_db); + if (dm_index(repo, full_db, NULL) != 0) { + printf(" %s: full index failed\n", what); + return -1; + } + char *inc = dg_state(inc_db); + char *full = dg_state(full_db); + int rc = inc && full && strcmp(inc, full) == 0 && full[0] ? 0 : -1; + if (rc != 0) { + printf(" %s: incremental != full\n--- incremental\n%s--- full\n%s", what, + inc ? inc : "(null)", full ? full : "(null)"); + } + free(inc); + free(full); + return rc; +} + +static void dg_write_all(const char *repo, int round) { + char text[1024]; + snprintf(text, sizeof(text), + "# Arch%s\n\n```mermaid\nflowchart LR\n A[Api] --> B[Db%d]\n" + " click A \"../src/api.py\"\n```\n", + round == 2 ? "itecture" : "", round); + th_write_file(TH_PATH(repo, "docs/arch.md"), text); + snprintf(text, sizeof(text), + "@startuml\nclass Api [[../src/api.py]]\nclass Db\nApi --> Db : r%d\n" + "@enduml\n", + round); + th_write_file(TH_PATH(repo, "diagrams/m.puml"), text); + snprintf(text, sizeof(text), "digraph { a -> b [label=\"r%d\"]; }\n", round); + th_write_file(TH_PATH(repo, "diagrams/g.dot"), text); + snprintf(text, sizeof(text), "x: X\ny: Y%d\nx -> y\n", round); + th_write_file(TH_PATH(repo, "diagrams/s.d2"), text); + snprintf(text, sizeof(text), + "" + "" + "", + round); + th_write_file(TH_PATH(repo, "diagrams/p.drawio"), text); + snprintf(text, sizeof(text), + "Guide\n=====\n\n.. mermaid::\n\n flowchart LR\n G --> H%d\n", round); + th_write_file(TH_PATH(repo, "docs/guide.rst"), text); + snprintf(text, sizeof(text), "= Guide\n\n== Flow\n\n[plantuml]\n----\n[P] --> [Q%d]\n----\n", + round); + th_write_file(TH_PATH(repo, "docs/flow.adoc"), text); + if (round < 2) { + snprintf(text, sizeof(text), + "{\"type\":\"excalidraw\",\"elements\":[" + "{\"id\":\"r1\",\"type\":\"rectangle\"},{\"id\":\"t1\",\"type\":\"text\"," + "\"text\":\"U%d\",\"containerId\":\"r1\"},{\"id\":\"r2\",\"type\":\"rectangle\"}," + "{\"id\":\"t2\",\"type\":\"text\",\"text\":\"V\",\"containerId\":\"r2\"}," + "{\"id\":\"a\",\"type\":\"arrow\",\"startBinding\":{\"elementId\":\"r1\"}," + "\"endBinding\":{\"elementId\":\"r2\"}}]}", + round); + th_write_file(TH_PATH(repo, "diagrams/e.excalidraw"), text); + } +} + +TEST(diagrams_incremental_equals_full) { + dm_ship_held_families(); + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dg_inc_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[512]; + char inc_db[512]; + char full_db[512]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + snprintf(inc_db, sizeof(inc_db), "%s/inc.db", tmp); + snprintf(full_db, sizeof(full_db), "%s/full.db", tmp); + th_write_file(TH_PATH(repo, "src/api.py"), "def handle():\n return 1\n"); + dg_write_all(repo, 0); + ASSERT_EQ(dm_index(repo, inc_db, NULL), 0); + /* every diagram's text edited */ + dg_write_all(repo, 1); + ASSERT_EQ(dg_step(repo, inc_db, full_db, "edit"), 0); + /* a heading renamed (the diagram's parent with it), a diagram file gone */ + dg_write_all(repo, 2); + char gone[600]; + snprintf(gone, sizeof(gone), "%s/diagrams/e.excalidraw", repo); + unlink(gone); + ASSERT_EQ(dg_step(repo, inc_db, full_db, "rename + delete"), 0); + /* and back */ + dg_write_all(repo, 1); + ASSERT_EQ(dg_step(repo, inc_db, full_db, "re-add"), 0); + th_cleanup(tmp); + cbm_doclink_test_reset_ships(); + PASS(); +} + +SUITE(diagrams) { + RUN_TEST(diagrams_markdown_mermaid); + RUN_TEST(diagrams_mermaid_kinds); + RUN_TEST(diagrams_other_fences); + RUN_TEST(diagrams_plantuml); + RUN_TEST(diagrams_dot); + RUN_TEST(diagrams_d2); + RUN_TEST(diagrams_drawio); + RUN_TEST(diagrams_excalidraw); + RUN_TEST(diagrams_d2_blocks); + RUN_TEST(diagrams_excalidraw_layout); + RUN_TEST(diagrams_excalidraw_svg); + RUN_TEST(diagrams_rst); + RUN_TEST(diagrams_asciidoc); + RUN_TEST(diagrams_links); + RUN_TEST(diagrams_links_graph); + RUN_TEST(diagrams_incremental_equals_full); + RUN_TEST(diagrams_graph); +} diff --git a/tests/test_doc_mentions_helpers.h b/tests/test_doc_mentions_helpers.h index 95e5f0503..778796393 100644 --- a/tests/test_doc_mentions_helpers.h +++ b/tests/test_doc_mentions_helpers.h @@ -430,19 +430,40 @@ static inline int dm_reason_lines(const char *db, const char *block) { * so it is the resolution an audit would ship; doc_links_md_ship_gate tests * the production gate itself. */ static const int DM_HELD_FAMILIES[] = { - CBM_DOCLINK_MD_BARE_PATH, CBM_DOCLINK_MD_SUPERSEDES_PROSE, - CBM_DOCLINK_RST_INCLUDE, CBM_DOCLINK_RST_KERNEL_DOC, - CBM_DOCLINK_ADOC_ATTRIBUTE, CBM_DOCLINK_ADOC_CODE_PATH, - CBM_DOCLINK_ADOC_CODE_NAME, CBM_DOCLINK_GO_DOC_LINK, - CBM_DOCLINK_JAVA_LINK, CBM_DOCLINK_JAVA_LINKPLAIN, - CBM_DOCLINK_JAVA_SEE, CBM_DOCLINK_JAVA_THROWS, - CBM_DOCLINK_JAVA_VALUE, CBM_DOCLINK_KT_KDOC_LINK, - CBM_DOCLINK_KT_SEE, CBM_DOCLINK_KT_THROWS, - CBM_DOCLINK_KT_SAMPLE, CBM_DOCLINK_RS_INTRA_DOC, - CBM_DOCLINK_RS_LINK_DEF, CBM_DOCLINK_PERL_POD_L, - CBM_DOCLINK_TS_LINK, CBM_DOCLINK_TS_LINKCODE, - CBM_DOCLINK_TS_SEE, CBM_DOCLINK_PHP_SEE, - CBM_DOCLINK_C_REF, CBM_DOCLINK_C_SEE, + CBM_DOCLINK_MD_BARE_PATH, + CBM_DOCLINK_MD_SUPERSEDES_PROSE, + CBM_DOCLINK_RST_INCLUDE, + CBM_DOCLINK_RST_KERNEL_DOC, + CBM_DOCLINK_ADOC_ATTRIBUTE, + CBM_DOCLINK_ADOC_CODE_PATH, + CBM_DOCLINK_ADOC_CODE_NAME, + CBM_DOCLINK_GO_DOC_LINK, + CBM_DOCLINK_JAVA_LINK, + CBM_DOCLINK_JAVA_LINKPLAIN, + CBM_DOCLINK_JAVA_SEE, + CBM_DOCLINK_JAVA_THROWS, + CBM_DOCLINK_JAVA_VALUE, + CBM_DOCLINK_KT_KDOC_LINK, + CBM_DOCLINK_KT_SEE, + CBM_DOCLINK_KT_THROWS, + CBM_DOCLINK_KT_SAMPLE, + CBM_DOCLINK_RS_INTRA_DOC, + CBM_DOCLINK_RS_LINK_DEF, + CBM_DOCLINK_PERL_POD_L, + CBM_DOCLINK_TS_LINK, + CBM_DOCLINK_TS_LINKCODE, + CBM_DOCLINK_TS_SEE, + CBM_DOCLINK_PHP_SEE, + CBM_DOCLINK_C_REF, + CBM_DOCLINK_C_SEE, + CBM_DOCLINK_DG_MERMAID_CLICK, + CBM_DOCLINK_DG_PLANTUML_URL, + CBM_DOCLINK_DG_PLANTUML_INCLUDE, + CBM_DOCLINK_DG_DRAWIO_LINK, + CBM_DOCLINK_DG_EXCALIDRAW_LINK, + CBM_DOCLINK_DG_DOT_URL, + CBM_DOCLINK_DG_D2_LINK, + CBM_DOCLINK_OPENAPI_OPERATION_ID, }; static inline void dm_ship_held_families(void) { diff --git a/tests/test_grammar_labels.c b/tests/test_grammar_labels.c index 7246e5b95..f2e58074d 100644 --- a/tests/test_grammar_labels.c +++ b/tests/test_grammar_labels.c @@ -242,7 +242,7 @@ static const LabelGolden LABEL_GOLDENS[] = { {"astro", "Module:1,Variable:1"}, {"templ", "Class:1,Module:1"}, {"typst", "Module:1"}, - {"mermaid", "Module:1"}, + {"mermaid", "Diagram:1,DiagramElement:2,Module:1"}, /* the fixture's flowchart */ {NULL, NULL}, }; diff --git a/tests/test_main.c b/tests/test_main.c index de3d80508..2bee46fef 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -1832,6 +1832,8 @@ extern void suite_doc_mentions_perl(void); extern void suite_doc_mentions_ts(void); extern void suite_doc_mentions_php(void); extern void suite_doc_mentions_c(void); +extern void suite_diagrams(void); +extern void suite_openapi(void); extern void suite_infrascan(void); extern void suite_cli(void); extern void suite_agent_clients(void); @@ -2310,6 +2312,8 @@ int main(int argc, char **argv) { RUN_SELECTED_SUITE(doc_mentions_ts); RUN_SELECTED_SUITE(doc_mentions_php); RUN_SELECTED_SUITE(doc_mentions_c); + RUN_SELECTED_SUITE(diagrams); + RUN_SELECTED_SUITE(openapi); /* Infrastructure scanning */ RUN_SELECTED_SUITE(infrascan); diff --git a/tests/test_openapi.c b/tests/test_openapi.c new file mode 100644 index 000000000..acc336fed --- /dev/null +++ b/tests/test_openapi.c @@ -0,0 +1,242 @@ +/* + * test_openapi.c -- OpenAPI / Swagger documents: each operation a Route named + * as code names it, each operationId a reference to its handler (openapi.c). + */ +#include "../src/foundation/compat.h" +#include "test_framework.h" +#include "test_helpers.h" +#include "test_doc_mentions_helpers.h" +#include "cbm.h" +#include "discover/discover.h" +#include +#include +#include + +static const CBMDefinition *oa_route(const CBMFileResult *r, const char *qn) { + for (int i = 0; i < r->defs.count; i++) { + const CBMDefinition *d = &r->defs.items[i]; + if (d->qualified_name && strcmp(d->qualified_name, qn) == 0) { + return d; + } + } + printf(" no definition %s\n", qn); + return NULL; +} + +static const char *oa_prop(const CBMDefinition *d, const char *key) { + for (int i = 0; d && d->extra_props && d->extra_props[i]; i += 2) { + if (strcmp(d->extra_props[i], key) == 0) { + return d->extra_props[i + 1]; + } + } + return NULL; +} + +static int oa_count(const CBMFileResult *r, const char *label) { + int n = 0; + for (int i = 0; i < r->defs.count; i++) { + n += r->defs.items[i].label && strcmp(r->defs.items[i].label, label) == 0; + } + return n; +} + +static int oa_link(const CBMFileResult *r, const char *source_qn, const char *raw) { + int n = 0; + for (int i = 0; i < r->doc_links.count; i++) { + const CBMDocLink *l = &r->doc_links.items[i]; + n += l->syntax == CBM_DOCLINK_OPENAPI_OPERATION_ID && + strcmp(l->source_qn, source_qn) == 0 && strcmp(l->raw, raw) == 0; + } + if (n != 1) { + printf(" link %s -> %s: %d found\n", source_qn, raw, n); + } + return n; +} + +static const char OA_YAML[] = "openapi: 3.0.0\n" + "info:\n" + " title: Users\n" + "paths:\n" + " /users/{id}:\n" + " x-openapi-router-controller: app.api.users\n" + " get:\n" + " operationId: get_user\n" + " delete:\n" + " operationId: deleteUser\n" + " x-openapi-router-controller: app.admin\n" + " /users:\n" + " post:\n" + " operationId: \"app.api.users.create_user\"\n" + " parameters: []\n" + " /health:\n" + " get:\n" + " summary: no id\n"; + +TEST(openapi_yaml) { + CBMFileResult *r = dm_extract(OA_YAML, CBM_LANG_YAML, "api/openapi.yaml"); + ASSERT_NOT_NULL(r); + const CBMDefinition *g = oa_route(r, "__route__GET__/users/{}"); + ASSERT_NOT_NULL(g); + ASSERT_STR_EQ(g->label, "Route"); + ASSERT_STR_EQ(g->name, "/users/{id}"); + ASSERT_STR_EQ(oa_prop(g, "method"), "GET"); + ASSERT_STR_EQ(oa_prop(g, "source"), "openapi"); + ASSERT_STR_EQ(oa_prop(g, "operation_id"), "get_user"); + ASSERT_NOT_NULL(oa_route(r, "__route__DELETE__/users/{}")); + ASSERT_NOT_NULL(oa_route(r, "__route__POST__/users")); + ASSERT_NULL(oa_prop(oa_route(r, "__route__GET__/health"), "operation_id")); + ASSERT_EQ(oa_count(r, "Route"), 4); /* `parameters` is no operation */ + /* the controller names the module of a bare id; a dotted id is whole */ + ASSERT_EQ(oa_link(r, "__route__GET__/users/{}", "app.api.users.get_user"), 1); + ASSERT_EQ(oa_link(r, "__route__DELETE__/users/{}", "app.admin.deleteUser"), 1); + ASSERT_EQ(oa_link(r, "__route__POST__/users", "app.api.users.create_user"), 1); + ASSERT_EQ(r->doc_links.count, 3); + cbm_free_result(r); + + /* YAML that describes no API is none */ + r = dm_extract("paths:\n /x:\n get:\n operationId: x\n", CBM_LANG_YAML, "conf.yaml"); + ASSERT_NOT_NULL(r); + ASSERT_EQ(oa_count(r, "Route"), 0); + cbm_free_result(r); + PASS(); +} + +TEST(openapi_json) { + static const char spec[] = + "{\"swagger\": \"2.0\", \"info\": {\"title\": \"Pets\"},\n" + " \"paths\": {\"/pets\": {\"get\": {\"operationId\": \"listPets\"},\n" + " \"post\": {\"operationId\": \"pets.create\"}}}}\n"; + /* openapi.json: its operations, not a node per key */ + CBMFileResult *r = dm_extract(spec, CBM_LANG_OPENAPI, "openapi.json"); + ASSERT_NOT_NULL(r); + ASSERT_NOT_NULL(oa_route(r, "__route__GET__/pets")); + ASSERT_EQ(oa_link(r, "__route__GET__/pets", "listPets"), 1); + ASSERT_EQ(oa_link(r, "__route__POST__/pets", "pets.create"), 1); + ASSERT_EQ(oa_count(r, "Variable"), 0); + cbm_free_result(r); + /* a JSON file of another name that describes an API */ + r = dm_extract(spec, CBM_LANG_JSON, "spec/pets-api.json"); + ASSERT_NOT_NULL(r); + ASSERT_EQ(oa_count(r, "Route"), 2); + cbm_free_result(r); + /* the document names it */ + ASSERT_EQ(cbm_language_for_filename("openapi.json"), CBM_LANG_OPENAPI); + ASSERT_EQ(cbm_language_for_filename("swagger.json"), CBM_LANG_OPENAPI); + PASS(); +} + +/* MENTIONS edges from the Route `route_qn` to a node whose QN ends with `tgt`. */ +static int oa_db_mentions(const char *db, const char *route_qn, const char *tgt) { + sqlite3 *h = NULL; + int n = 0; + if (sqlite3_open_v2(db, &h, SQLITE_OPEN_READONLY, NULL) != SQLITE_OK) { + sqlite3_close(h); + return -1; + } + sqlite3_stmt *st = NULL; + const char *sql = "SELECT count(*) FROM edges e JOIN nodes s ON s.id = e.source_id " + "JOIN nodes t ON t.id = e.target_id WHERE e.type = 'MENTIONS' " + "AND s.qualified_name = ?1 AND t.qualified_name LIKE '%' || ?2"; + if (sqlite3_prepare_v2(h, sql, -1, &st, NULL) == SQLITE_OK) { + sqlite3_bind_text(st, 1, route_qn, -1, SQLITE_TRANSIENT); + sqlite3_bind_text(st, 2, tgt, -1, SQLITE_TRANSIENT); + if (sqlite3_step(st) == SQLITE_ROW) { + n = sqlite3_column_int(st, 0); + } + } + sqlite3_finalize(st); + sqlite3_close(h); + return n; +} + +/* MENTIONS edges from the Route `route_qn` whose properties hold `part`. */ +static int oa_db_mentions_props(const char *db, const char *route_qn, const char *part) { + sqlite3 *h = NULL; + int n = 0; + if (sqlite3_open_v2(db, &h, SQLITE_OPEN_READONLY, NULL) != SQLITE_OK) { + sqlite3_close(h); + return -1; + } + sqlite3_stmt *st = NULL; + const char *sql = "SELECT count(*) FROM edges e JOIN nodes s ON s.id = e.source_id " + "WHERE e.type = 'MENTIONS' AND s.qualified_name = ?1 " + "AND e.properties LIKE '%' || ?2 || '%'"; + if (sqlite3_prepare_v2(h, sql, -1, &st, NULL) == SQLITE_OK) { + sqlite3_bind_text(st, 1, route_qn, -1, SQLITE_TRANSIENT); + sqlite3_bind_text(st, 2, part, -1, SQLITE_TRANSIENT); + if (sqlite3_step(st) == SQLITE_ROW) { + n = sqlite3_column_int(st, 0); + } + } + sqlite3_finalize(st); + sqlite3_close(h); + return n; +} + +static int oa_db_nodes(const char *db, const char *label, const char *qn) { + sqlite3 *h = NULL; + int n = 0; + if (sqlite3_open_v2(db, &h, SQLITE_OPEN_READONLY, NULL) != SQLITE_OK) { + sqlite3_close(h); + return -1; + } + sqlite3_stmt *st = NULL; + if (sqlite3_prepare_v2(h, "SELECT count(*) FROM nodes WHERE label = ?1 AND qualified_name = ?2", + -1, &st, NULL) == SQLITE_OK) { + sqlite3_bind_text(st, 1, label, -1, SQLITE_TRANSIENT); + sqlite3_bind_text(st, 2, qn, -1, SQLITE_TRANSIENT); + if (sqlite3_step(st) == SQLITE_ROW) { + n = sqlite3_column_int(st, 0); + } + } + sqlite3_finalize(st); + sqlite3_close(h); + return n; +} + +TEST(openapi_graph) { + dm_ship_held_families(); /* the operationId family is held until its audit */ + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_oa_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[512]; + char db[512]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + snprintf(db, sizeof(db), "%s/g.db", tmp); + th_write_file(TH_PATH(repo, "app/api/users.py"), + "def get_user(id):\n return id\n\n\ndef create_user():\n return 1\n"); + th_write_file(TH_PATH(repo, "api/openapi.yaml"), OA_YAML); + th_write_file(TH_PATH(repo, "openapi.json"), + "{\"openapi\": \"3.1.0\", \"paths\": {\"/pets\": {\"get\": " + "{\"operationId\": \"listPets\"}}, \"/dup\": {\"get\": " + "{\"operationId\": \"dup\"}}, \"/none\": {\"get\": " + "{\"operationId\": \"noSuchHandler\"}}}}\n"); + th_write_file(TH_PATH(repo, "app/pets.py"), "def listPets():\n return []\n\n\ndef dup():\n" + " return 1\n"); + th_write_file(TH_PATH(repo, "app/other.py"), "def dup():\n return 2\n"); + ASSERT_EQ(dm_index(repo, db, NULL), 0); + ASSERT_EQ(oa_db_nodes(db, "Route", "__route__GET__/users/{}"), 1); + ASSERT_EQ(oa_db_nodes(db, "Route", "__route__GET__/pets"), 1); + /* a dotted operationId names its handler */ + ASSERT_EQ(oa_db_mentions(db, "__route__GET__/users/{}", "users.get_user"), 1); + ASSERT_EQ(oa_db_mentions(db, "__route__POST__/users", "users.create_user"), 1); + /* through the API document, whichever resolver read its file (a YAML one) */ + ASSERT_EQ(oa_db_mentions_props(db, "__route__POST__/users", "\"via\":\"openapi\""), 1); + ASSERT_EQ(oa_db_mentions_props(db, "__route__POST__/users", "\"tier\":\"exact\""), 1); + ASSERT_EQ(oa_db_mentions_props(db, "__route__GET__/pets", "\"tier\":\"unique\""), 1); + /* a bare id names the one function of that name */ + ASSERT_EQ(oa_db_mentions(db, "__route__GET__/pets", "pets.listPets"), 1); + /* a module with no such function; a bare id two functions carry; one none does */ + ASSERT_EQ(oa_db_mentions(db, "__route__DELETE__/users/{}", ""), 0); + ASSERT_EQ(oa_db_mentions(db, "__route__GET__/dup", ""), 0); + ASSERT_EQ(oa_db_mentions(db, "__route__GET__/none", ""), 0); + th_cleanup(tmp); + cbm_doclink_test_reset_ships(); + PASS(); +} + +SUITE(openapi) { + RUN_TEST(openapi_yaml); + RUN_TEST(openapi_json); + RUN_TEST(openapi_graph); +} From 1dd84d744d3397d982f62dc20403cbf9fce52d47 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 11 Oct 2026 02:09:04 +0200 Subject: [PATCH 13/17] fix(doclinks): removed declarations repair incrementally in TS, PHP, C, Perl The TS/JS, PHP, C and Perl doc-link layers keep a scope blob per file but had no scope_delta hook, so any change of a file's declarations made the incremental closure repair decline (doc_scope_changed) and the index rebuild in full: removing a TypeScript function forced a full run. cbm_doclinks_scope_delta_lines compares two blobs line by line in order, as C# does: a dropped record that declares one name is repaired file by file (the name is reported, so the files whose references named it are re-resolved); anything new or changed, and every record others resolve through, stays a full run. The records that declare one name: TS D (a top-level declaration), PHP F and K (a function, a member), C D of a function, variable, enum constant, field or macro and A (a Doxygen anchor), Perl S (a sub). Signed-off-by: Martin Vogel --- src/pipeline/doc_links.c | 58 ++++++++++++++++++++++++++++++++++ src/pipeline/doc_links.h | 19 +++++++++++ src/pipeline/doc_links_c.c | 22 +++++++++++++ src/pipeline/doc_links_perl.c | 21 ++++++++++-- src/pipeline/doc_links_php.c | 30 ++++++++++++++++++ src/pipeline/doc_links_ts.c | 15 +++++++++ tests/test_doc_mentions_c.c | 7 +++- tests/test_doc_mentions_perl.c | 9 ++++-- tests/test_doc_mentions_php.c | 7 ++-- tests/test_doc_mentions_ts.c | 2 +- 10 files changed, 180 insertions(+), 10 deletions(-) diff --git a/src/pipeline/doc_links.c b/src/pipeline/doc_links.c index d1f0ab877..e50303dcc 100644 --- a/src/pipeline/doc_links.c +++ b/src/pipeline/doc_links.c @@ -120,6 +120,64 @@ int cbm_doclinks_scope_delta(const char *stored, const char *fresh, cbm_doclink_ return R->scope_delta(stored, fresh, removed, ud); } +/* One line of a scope blob at *p, without its newline; false at the end. */ +static bool scope_next_line(const char **p, const char **line, size_t *len) { + if (!*p || !**p) { + return false; + } + const char *nl = strchr(*p, '\n'); + *line = *p; + *len = nl ? (size_t)(nl - *p) : strlen(*p); + *p = nl ? nl + SKIP_ONE : *p + *len; + return true; +} + +int cbm_doclinks_scope_delta_lines(const char *stored, const char *fresh, + cbm_doclink_removable_fn removable, cbm_doclink_name_fn removed, + void *ud) { + const char *sp = stored; + const char *fp = fresh; + const char *sl = NULL; + const char *fl = NULL; + size_t slen = 0; + size_t flen = 0; + bool hs = scope_next_line(&sp, &sl, &slen); + bool hf = scope_next_line(&fp, &fl, &flen); + while (hs || hf) { + if (hs && hf && slen == flen && memcmp(sl, fl, slen) == 0) { + hs = scope_next_line(&sp, &sl, &slen); + hf = scope_next_line(&fp, &fl, &flen); + continue; + } + const char *name = NULL; + size_t nlen = 0; + if (!hs || !removable(sl, slen, &name, &nlen) || nlen == 0) { + return CBM_DOCLINK_DELTA_GLOBAL; /* new, changed, or resolved through */ + } + if (!removed || !removed(ud, name, nlen)) { + return CBM_NOT_FOUND; + } + hs = scope_next_line(&sp, &sl, &slen); + } + return CBM_DOCLINK_DELTA_LOCAL; +} + +bool cbm_doclinks_scope_field(const char *line, size_t len, int k, const char **field, + size_t *field_len) { + size_t s = 0; + for (int f = 0; f < k; f++) { + const char *tab = memchr(line + s, '\t', len - s); + if (!tab) { + return false; + } + s = (size_t)(tab - line) + SKIP_ONE; + } + const char *tab = memchr(line + s, '\t', len - s); + *field = line + s; + *field_len = tab ? (size_t)(tab - line) - s : len - s; + return true; +} + const char *cbm_doclinks_storable_scope(const char *scope, char **owned) { *owned = NULL; const cbm_doclink_resolver_t *R = resolver_of_scope(scope); diff --git a/src/pipeline/doc_links.h b/src/pipeline/doc_links.h index 4c56ef77f..2bfedc56b 100644 --- a/src/pipeline/doc_links.h +++ b/src/pipeline/doc_links.h @@ -313,6 +313,25 @@ bool cbm_doclinks_is_scope_input(const char *rel_path); int cbm_doclinks_scope_delta(const char *stored, const char *fresh, cbm_doclink_name_fn removed, void *ud); +/* A record of a scope blob that declares one name other files may reference + * and nothing others resolve THROUGH: true, with that name. */ +typedef bool (*cbm_doclink_removable_fn)(const char *line, size_t len, const char **name, + size_t *name_len); + +/* A scope_delta over the lines of two blobs, in order (a record may belong to + * the one above it): the same line in both is unchanged; a stored line the + * fresh blob dropped is LOCAL when `removable` names it -- the name goes to + * `removed` -- and GLOBAL otherwise; a line only the fresh blob has, new or + * changed, is GLOBAL. A cbm_doclink_delta_t, or CBM_NOT_FOUND when a name + * could not be recorded. */ +int cbm_doclinks_scope_delta_lines(const char *stored, const char *fresh, + cbm_doclink_removable_fn removable, cbm_doclink_name_fn removed, + void *ud); + +/* Field k (0: the record's kind) of a tab-separated scope line. */ +bool cbm_doclinks_scope_field(const char *line, size_t len, int k, const char **field, + size_t *field_len); + /* What is stored for a scope blob written in this run (the surface row's * `dl`, lsp_surface.c): the blob itself, or its language's rejected-scope * marker when that language's reader refuses it (scope_accepted), made into diff --git a/src/pipeline/doc_links_c.c b/src/pipeline/doc_links_c.c index 044d69852..b33883926 100644 --- a/src/pipeline/doc_links_c.c +++ b/src/pipeline/doc_links_c.c @@ -471,6 +471,27 @@ static void cr_resolve_name(const cr_index_t *ix, int src, const CBMDocLink *lin } } +/* A function, variable, enum constant, field or macro (D kind link name qn + * parent, kind f v e r m M) and a Doxygen anchor (A name) each declare one + * name. A type (t) is what members and Type::member are found through: + * GLOBAL, as is every new or changed record. */ +static bool cr_removable(const char *line, size_t len, const char **name, size_t *nlen) { + if (len <= PAIR_LEN || line[1] != '\t') { + return false; + } + if (line[0] == 'A') { + return cbm_doclinks_scope_field(line, len, 1, name, nlen); + } + return line[0] == 'D' && strchr("fverMm", line[PAIR_LEN]) && line[PAIR_LEN] && + line[PAIR_LEN + SKIP_ONE] == '\t' && + cbm_doclinks_scope_field(line, len, CBM_SZ_3, name, nlen); +} + +static int cr_scope_delta(const char *stored, const char *fresh, cbm_doclink_name_fn removed, + void *ud) { + return cbm_doclinks_scope_delta_lines(stored, fresh, cr_removable, removed, ud); +} + const cbm_doclink_resolver_t cbm_doclink_c_resolver = { .langs = {CBM_LANG_C, CBM_LANG_CPP}, .lang_count = PAIR_LEN, @@ -478,4 +499,5 @@ const cbm_doclink_resolver_t cbm_doclink_c_resolver = { .build = cr_build, .destroy = cr_destroy, .resolve = cr_resolve, + .scope_delta = cr_scope_delta, }; diff --git a/src/pipeline/doc_links_perl.c b/src/pipeline/doc_links_perl.c index 8736ce506..27ba5a9eb 100644 --- a/src/pipeline/doc_links_perl.c +++ b/src/pipeline/doc_links_perl.c @@ -26,9 +26,10 @@ * Test code (t/, xt/, *.t, and cbm_is_test_file) is bound only from test code. * Every edge is exact: a module name and a heading are spelled out in full. * - * The scope blob carries no lines (doclink_perl.c), and without a scope_delta - * hook every change of a blob is a full re-resolution: a change that adds or - * drops a package, a sub or a heading; an edit inside a sub changes no blob. + * The scope blob carries no lines (doclink_perl.c). A dropped sub is repaired + * file by file (pl_scope_delta); a change that adds a sub, or adds or drops a + * package or a heading, is a full re-resolution; an edit inside a sub changes + * no blob. */ #include "pipeline/doc_links.h" #include "pipeline/pipeline_internal.h" /* cbm_pipeline_file_node */ @@ -479,6 +480,19 @@ static void pr_resolve(const void *index, void *state, int run_file, const CBMDo pr_edge(&c, f, pr_module_node(&c, f)); } +/* A sub (S pkg name) declares one name. A package (its index places every + * sub after it), a heading (a section is found by its text) and the + * parse-error mark change how others resolve: GLOBAL. */ +static bool pl_removable(const char *line, size_t len, const char **name, size_t *nlen) { + return len > PAIR_LEN && line[0] == 'S' && line[1] == '\t' && + cbm_doclinks_scope_field(line, len, PAIR_LEN, name, nlen); +} + +static int pl_scope_delta(const char *stored, const char *fresh, cbm_doclink_name_fn removed, + void *ud) { + return cbm_doclinks_scope_delta_lines(stored, fresh, pl_removable, removed, ud); +} + const cbm_doclink_resolver_t cbm_doclink_perl_resolver = { .langs = {CBM_LANG_PERL}, .lang_count = 1, @@ -486,4 +500,5 @@ const cbm_doclink_resolver_t cbm_doclink_perl_resolver = { .build = pr_build, .destroy = pr_destroy, .resolve = pr_resolve, + .scope_delta = pl_scope_delta, }; diff --git a/src/pipeline/doc_links_php.c b/src/pipeline/doc_links_php.c index 2f8e03f5a..11e256c21 100644 --- a/src/pipeline/doc_links_php.c +++ b/src/pipeline/doc_links_php.c @@ -954,6 +954,35 @@ static void pr_resolve(const void *index, void *state, int run_file, const CBMDo pr_out(&c, res, out); } +/* A top-level function (F fqn flags) or a member of the type above it (K kind + * name) declares one name; reported as a reference spells it, the last + * segment of a qualified name. A type, a supertype, a namespace block or a + * use import changes how others resolve: GLOBAL. */ +static bool pr_removable(const char *line, size_t len, const char **name, size_t *nlen) { + if (len <= PAIR_LEN || line[1] != '\t') { + return false; + } + if (line[0] == 'K') { + return cbm_doclinks_scope_field(line, len, PAIR_LEN, name, nlen); + } + if (line[0] != 'F' || !cbm_doclinks_scope_field(line, len, 1, name, nlen)) { + return false; + } + for (size_t i = *nlen; i > 0; i--) { + if ((*name)[i - SKIP_ONE] == '\\') { + *nlen -= i; + *name += i; + break; + } + } + return true; +} + +static int pr_scope_delta(const char *stored, const char *fresh, cbm_doclink_name_fn removed, + void *ud) { + return cbm_doclinks_scope_delta_lines(stored, fresh, pr_removable, removed, ud); +} + const cbm_doclink_resolver_t cbm_doclink_php_resolver = { .langs = {CBM_LANG_PHP}, .lang_count = SKIP_ONE, @@ -961,4 +990,5 @@ const cbm_doclink_resolver_t cbm_doclink_php_resolver = { .build = pr_build, .destroy = pr_destroy, .resolve = pr_resolve, + .scope_delta = pr_scope_delta, }; diff --git a/src/pipeline/doc_links_ts.c b/src/pipeline/doc_links_ts.c index c52d2e6da..9ac4e4fce 100644 --- a/src/pipeline/doc_links_ts.c +++ b/src/pipeline/doc_links_ts.c @@ -1083,6 +1083,20 @@ static void tr_resolve(const void *index, void *state, int run_file, const CBMDo tr_out(&c, r); } +/* A top-level declaration (D name flags) declares one name: other files reach + * it through an import of it, so a reference that did either bound it (an + * edge into this file) or names it in its unresolved row. Imports, exports + * and namespaces change how others resolve: GLOBAL. */ +static bool tr_removable(const char *line, size_t len, const char **name, size_t *nlen) { + return len > PAIR_LEN && line[0] == 'D' && line[1] == '\t' && + cbm_doclinks_scope_field(line, len, 1, name, nlen); +} + +static int tr_scope_delta(const char *stored, const char *fresh, cbm_doclink_name_fn removed, + void *ud) { + return cbm_doclinks_scope_delta_lines(stored, fresh, tr_removable, removed, ud); +} + const cbm_doclink_resolver_t cbm_doclink_ts_resolver = { .langs = {CBM_LANG_TYPESCRIPT, CBM_LANG_TSX, CBM_LANG_JAVASCRIPT}, .lang_count = CBM_SZ_3, @@ -1090,4 +1104,5 @@ const cbm_doclink_resolver_t cbm_doclink_ts_resolver = { .build = tr_build, .destroy = tr_destroy, .resolve = tr_resolve, + .scope_delta = tr_scope_delta, }; diff --git a/tests/test_doc_mentions_c.c b/tests/test_doc_mentions_c.c index b4fe96e84..93fa5f4cc 100644 --- a/tests/test_doc_mentions_c.c +++ b/tests/test_doc_mentions_c.c @@ -287,7 +287,12 @@ TEST(doc_mentions_c_incremental_equals_full) { ASSERT_EQ(dm_step(repo, inc_db, full_db, "body edit", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0); ASSERT_TRUE(c_edge(inc_db, "src.use.user", "src.lib.a")); - /* a becomes static: without a scope_delta hook that is a full run */ + /* b goes: the files that name it are repaired, nothing else */ + th_write_file(TH_PATH(repo, "src/lib.c"), "int a(void) { return 2; }\n"); + ASSERT_EQ(dm_step(repo, inc_db, full_db, "b removed", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0); + ASSERT_TRUE(c_edge(inc_db, "src.use.user", "src.lib.a")); + + /* a becomes static (and b is back): changed records, a full run */ th_write_file(TH_PATH(repo, "src/lib.c"), "static int a(void) { return 0; }\nint b(void) { return 1; }\n"); ASSERT_EQ(dm_step(repo, inc_db, full_db, "made static", CBM_INCREMENTAL_ROUTE_FORCED_FULL), 0); diff --git a/tests/test_doc_mentions_perl.c b/tests/test_doc_mentions_perl.c index 177d4a9d5..bc9b7aabe 100644 --- a/tests/test_doc_mentions_perl.c +++ b/tests/test_doc_mentions_perl.c @@ -229,8 +229,7 @@ TEST(doc_mentions_perl_incremental_equals_full) { dm_step(repo, inc_db, full_db, "module doc link", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0); ASSERT_TRUE(pl_edge(inc_db, "lib.Knob.pm.__file__", "lib.Knob.Dial.pm.__file__")); - /* a new heading changes the file's scope blob: without a scope_delta hook - * that is a full run */ + /* a new heading changes how a section is found: a full run */ th_write_file(TH_PATH(repo, "lib/Knob.pm"), "package Knob;\n\n=head1 NAME\n\nKnob, a knob - see L and\n" "L.\n\n=head1 SEE ALSO\n\nL\n\n=cut\n\n" @@ -249,6 +248,12 @@ TEST(doc_mentions_perl_incremental_equals_full) { int n = 0; dm_edge(inc_db, "lib.Knob.pm.__file__", "lib.Knob.Dial.spin", props, sizeof(props), &n); ASSERT_EQ(n, 0); + + /* the sub goes: the files that name it are repaired, nothing else */ + th_write_file(TH_PATH(repo, "lib/Knob/Dial.pm"), + "package Knob::Dial;\n\n=head1 OVERVIEW\n\nThe dial.\n\n=cut\n\n1;\n"); + ASSERT_EQ(dm_step(repo, inc_db, full_db, "sub removed", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), + 0); th_cleanup(tmp); PASS(); } diff --git a/tests/test_doc_mentions_php.c b/tests/test_doc_mentions_php.c index 1e31b2c5d..46538f890 100644 --- a/tests/test_doc_mentions_php.c +++ b/tests/test_doc_mentions_php.c @@ -425,10 +425,11 @@ TEST(doc_mentions_php_incremental_equals_full) { ASSERT_EQ(dm_step(repo, inc_db, full_db, "body edit", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0); ASSERT_TRUE(php_edge(inc_db, "src.use.user", "src.lib.a")); - /* the function goes: without a scope_delta hook that is a full run */ + /* the function goes: the files that name it are repaired, nothing else */ th_write_file(TH_PATH(repo, "src/lib.php"), " Date: Sun, 11 Oct 2026 12:00:31 +0200 Subject: [PATCH 14/17] feat(docs): a section's text keeps its diagram's labels, not its syntax A Markdown Section's docstring is its body collapsed and cut at 500 bytes. A diagram fence in that body put the format's syntax into it (`-->`, `participant`, `classDiagram`, `subgraph`), which names no code, and pushed the section's prose out of the cap. The doc -> code candidates are scored on that text. Each top-level diagram fence (mermaid, plantuml, puml, uml, dot, graphviz, d2) is now replaced in the section's text by the labels its diagram parser reads: the names of its elements, then its arrow labels, in source order. The parse runs into a scratch result, so the file's own Diagram and DiagramElement nodes are untouched. A section without such a fence keeps its text byte for byte; other fences stay as written. Measured before it was chosen, on held-out repositories (147 sections that embed diagrams, 503 section/candidate pairs, three blind reading rounds): fence as written 277 stored precision 0.45 42 sections with a correct candidate labels (this) 310 stored precision 0.54 56 sections with a correct candidate fence dropped 127 stored precision 0.54 29 sections with a correct candidate Labels gave 20 sections a correct candidate that the fence as written did not, and lost 6 (exact McNemar p = 0.009). Dropping the fence lost more than it gained. Signed-off-by: Martin Vogel --- internal/cbm/cbm.h | 5 ++ internal/cbm/diagram.c | 92 ++++++++++++++++++++++++++++++++++--- internal/cbm/extract_defs.c | 66 ++++++++++++++++++++++++-- tests/test_diagrams.c | 52 +++++++++++++++++++++ 4 files changed, 204 insertions(+), 11 deletions(-) diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index 7d3bf15a0..f5b90a1f5 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -1167,6 +1167,11 @@ void cbm_extract_channels(CBMExtractCtx *ctx); * or AsciiDoc block, or a diagram file -- with a DiagramElement definition per * element and the arrows between them. */ void cbm_extract_diagrams(CBMExtractCtx *ctx); +/* Is a Markdown block a diagram fence, and the labels its parser reads from + * it (space-joined; NULL for none): a Section's text keeps those labels in + * place of the fence (extract_markdown_section_body). */ +bool cbm_diagram_fence_is_diagram(const char *source, TSNode block); +const char *cbm_diagram_fence_labels(CBMExtractCtx *ctx, TSNode block); /* A diagram file (CBM_LANG_DIAGRAM): its format comes from its extension. */ void cbm_extract_diagram_file(CBMExtractCtx *ctx); diff --git a/internal/cbm/diagram.c b/internal/cbm/diagram.c index 228c3ca67..cdd3785f4 100644 --- a/internal/cbm/diagram.c +++ b/internal/cbm/diagram.c @@ -2813,24 +2813,31 @@ static bool dg_kind(TSNode n, const char *k) { return !ts_node_is_null(n) && strcmp(ts_node_type(n), k) == 0; } -/* A fenced block of a Markdown file: a diagram when its info string says so. */ -static void dg_markdown_fence(CBMExtractCtx *ctx, TSNode block, dg_ordinal_t *ords, int *nords) { +/* The diagram format a Markdown fenced block's info string names, NULL for + * none; *content is the block's text. */ +static const char *dg_fence_format(const char *source, TSNode block, TSNode *content) { TSNode info = {0}; - TSNode content = {0}; + *content = (TSNode){0}; uint32_t nc = ts_node_named_child_count(block); for (uint32_t i = 0; i < nc; i++) { TSNode c = ts_node_named_child(block, i); info = dg_kind(c, "info_string") ? c : info; - content = dg_kind(c, "code_fence_content") ? c : content; + *content = dg_kind(c, "code_fence_content") ? c : *content; } - if (ts_node_is_null(info) || ts_node_is_null(content)) { - return; + if (ts_node_is_null(info) || ts_node_is_null(*content)) { + return NULL; } TSNode lang = ts_node_named_child(info, 0); TSNode word = dg_kind(lang, "language") ? lang : info; uint32_t wb = ts_node_start_byte(word); uint32_t we = ts_node_end_byte(word); - const char *format = dg_format_of(ctx->source + wb, we - wb); + return dg_format_of(source + wb, we - wb); +} + +/* A fenced block of a Markdown file: a diagram when its info string says so. */ +static void dg_markdown_fence(CBMExtractCtx *ctx, TSNode block, dg_ordinal_t *ords, int *nords) { + TSNode content; + const char *format = dg_fence_format(ctx->source, block, &content); if (!format) { return; } @@ -3508,6 +3515,77 @@ void cbm_extract_diagram_file(CBMExtractCtx *ctx) { } } +/* ── What a Markdown section's text keeps of its diagram fences ───── */ + +bool cbm_diagram_fence_is_diagram(const char *source, TSNode block) { + TSNode content; + return dg_kind(block, "fenced_code_block") && dg_fence_format(source, block, &content); +} + +/* The labels of one Markdown diagram fence, space-joined in source order: the + * DiagramElement names and arrow texts its parser produces, without the + * format's syntax. The parse runs into a scratch result, so the file's own + * definitions are untouched. NULL when the fence says nothing. */ +const char *cbm_diagram_fence_labels(CBMExtractCtx *ctx, TSNode block) { + TSNode content; + const char *format = dg_fence_format(ctx->source, block, &content); + if (!format) { + return NULL; + } + CBMFileResult scratch; + memset(&scratch, 0, sizeof(scratch)); + CBMExtractCtx sc = *ctx; + sc.result = &scratch; + uint32_t cb = ts_node_start_byte(content); + dg_source_t src = { + .format = format, + .parent_qn = "e1", + .ordinal = SKIP_ONE, + .text = ctx->source + cb, + .len = ts_node_end_byte(content) - cb, + .first_line = ts_node_start_point(content).row + SKIP_ONE, + .start_line = ts_node_start_point(block).row + SKIP_ONE, + .end_line = ts_node_end_point(block).row + SKIP_ONE, + }; + dg_build(&sc, &src); + size_t total = 0; + for (int i = 0; i < scratch.defs.count; i++) { + const CBMDefinition *d = &scratch.defs.items[i]; + if (d->name && d->label && strcmp(d->label, "DiagramElement") == 0) { + total += strlen(d->name) + SKIP_ONE; + } + } + for (int i = 0; i < scratch.diagram_arrows.count; i++) { + const char *l = scratch.diagram_arrows.items[i].label; + total += l ? strlen(l) + SKIP_ONE : 0; + } + char *out = total ? (char *)cbm_arena_alloc(ctx->arena, total + SKIP_ONE) : NULL; + if (!out) { + return NULL; + } + size_t w = 0; + for (int i = 0; i < scratch.defs.count; i++) { + const CBMDefinition *d = &scratch.defs.items[i]; + if (d->name && d->label && strcmp(d->label, "DiagramElement") == 0) { + size_t l = strlen(d->name); + memcpy(out + w, d->name, l); + w += l; + out[w++] = ' '; + } + } + for (int i = 0; i < scratch.diagram_arrows.count; i++) { + const char *l = scratch.diagram_arrows.items[i].label; + if (l) { + size_t n = strlen(l); + memcpy(out + w, l, n); + w += n; + out[w++] = ' '; + } + } + out[w] = '\0'; + return out; +} + void cbm_extract_diagrams(CBMExtractCtx *ctx) { if (!ctx || !ctx->result || !ctx->source) { return; diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index 5917c0804..e046bca83 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -6840,10 +6840,22 @@ static bool is_markdown_heading_kind(const char *kind) { * with nested subsections as further `section` children that own their own * text. Stopping at either a `section` or another heading therefore gives each * Section exactly its OWN body under both that shape and a flat one. The bytes - * between are contiguous in the source, so one slice beats concatenation. */ -static const char *extract_markdown_section_body(CBMArena *a, TSNode heading, const char *source) { + * between are contiguous in the source, so one slice beats concatenation. + * + * A top-level diagram fence (```mermaid, plantuml, dot, d2 ...) is the one + * exception: the body keeps the labels its diagram parser reads, not the + * format's syntax. Arrows and keywords (`-->`, `participant`, `classDiagram`) + * name no code and crowd the prose out of the 500-byte cap, while the labels + * are what name the code. Judged on held-out sections that embed diagrams, + * labels gave more sections a correct code candidate (56 vs 42 of 147) at a + * higher precision (0.54 vs 0.45) than the fence as written; dropping the + * fence lost candidates (29). The diagram itself keeps its own nodes. */ +static const char *extract_markdown_section_body(CBMExtractCtx *ctx, TSNode heading) { + CBMArena *a = ctx->arena; + const char *source = ctx->source; uint32_t start = ts_node_end_byte(heading); uint32_t end = start; + int fences = 0; for (TSNode sib = ts_node_next_sibling(heading); !ts_node_is_null(sib); sib = ts_node_next_sibling(sib)) { const char *sk = ts_node_type(sib); @@ -6851,11 +6863,57 @@ static const char *extract_markdown_section_body(CBMArena *a, TSNode heading, co break; } end = ts_node_end_byte(sib); + fences += cbm_diagram_fence_is_diagram(source, sib); } if (end <= start) { return NULL; } - return collapse_prose(a, source + start, (size_t)(end - start)); + if (fences == 0) { + return collapse_prose(a, source + start, (size_t)(end - start)); + } + /* The body with each diagram fence replaced by its labels. */ + const char **labels = (const char **)cbm_arena_alloc(a, (size_t)fences * sizeof(char *)); + if (!labels) { + return NULL; + } + size_t total = 0; + int k = 0; + for (TSNode sib = ts_node_next_sibling(heading); !ts_node_is_null(sib) && k < fences; + sib = ts_node_next_sibling(sib)) { + if (cbm_diagram_fence_is_diagram(source, sib)) { + labels[k] = cbm_diagram_fence_labels(ctx, sib); + total += labels[k] ? strlen(labels[k]) + SKIP_ONE : 0; + k++; + } + } + char *buf = (char *)cbm_arena_alloc(a, (size_t)(end - start) + total + SKIP_ONE); + if (!buf) { + return NULL; + } + size_t w = 0; + uint32_t prev = start; + k = 0; + for (TSNode sib = ts_node_next_sibling(heading); !ts_node_is_null(sib) && k < fences; + sib = ts_node_next_sibling(sib)) { + if (!cbm_diagram_fence_is_diagram(source, sib)) { + continue; + } + uint32_t sb = ts_node_start_byte(sib); + memcpy(buf + w, source + prev, sb - prev); + w += sb - prev; + if (labels[k]) { + size_t n = strlen(labels[k]); + memcpy(buf + w, labels[k], n); + w += n; + buf[w++] = ' '; + } + k++; + prev = ts_node_end_byte(sib); + } + memcpy(buf + w, source + prev, end - prev); + w += end - prev; + buf[w] = '\0'; + return collapse_prose(a, buf, w); } /* Push a config-language definition that carries prose. push_simple_class_def @@ -7055,7 +7113,7 @@ static bool extract_config_class_def(CBMExtractCtx *ctx, TSNode node, const char // "Class"; that assertion is the inaccurate side and is flagged for review. label = "Section"; // #518: index what the section SAYS, not just its title. - docstring = extract_markdown_section_body(a, node, ctx->source); + docstring = extract_markdown_section_body(ctx, node); } else if (ctx->language == CBM_LANG_HCL && strcmp(kind, "block") == 0) { name = find_hcl_block_name(a, node, ctx->source); } else { diff --git a/tests/test_diagrams.c b/tests/test_diagrams.c index 42e2115f0..574f27148 100644 --- a/tests/test_diagrams.c +++ b/tests/test_diagrams.c @@ -136,6 +136,57 @@ TEST(diagrams_markdown_mermaid) { PASS(); } +/* A Section's text keeps a diagram fence's labels, not the format's syntax; + * prose around it stays in order, other fences stay as written. */ +TEST(diagrams_section_text_keeps_labels) { + CBMFileResult *r = dm_extract(DG_DOC, CBM_LANG_MARKDOWN, "docs/arch.md"); + ASSERT_NOT_NULL(r); + const CBMDefinition *arch = dg_def(r, ".Arch"); + ASSERT_NOT_NULL(arch); + ASSERT_STR_EQ(arch->label, "Section"); + const char *doc = arch->docstring; + ASSERT_NOT_NULL(doc); + const char *kept[] = {"Web app", "API", "DB", "Cache", "Backend", "calls", + "retries", "User", "login", "token", NULL}; + for (int i = 0; kept[i]; i++) { + if (!strstr(doc, kept[i])) { + printf(" label %s missing from: %s\n", kept[i], doc); + } + ASSERT(strstr(doc, kept[i]) != NULL); + } + const char *dropped[] = {"```", "mermaid", "flowchart", "-->", "sequenceDiagram", + "participant", "subgraph", NULL}; + for (int i = 0; dropped[i]; i++) { + if (strstr(doc, dropped[i])) { + printf(" syntax %s kept in: %s\n", dropped[i], doc); + } + ASSERT(strstr(doc, dropped[i]) == NULL); + } + ASSERT_STR_EQ(dg_def(r, ".Doc")->docstring, "Intro."); /* no fence: as before */ + cbm_free_result(r); + + r = dm_extract("# Mixed\n" + "\n" + "Before.\n" + "\n" + "```mermaid\n" + "flowchart LR\n" + " A[Login] --> B[Session]\n" + "```\n" + "\n" + "```python\n" + "x = 1\n" + "```\n" + "\n" + "After.\n", + CBM_LANG_MARKDOWN, "docs/mixed.md"); + ASSERT_NOT_NULL(r); + ASSERT_STR_EQ(dg_def(r, ".Mixed")->docstring, + "Before. Login Session ```python x = 1 ``` After."); + cbm_free_result(r); + PASS(); +} + TEST(diagrams_mermaid_kinds) { CBMFileResult *r = dm_extract("erDiagram\n" " CUSTOMER ||--o{ ORDER : places\n" @@ -1046,6 +1097,7 @@ TEST(diagrams_incremental_equals_full) { SUITE(diagrams) { RUN_TEST(diagrams_markdown_mermaid); + RUN_TEST(diagrams_section_text_keeps_labels); RUN_TEST(diagrams_mermaid_kinds); RUN_TEST(diagrams_other_fences); RUN_TEST(diagrams_plantuml); From 78eace4fd3631485b8784796b5e0ee4de143a105 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 11 Oct 2026 12:43:35 +0200 Subject: [PATCH 15/17] doc-links: ship the doc-comment families that passed their held-out audit The held-out audit of the eight new languages is scored. Each tier is a language and a reference form, read on a sample of 150 links (census below 150) from repositories never used to build or tune the resolvers. Three blind reading rounds; bar: Wilson 95% lower bound >= 0.90 and point >= 0.95. tier population judged correct Wilson low go doc_link 5980 150 150 0.975 java link 23934 150 150 0.975 java linkplain 1372 150 150 0.975 java see 7711 150 150 0.975 java throws 650 150 150 0.975 java value 79 79 79 0.954 kotlin kdoc_link 3888 150 150 0.975 kotlin sample 279 150 150 0.975 kotlin see 369 150 150 0.975 kotlin throws 73 73 73 0.950 rust intra_doc 14119 150 149 0.963 rust link_def 1491 150 150 0.975 perl pod_L 2255 150 150 0.975 ts/js link 1222 150 150 0.975 php see 5171 150 150 0.975 c doxygen_ref 463 150 150 0.975 c see 509 150 145 0.924 These 17 families now ship: a resolved reference is a MENTIONS edge. TS/JS {@linkcode} (10 held-out links) and @see (18) were all judged correct, but are too few to reach the bar; they stay held, as do the diagram and OpenAPI link families (3 links on 21 held-out repositories). The Go ship-gate test turned its family off and restored only the held families afterwards; with Go shipping it left Go off for the tests that follow it. It restores the production gates first now. Signed-off-by: Martin Vogel --- internal/cbm/doclink.h | 49 ++++++++++++++++--------------- tests/test_doc_links_md.c | 12 ++++++++ tests/test_doc_mentions_go.c | 5 ++-- tests/test_doc_mentions_helpers.h | 40 ++++--------------------- tests/test_doc_mentions_jvm.c | 4 +-- 5 files changed, 48 insertions(+), 62 deletions(-) diff --git a/internal/cbm/doclink.h b/internal/cbm/doclink.h index 0ccdcec20..c645d7346 100644 --- a/internal/cbm/doclink.h +++ b/internal/cbm/doclink.h @@ -89,32 +89,33 @@ X(CBM_DOCLINK_CS_SEEALSO, CBM_LANG_CSHARP, "seealso", false, true, "MENTIONS") \ X(CBM_DOCLINK_CS_EXCEPTION, CBM_LANG_CSHARP, "exception", false, true, "MENTIONS") \ X(CBM_DOCLINK_CS_INHERITDOC, CBM_LANG_CSHARP, "inheritdoc", false, true, "MENTIONS") \ - /* go (doclink_go.c): [Name] doc links; held until its held-out audit passes */ \ - X(CBM_DOCLINK_GO_DOC_LINK, CBM_LANG_GO, "doc_link", false, false, "MENTIONS") \ - /* java and kotlin (doclink_jvm.c): Javadoc and KDoc; held until their audit passes */ \ - X(CBM_DOCLINK_JAVA_LINK, CBM_LANG_JAVA, "link", false, false, "MENTIONS") \ - X(CBM_DOCLINK_JAVA_LINKPLAIN, CBM_LANG_JAVA, "linkplain", false, false, "MENTIONS") \ - X(CBM_DOCLINK_JAVA_SEE, CBM_LANG_JAVA, "see", false, false, "MENTIONS") \ - X(CBM_DOCLINK_JAVA_THROWS, CBM_LANG_JAVA, "throws", false, false, "MENTIONS") \ - X(CBM_DOCLINK_JAVA_VALUE, CBM_LANG_JAVA, "value", false, false, "MENTIONS") \ - X(CBM_DOCLINK_KT_KDOC_LINK, CBM_LANG_KOTLIN, "kdoc_link", false, false, "MENTIONS") \ - X(CBM_DOCLINK_KT_SEE, CBM_LANG_KOTLIN, "see", false, false, "MENTIONS") \ - X(CBM_DOCLINK_KT_THROWS, CBM_LANG_KOTLIN, "throws", false, false, "MENTIONS") \ - X(CBM_DOCLINK_KT_SAMPLE, CBM_LANG_KOTLIN, "sample", false, false, "MENTIONS") \ - /* rust (doclink_rs.c): intra-doc links; held until their audit passes */ \ - X(CBM_DOCLINK_RS_INTRA_DOC, CBM_LANG_RUST, "intra_doc", false, false, "MENTIONS") \ - X(CBM_DOCLINK_RS_LINK_DEF, CBM_LANG_RUST, "link_def", false, false, "MENTIONS") \ - /* perl (doclink_perl.c): POD L<> codes; held until their audit passes */ \ - X(CBM_DOCLINK_PERL_POD_L, CBM_LANG_PERL, "pod_L", false, false, "MENTIONS") \ - /* typescript and javascript (doclink_ts.c): JSDoc/TSDoc; held until their audit */ \ - X(CBM_DOCLINK_TS_LINK, CBM_LANG_TYPESCRIPT, "link", false, false, "MENTIONS") \ + /* go (doclink_go.c): [Name] doc links; held-out audit passed */ \ + X(CBM_DOCLINK_GO_DOC_LINK, CBM_LANG_GO, "doc_link", false, true, "MENTIONS") \ + /* java and kotlin (doclink_jvm.c): Javadoc and KDoc; held-out audit passed */ \ + X(CBM_DOCLINK_JAVA_LINK, CBM_LANG_JAVA, "link", false, true, "MENTIONS") \ + X(CBM_DOCLINK_JAVA_LINKPLAIN, CBM_LANG_JAVA, "linkplain", false, true, "MENTIONS") \ + X(CBM_DOCLINK_JAVA_SEE, CBM_LANG_JAVA, "see", false, true, "MENTIONS") \ + X(CBM_DOCLINK_JAVA_THROWS, CBM_LANG_JAVA, "throws", false, true, "MENTIONS") \ + X(CBM_DOCLINK_JAVA_VALUE, CBM_LANG_JAVA, "value", false, true, "MENTIONS") \ + X(CBM_DOCLINK_KT_KDOC_LINK, CBM_LANG_KOTLIN, "kdoc_link", false, true, "MENTIONS") \ + X(CBM_DOCLINK_KT_SEE, CBM_LANG_KOTLIN, "see", false, true, "MENTIONS") \ + X(CBM_DOCLINK_KT_THROWS, CBM_LANG_KOTLIN, "throws", false, true, "MENTIONS") \ + X(CBM_DOCLINK_KT_SAMPLE, CBM_LANG_KOTLIN, "sample", false, true, "MENTIONS") \ + /* rust (doclink_rs.c): intra-doc links; held-out audit passed */ \ + X(CBM_DOCLINK_RS_INTRA_DOC, CBM_LANG_RUST, "intra_doc", false, true, "MENTIONS") \ + X(CBM_DOCLINK_RS_LINK_DEF, CBM_LANG_RUST, "link_def", false, true, "MENTIONS") \ + /* perl (doclink_perl.c): POD L<> codes; held-out audit passed */ \ + X(CBM_DOCLINK_PERL_POD_L, CBM_LANG_PERL, "pod_L", false, true, "MENTIONS") \ + /* typescript and javascript (doclink_ts.c): JSDoc/TSDoc; {@link} passed its held-out */ \ + /* audit; {@linkcode} (10) and @see (18) had too few held-out links to pass: held */ \ + X(CBM_DOCLINK_TS_LINK, CBM_LANG_TYPESCRIPT, "link", false, true, "MENTIONS") \ X(CBM_DOCLINK_TS_LINKCODE, CBM_LANG_TYPESCRIPT, "linkcode", false, false, "MENTIONS") \ X(CBM_DOCLINK_TS_SEE, CBM_LANG_TYPESCRIPT, "see", false, false, "MENTIONS") \ - /* php (doclink_php.c): PHPDoc @see and {@see}; held until their audit passes */ \ - X(CBM_DOCLINK_PHP_SEE, CBM_LANG_PHP, "see", false, false, "MENTIONS") \ - /* c (doclink_c.c): Doxygen @ref and @see in .c and .h files; held until their audit */ \ - X(CBM_DOCLINK_C_REF, CBM_LANG_C, "doxygen_ref", false, false, "MENTIONS") \ - X(CBM_DOCLINK_C_SEE, CBM_LANG_C, "see", false, false, "MENTIONS") \ + /* php (doclink_php.c): PHPDoc @see and {@see}; held-out audit passed */ \ + X(CBM_DOCLINK_PHP_SEE, CBM_LANG_PHP, "see", false, true, "MENTIONS") \ + /* c (doclink_c.c): Doxygen @ref and @see in .c and .h files; held-out audit passed */ \ + X(CBM_DOCLINK_C_REF, CBM_LANG_C, "doxygen_ref", false, true, "MENTIONS") \ + X(CBM_DOCLINK_C_SEE, CBM_LANG_C, "see", false, true, "MENTIONS") \ /* markdown (doclink_md.c): what a document's text names explicitly */ \ X(CBM_DOCLINK_MD_LINK, CBM_LANG_MARKDOWN, "link", false, true, "MENTIONS") \ X(CBM_DOCLINK_MD_PATH, CBM_LANG_MARKDOWN, "path", false, true, "MENTIONS") \ diff --git a/tests/test_doc_links_md.c b/tests/test_doc_links_md.c index 2b66e63bd..21e74f37e 100644 --- a/tests/test_doc_links_md.c +++ b/tests/test_doc_links_md.c @@ -602,6 +602,18 @@ TEST(doc_links_md_ship_gate) { ASSERT_FALSE(cbm_doclink_syntax_ships(DM_HELD_FAMILIES[i])); } ASSERT_TRUE(cbm_doclink_syntax_ships(CBM_DOCLINK_MD_LINK)); + /* the doc-comment families whose held-out tiers passed (2026-10-11) */ + static const int passed[] = { + CBM_DOCLINK_GO_DOC_LINK, CBM_DOCLINK_JAVA_LINK, CBM_DOCLINK_JAVA_LINKPLAIN, + CBM_DOCLINK_JAVA_SEE, CBM_DOCLINK_JAVA_THROWS, CBM_DOCLINK_JAVA_VALUE, + CBM_DOCLINK_KT_KDOC_LINK, CBM_DOCLINK_KT_SEE, CBM_DOCLINK_KT_THROWS, + CBM_DOCLINK_KT_SAMPLE, CBM_DOCLINK_RS_INTRA_DOC, CBM_DOCLINK_RS_LINK_DEF, + CBM_DOCLINK_PERL_POD_L, CBM_DOCLINK_TS_LINK, CBM_DOCLINK_PHP_SEE, + CBM_DOCLINK_C_REF, CBM_DOCLINK_C_SEE, + }; + for (size_t i = 0; i < sizeof(passed) / sizeof(passed[0]); i++) { + ASSERT_TRUE(cbm_doclink_syntax_ships(passed[i])); + } ASSERT_TRUE(cbm_doclink_syntax_ships(CBM_DOCLINK_RST_ROLE)); ASSERT_TRUE(cbm_doclink_syntax_ships(CBM_DOCLINK_ADOC_INCLUDE)); ASSERT_TRUE(cbm_doclink_syntax_ships(CBM_DOCLINK_RST_LITERALINCLUDE)); diff --git a/tests/test_doc_mentions_go.c b/tests/test_doc_mentions_go.c index f3d9b58b5..1f81030db 100644 --- a/tests/test_doc_mentions_go.c +++ b/tests/test_doc_mentions_go.c @@ -176,11 +176,11 @@ TEST(doc_mentions_go_tokens) { const CBMDocLinkFamily *f = cbm_doclink_family(CBM_DOCLINK_GO_DOC_LINK); ASSERT_NOT_NULL(f); ASSERT_EQ(f->lang, CBM_LANG_GO); - /* held in production until its held-out audit passes; the suite ships it */ + /* ships in production: its held-out audit passed */ cbm_doclink_test_reset_ships(); bool ships_in_production = cbm_doclink_syntax_ships(CBM_DOCLINK_GO_DOC_LINK); dm_ship_held_families(); - ASSERT_FALSE(ships_in_production); + ASSERT_TRUE(ships_in_production); ASSERT_TRUE(cbm_doclink_syntax_ships(CBM_DOCLINK_GO_DOC_LINK)); PASS(); } @@ -1748,6 +1748,7 @@ TEST(doc_mentions_go_ship_gate) { char db[512]; cbm_doclink_test_set_ships(CBM_DOCLINK_GO_DOC_LINK, false); int rc = go_index_lookup(tmp, sizeof(tmp), db, sizeof(db)); + cbm_doclink_test_reset_ships(); /* production gates again for the tests after this one */ dm_ship_held_families(); ASSERT_EQ(rc, 0); /* a resolved doc link is a row with the gate's reason and no edge */ diff --git a/tests/test_doc_mentions_helpers.h b/tests/test_doc_mentions_helpers.h index 778796393..7fc75e1ad 100644 --- a/tests/test_doc_mentions_helpers.h +++ b/tests/test_doc_mentions_helpers.h @@ -430,40 +430,12 @@ static inline int dm_reason_lines(const char *db, const char *block) { * so it is the resolution an audit would ship; doc_links_md_ship_gate tests * the production gate itself. */ static const int DM_HELD_FAMILIES[] = { - CBM_DOCLINK_MD_BARE_PATH, - CBM_DOCLINK_MD_SUPERSEDES_PROSE, - CBM_DOCLINK_RST_INCLUDE, - CBM_DOCLINK_RST_KERNEL_DOC, - CBM_DOCLINK_ADOC_ATTRIBUTE, - CBM_DOCLINK_ADOC_CODE_PATH, - CBM_DOCLINK_ADOC_CODE_NAME, - CBM_DOCLINK_GO_DOC_LINK, - CBM_DOCLINK_JAVA_LINK, - CBM_DOCLINK_JAVA_LINKPLAIN, - CBM_DOCLINK_JAVA_SEE, - CBM_DOCLINK_JAVA_THROWS, - CBM_DOCLINK_JAVA_VALUE, - CBM_DOCLINK_KT_KDOC_LINK, - CBM_DOCLINK_KT_SEE, - CBM_DOCLINK_KT_THROWS, - CBM_DOCLINK_KT_SAMPLE, - CBM_DOCLINK_RS_INTRA_DOC, - CBM_DOCLINK_RS_LINK_DEF, - CBM_DOCLINK_PERL_POD_L, - CBM_DOCLINK_TS_LINK, - CBM_DOCLINK_TS_LINKCODE, - CBM_DOCLINK_TS_SEE, - CBM_DOCLINK_PHP_SEE, - CBM_DOCLINK_C_REF, - CBM_DOCLINK_C_SEE, - CBM_DOCLINK_DG_MERMAID_CLICK, - CBM_DOCLINK_DG_PLANTUML_URL, - CBM_DOCLINK_DG_PLANTUML_INCLUDE, - CBM_DOCLINK_DG_DRAWIO_LINK, - CBM_DOCLINK_DG_EXCALIDRAW_LINK, - CBM_DOCLINK_DG_DOT_URL, - CBM_DOCLINK_DG_D2_LINK, - CBM_DOCLINK_OPENAPI_OPERATION_ID, + CBM_DOCLINK_MD_BARE_PATH, CBM_DOCLINK_MD_SUPERSEDES_PROSE, CBM_DOCLINK_RST_INCLUDE, + CBM_DOCLINK_RST_KERNEL_DOC, CBM_DOCLINK_ADOC_ATTRIBUTE, CBM_DOCLINK_ADOC_CODE_PATH, + CBM_DOCLINK_ADOC_CODE_NAME, CBM_DOCLINK_TS_LINKCODE, CBM_DOCLINK_TS_SEE, + CBM_DOCLINK_DG_MERMAID_CLICK, CBM_DOCLINK_DG_PLANTUML_URL, CBM_DOCLINK_DG_PLANTUML_INCLUDE, + CBM_DOCLINK_DG_DRAWIO_LINK, CBM_DOCLINK_DG_EXCALIDRAW_LINK, CBM_DOCLINK_DG_DOT_URL, + CBM_DOCLINK_DG_D2_LINK, CBM_DOCLINK_OPENAPI_OPERATION_ID, }; static inline void dm_ship_held_families(void) { diff --git a/tests/test_doc_mentions_jvm.c b/tests/test_doc_mentions_jvm.c index 31602e942..baa713001 100644 --- a/tests/test_doc_mentions_jvm.c +++ b/tests/test_doc_mentions_jvm.c @@ -2556,14 +2556,14 @@ TEST(doc_mentions_jvm_family_names) { ASSERT_STR_EQ(cbm_doclink_syntax_name(families[i].syntax), families[i].name); ASSERT_FALSE(cbm_doclink_syntax_is_external(families[i].syntax)); } - /* held in production until their held-out audit passes; the suite ships them */ + /* all ship in production: their held-out audit passed */ cbm_doclink_test_reset_ships(); int shipping_in_production = 0; for (size_t i = 0; i < sizeof(families) / sizeof(families[0]); i++) { shipping_in_production += cbm_doclink_syntax_ships(families[i].syntax) ? 1 : 0; } dm_ship_held_families(); - ASSERT_EQ(shipping_in_production, 0); + ASSERT_EQ(shipping_in_production, (int)(sizeof(families) / sizeof(families[0]))); for (size_t i = 0; i < sizeof(families) / sizeof(families[0]); i++) { ASSERT_TRUE(cbm_doclink_syntax_ships(families[i].syntax)); } From 72b8176ffec7a0ae06a5a76c1cc3112a87974414 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 11 Oct 2026 15:46:00 +0200 Subject: [PATCH 16/17] feat(docs): a diagram section's candidates carry its diagram family's p A Markdown section that embeds a diagram is about its candidate functions more often than the Markdown curve says, by how much depends on the diagram's kind. 553 candidates from such sections in 16 held-out repositories, judged blind in three rounds: 0.20-0.30 0.30-0.40 0.40+ local extras prose 0.24 0.28 0.45 0.275 structural 0.32 0.46 0.57 0.78 (9 items) Structural is Mermaid sequence/class/ER/C4/architecture and PlantUML C4/class/sequence/use-case/component; prose any other Mermaid or PlantUML kind. A section takes the highest family of the Diagram nodes it defines; its function candidates at or inside the doc's home and its local extras carry the family's value, and the first band is now stored. Outside functions, files, folders, other formats and sections without a diagram keep the Markdown curve. On kubernetes' apiserver folder the sequence-diagram section's candidates move from 0.22/0.51 to 0.46/0.57, the flowchart sections gain five first-band candidates at 0.24, other sections and the graph are unchanged. Signed-off-by: Martin Vogel --- src/pipeline/pass_semantic_edges.c | 46 +++++++-- src/pipeline/pipeline_internal.h | 8 +- src/semantic/semantic.c | 66 +++++++++++++ src/semantic/semantic.h | 22 +++++ tests/test_pipeline.c | 149 +++++++++++++++++++++++++++-- tests/test_semantic.c | 82 ++++++++++++++++ 6 files changed, 356 insertions(+), 17 deletions(-) diff --git a/src/pipeline/pass_semantic_edges.c b/src/pipeline/pass_semantic_edges.c index 30b2c4954..e5822d809 100644 --- a/src/pipeline/pass_semantic_edges.c +++ b/src/pipeline/pass_semantic_edges.c @@ -2904,12 +2904,39 @@ typedef struct { int n; } doc_rows_t; +/* The diagram family of a section: the highest of the Diagram nodes it + * DEFINES (cbm_sem_diag_family), CBM_SEM_DIAG_NONE without one. */ +static cbm_sem_diag_family_t doc_section_family(const cbm_gbuf_t *gbuf, + const cbm_gbuf_node_t *sec) { + const cbm_gbuf_edge_t **edges = NULL; + int n = 0; + cbm_sem_diag_family_t fam = CBM_SEM_DIAG_NONE; + if (cbm_gbuf_find_edges_by_source_type(gbuf, sec->id, "DEFINES", &edges, &n) != 0) { + return fam; + } + for (int i = 0; i < n; i++) { + const cbm_gbuf_node_t *t = cbm_gbuf_find_by_id(gbuf, edges[i]->target_id); + if (!t || !t->label || strcmp(t->label, "Diagram") != 0 || !t->properties_json) { + continue; + } + char fmt[CBM_SZ_32]; + char kind[CBM_SZ_32]; + const char *f = json_str_value(t->properties_json, "format", fmt, sizeof(fmt)); + const char *k = json_str_value(t->properties_json, "diagram_kind", kind, sizeof(kind)); + cbm_sem_diag_family_t df = cbm_sem_diag_family(f, k); + if (df > fam) { + fam = df; + } + } + return fam; +} + /* Append the candidate (section -> target) when its judged p is above 0 and * the graph does not link the pair exactly; false: out of memory (the row at * rw->n may hold partial strings). */ static bool doc_row_add(doc_rows_t *rw, const cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *sec, - const cbm_gbuf_node_t *target, const doc_hit_t *hit, - cbm_sem_doc_kind_t kind, const char *home) { + cbm_sem_diag_family_t fam, const cbm_gbuf_node_t *target, + const doc_hit_t *hit, cbm_sem_doc_kind_t kind, const char *home) { if (!target || !target->qualified_name || !sec->qualified_name) { return true; } @@ -2917,7 +2944,7 @@ static bool doc_row_add(doc_rows_t *rw, const cbm_gbuf_t *gbuf, const cbm_gbuf_n cbm_sem_doc_format_t fmt = cbm_sem_doc_format(sec->file_path); float p = kind == CBM_SEM_DOC_KIND_FOLDER ? cbm_sem_doc_folder_p(fmt, sec->file_path) - : cbm_sem_doc_calibrated_p(fmt, kind, pos, hit->tfidf); + : cbm_sem_doc_section_p(fmt, fam, kind, pos, hit->tfidf); if (p <= 0.0F || doc_pair_linked(gbuf, sec->id, target->id)) { return true; /* not stored for this band, kind or format; or linked exactly */ } @@ -2944,25 +2971,26 @@ static bool doc_section_rows(doc_rows_t *rw, const cbm_gbuf_t *gbuf, const doc_c const doc_place_t *pl, int s) { const cbm_gbuf_node_t *sec = dc->secs[s]; const doc_result_t *r = &dc->res[s]; + cbm_sem_diag_family_t fam = doc_section_family(gbuf, sec); bool ok = true; for (int h = 0; ok && h < r->n + r->nlocal; h++) { const doc_hit_t *hit = h < r->n ? &r->hits[h] : &r->local[h - r->n]; if (h < r->n && (hit->rank > CBM_SEM_DOC_TOP_K || hit->tfidf < CBM_SEM_DOC_MIN_SCORE)) { continue; } - ok = - doc_row_add(rw, gbuf, sec, cbm_gbuf_find_by_id(gbuf, dc->funcs[hit->func].node_id), hit, - h < r->n ? CBM_SEM_DOC_KIND_FUNCTION : CBM_SEM_DOC_KIND_LOCAL, pl->home[s]); + ok = doc_row_add( + rw, gbuf, sec, fam, cbm_gbuf_find_by_id(gbuf, dc->funcs[hit->func].node_id), hit, + h < r->n ? CBM_SEM_DOC_KIND_FUNCTION : CBM_SEM_DOC_KIND_LOCAL, pl->home[s]); } for (int h = 0; ok && h < r->nfiles; h++) { const doc_file_hit_t *fh = &r->files[h]; doc_hit_t hit = {.tfidf = fh->tfidf, .rank = fh->rank, .shared = fh->shared}; - ok = doc_row_add(rw, gbuf, sec, dc->files->node[fh->file], &hit, CBM_SEM_DOC_KIND_FILE, + ok = doc_row_add(rw, gbuf, sec, fam, dc->files->node[fh->file], &hit, CBM_SEM_DOC_KIND_FILE, pl->home[s]); } doc_hit_t first = {.rank = SKIP_ONE}; - return ok && doc_row_add(rw, gbuf, sec, doc_home_folder(pl, s), &first, CBM_SEM_DOC_KIND_FOLDER, - pl->home[s]); + return ok && doc_row_add(rw, gbuf, sec, fam, doc_home_folder(pl, s), &first, + CBM_SEM_DOC_KIND_FOLDER, pl->home[s]); } /* The stored candidates of every section -- its best CBM_SEM_DOC_TOP_K diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index b40ea682b..da925d2ab 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -1282,7 +1282,13 @@ int cbm_pipeline_build_fresh_semantic_manifest(cbm_pipeline_t *p, const char *pr * properties are read with their escapes, admission is best-first and * every edge carries "p"; doc_link_candidates holds doc section -> code * candidates. An index built before has the old pairs and no - * candidates, so it rebuilds once on upgrade. */ + * candidates, so it rebuilds once on upgrade. Released together with + * 7 (no release in between, so no number of its own): doc comments of + * Go, Java, Kotlin, Rust, Perl, TS/JS, PHP and C hold MENTIONS edges; + * diagrams are Diagram/DiagramElement nodes with DIAGRAM_ARROW edges; + * OpenAPI operations are Routes; a Markdown section's text keeps a + * diagram fence's labels instead of its syntax, and its candidates + * carry the p of its diagrams' kind family. */ enum { CBM_SEMANTIC_INDEX_VERSION = 7 }; typedef struct { diff --git a/src/semantic/semantic.c b/src/semantic/semantic.c index af738151e..704994ede 100644 --- a/src/semantic/semantic.c +++ b/src/semantic/semantic.c @@ -1973,6 +1973,72 @@ float cbm_sem_doc_folder_p(cbm_sem_doc_format_t fmt, const char *doc_path) { return readme ? 0.867F : 0.0F; } +cbm_sem_diag_family_t cbm_sem_diag_family(const char *format, const char *diagram_kind) { + static const char *const mermaid[] = {"sequence", "class", "er", + "c4_context", "c4_container", "c4_component", + "c4_dynamic", "c4_deployment", "architecture"}; + static const char *const plantuml[] = {"c4", "class", "sequence", "usecase", "component"}; + if (!format || !format[0]) { + return CBM_SEM_DIAG_NONE; + } + const char *const *structural = NULL; + size_t n = 0; + if (strcmp(format, "mermaid") == 0) { + structural = mermaid; + n = sizeof(mermaid) / sizeof(mermaid[0]); + } else if (strcmp(format, "plantuml") == 0) { + structural = plantuml; + n = sizeof(plantuml) / sizeof(plantuml[0]); + } else { + return CBM_SEM_DIAG_GRAPH; + } + for (size_t i = 0; diagram_kind && i < n; i++) { + if (strcmp(diagram_kind, structural[i]) == 0) { + return CBM_SEM_DIAG_STRUCTURAL; + } + } + return CBM_SEM_DIAG_PROSE; +} + +/* Markdown sections that embed a diagram, by the section's family (DIAG-C, + * 2026-10-11: 553 blind-judged candidates from sections of 16 held-out + * repositories that embed a diagram, three reading rounds; function + * candidates at or inside the home pooled, as both curves rose by band). + * A diagram section is about its candidates more often than SEM7's Markdown + * curve says, so its first band is stored too: + * prose 0.24 / 0.28 / 0.45, local extras 11 of 40 = 0.275 + * structural 0.32 / 0.46 / 0.57, local extras 7 of 9 = 0.78 (every + * local extra of the family; 95% interval 0.45-0.94) + * Graph diagrams (DOT, D2) were not found in Markdown sections: SEM7. */ +static const struct { + bool fitted; + float function[SEM_DOC_P_NBANDS]; + float local; +} SEM_DOC_P_MD_DIAGRAM[CBM_SEM_DIAG_COUNT] = { + [CBM_SEM_DIAG_PROSE] = {true, {0.0F, 0.24F, 0.28F, 0.45F}, 0.275F}, + [CBM_SEM_DIAG_STRUCTURAL] = {true, {0.0F, 0.32F, 0.464F, 0.567F}, 0.778F}, +}; + +float cbm_sem_doc_section_p(cbm_sem_doc_format_t fmt, cbm_sem_diag_family_t fam, + cbm_sem_doc_kind_t kind, cbm_sem_doc_pos_t pos, float score) { + bool fitted = fmt == CBM_SEM_DOC_FMT_MARKDOWN && (int)fam > 0 && fam < CBM_SEM_DIAG_COUNT && + SEM_DOC_P_MD_DIAGRAM[fam].fitted; + bool home = pos == CBM_SEM_DOC_POS_GLOBAL || pos == CBM_SEM_DOC_POS_LOCAL; + if (!fitted || !((kind == CBM_SEM_DOC_KIND_FUNCTION && home) || + (kind == CBM_SEM_DOC_KIND_LOCAL && pos == CBM_SEM_DOC_POS_LOCAL))) { + return cbm_sem_doc_calibrated_p(fmt, kind, pos, score); + } + size_t band = 0; + while (band < SEM_DOC_P_NBANDS - 1 && score >= SEM_DOC_P_CUTS[band]) { + band++; + } + if (band == 0) { + return 0.0F; /* below CBM_SEM_DOC_MIN_SCORE: never measured, nothing stored */ + } + return kind == CBM_SEM_DOC_KIND_LOCAL ? SEM_DOC_P_MD_DIAGRAM[fam].local + : SEM_DOC_P_MD_DIAGRAM[fam].function[band]; +} + float cbm_sem_combined_score(const cbm_sem_func_t *a, const cbm_sem_func_t *b, const cbm_sem_config_t *cfg) { if (!a || !b || !cfg) { diff --git a/src/semantic/semantic.h b/src/semantic/semantic.h index 66f4cc0a1..9f40c1041 100644 --- a/src/semantic/semantic.h +++ b/src/semantic/semantic.h @@ -319,6 +319,28 @@ float cbm_sem_doc_calibrated_p(cbm_sem_doc_format_t fmt, cbm_sem_doc_kind_t kind * stored. */ float cbm_sem_doc_folder_p(cbm_sem_doc_format_t fmt, const char *doc_path); +/* The kind family of a diagram a section embeds, by its format and + * diagram_kind: structural (Mermaid sequence, class, ER, C4 and architecture; + * PlantUML C4, class, sequence, use-case and component), prose (any other + * Mermaid or PlantUML diagram) or graph (any other format). A section's family + * is the highest any of its diagrams has, in this order. */ +typedef enum { + CBM_SEM_DIAG_NONE = 0, + CBM_SEM_DIAG_GRAPH, + CBM_SEM_DIAG_PROSE, + CBM_SEM_DIAG_STRUCTURAL, + CBM_SEM_DIAG_COUNT, +} cbm_sem_diag_family_t; +cbm_sem_diag_family_t cbm_sem_diag_family(const char *format, const char *diagram_kind); + +/* cbm_sem_doc_calibrated_p for a section that embeds diagrams of family fam: + * a Markdown section of a family with its own judged curve takes that curve + * for function candidates at or inside the home and for local extras; every + * other candidate, and every section of another family, as + * cbm_sem_doc_calibrated_p. */ +float cbm_sem_doc_section_p(cbm_sem_doc_format_t fmt, cbm_sem_diag_family_t fam, + cbm_sem_doc_kind_t kind, cbm_sem_doc_pos_t pos, float score); + /* p as stored and shown: two decimals, half up (0.225 -> 0.23, 0.325 -> * 0.33; the epsilon absorbs the float representation of a judged x.xx5). */ double cbm_sem_p_2dp(float p); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index ddf7bc5a5..7684868e4 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -18948,11 +18948,12 @@ TEST(pipeline_semantic_edges_carry_p) { PASS(); } -/* The p a stored candidate row must carry: the curve of the row's kind and - * position (read from its evidence) at its score, two decimals; -1 for an - * unknown kind or position. *is_function: a function candidate (top 5). */ -static double doc_row_want(const char *evidence, cbm_sem_doc_format_t fmt, float score, - bool *is_function) { +/* The p a stored candidate row of a section embedding diagrams of family fam + * must carry: the curve of the row's kind and position (read from its + * evidence) at its score, two decimals; -1 for an unknown kind or position. + * *is_function: a function candidate (top 5). */ +static double doc_row_want_family(const char *evidence, cbm_sem_doc_format_t fmt, + cbm_sem_diag_family_t fam, float score, bool *is_function) { static const struct { const char *name; cbm_sem_doc_kind_t kind; @@ -18973,8 +18974,14 @@ static double doc_row_want(const char *evidence, cbm_sem_doc_format_t fmt, float } *is_function = k == 0; return k < 0 || p < 0 ? -1.0 - : cbm_sem_p_2dp(cbm_sem_doc_calibrated_p(fmt, KINDS[k].kind, - POSITIONS[p].pos, score)); + : cbm_sem_p_2dp(cbm_sem_doc_section_p(fmt, fam, KINDS[k].kind, + POSITIONS[p].pos, score)); +} + +/* doc_row_want_family for a section without a diagram. */ +static double doc_row_want(const char *evidence, cbm_sem_doc_format_t fmt, float score, + bool *is_function) { + return doc_row_want_family(evidence, fmt, CBM_SEM_DIAG_NONE, score, is_function); } /* Count a section's stored doc -> code candidates (section QN containing @@ -19219,6 +19226,133 @@ TEST(pipeline_doc_candidates_skip_exact_links_and_use_format_curves) { PASS(); } +/* The rows of a section (its QN contains heading): their count; *bad the + * rows off the curve of the section's diagram family fam; *own the rows + * whose p differs from a section without a diagram. */ +static int doc_family_rows(cbm_store_t *s, const char *heading, cbm_sem_diag_family_t fam, int *bad, + int *own) { + sqlite3_stmt *st = NULL; + *bad = 0; + *own = 0; + if (sqlite3_prepare_v2(cbm_store_get_db(s), + "SELECT score, p, evidence FROM doc_link_candidates " + "WHERE instr(section_qn, ?1) > 0", + -1, &st, NULL) != SQLITE_OK) { + return -1; + } + sqlite3_bind_text(st, 1, heading, -1, SQLITE_TRANSIENT); + int n = 0; + while (sqlite3_step(st) == SQLITE_ROW) { + float score = (float)sqlite3_column_double(st, 0); + double p = sqlite3_column_double(st, 1); + const char *ev = (const char *)sqlite3_column_text(st, 2); + bool is_function = false; + double want = doc_row_want_family(ev, CBM_SEM_DOC_FMT_MARKDOWN, fam, score, &is_function); + *bad += want <= 0.0 || p != want; + *own += want != doc_row_want(ev, CBM_SEM_DOC_FMT_MARKDOWN, score, &is_function); + n++; + } + sqlite3_finalize(st); + return n; +} + +/* The diagram_kind of the Diagram a section (its QN contains heading) + * DEFINES, "" without one. */ +static void doc_section_diagram_kind(cbm_store_t *s, const char *heading, char *out, + size_t out_sz) { + sqlite3_stmt *st = NULL; + out[0] = '\0'; + if (sqlite3_prepare_v2(cbm_store_get_db(s), + "SELECT json_extract(d.properties, '$.diagram_kind') FROM edges e " + "JOIN nodes a ON a.id = e.source_id JOIN nodes d ON d.id = e.target_id " + "WHERE e.type = 'DEFINES' AND a.label = 'Section' " + "AND d.label = 'Diagram' AND instr(a.qualified_name, ?1) > 0", + -1, &st, NULL) != SQLITE_OK) { + return; + } + sqlite3_bind_text(st, 1, heading, -1, SQLITE_TRANSIENT); + if (sqlite3_step(st) == SQLITE_ROW && sqlite3_column_text(st, 0)) { + snprintf(out, out_sz, "%s", (const char *)sqlite3_column_text(st, 0)); + } + sqlite3_finalize(st); +} + +/* A Markdown section that embeds a diagram takes its diagram family's judged + * curve (DIAG-C): a sequence diagram is structural, a flowchart is prose; a + * section without a diagram keeps the Markdown curve. Some rows of each + * diagram section must carry a p the Markdown curve would not give. */ +TEST(pipeline_doc_candidates_diagram_family_curves) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_docdiag_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + static const char *const subjects[] = {"mail", "cache", "queue", "user", "server", "client"}; + for (size_t i = 0; i < sizeof(subjects) / sizeof(subjects[0]); i++) { + write_py_family(tmp, subjects[i]); + } + char docs[512]; + snprintf(docs, sizeof(docs), "%s/docs", tmp); + ASSERT_TRUE(cbm_mkdir_p(docs, 0755)); + write_temp_file( + tmp, "docs/mail.md", + "# Mail loading\n\n" + "Load the mail config from disk and validate every mail field before the " + "parsed mail settings are returned; missing mail fields fail the validation.\n\n" + "```mermaid\nsequenceDiagram\n" + " participant Loader\n participant Validator\n" + " Loader->>Validator: validate mail config\n```\n\n" + "# Mail writing\n\n" + "Write the mail config back to disk after validating it, so the written " + "mail config file always carries every mail field it needs.\n\n" + "```mermaid\nflowchart TD\n" + " validate[validate mail config] --> write[write mail config]\n```\n\n" + "# Mail parsing\n\n" + "Parse the mail config file and return the parsed mail settings; the mail " + "config is loaded from disk and every mail field is validated first.\n"); + char db[512]; + snprintf(db, sizeof(db), "%s/docdiag.db", tmp); + cbm_pipeline_incremental_test_reset_faults(); + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + cbm_pipeline_free(p); + + cbm_store_t *s = cbm_store_open_path(db); + ASSERT_NOT_NULL(s); + char seq_kind[32]; + char flow_kind[32]; + char none_kind[32]; + doc_section_diagram_kind(s, "Mail-loading", seq_kind, sizeof(seq_kind)); + doc_section_diagram_kind(s, "Mail-writing", flow_kind, sizeof(flow_kind)); + doc_section_diagram_kind(s, "Mail-parsing", none_kind, sizeof(none_kind)); + int seq_bad = 0; + int seq_own = 0; + int flow_bad = 0; + int flow_own = 0; + int none_bad = 0; + int none_own = 0; + int seq = doc_family_rows(s, "Mail-loading", CBM_SEM_DIAG_STRUCTURAL, &seq_bad, &seq_own); + int flow = doc_family_rows(s, "Mail-writing", CBM_SEM_DIAG_PROSE, &flow_bad, &flow_own); + int none = doc_family_rows(s, "Mail-parsing", CBM_SEM_DIAG_NONE, &none_bad, &none_own); + cbm_store_close(s); + th_rmtree(tmp); + cbm_pipeline_incremental_test_reset_faults(); + ASSERT_STR_EQ(seq_kind, "sequence"); + ASSERT_STR_EQ(flow_kind, "flowchart"); + ASSERT_STR_EQ(none_kind, ""); + ASSERT_EQ(cbm_sem_diag_family("mermaid", seq_kind), CBM_SEM_DIAG_STRUCTURAL); + ASSERT_EQ(cbm_sem_diag_family("mermaid", flow_kind), CBM_SEM_DIAG_PROSE); + ASSERT_GT(seq, 0); + ASSERT_EQ(seq_bad, 0); + ASSERT_GT(seq_own, 0); + ASSERT_GT(flow, 0); + ASSERT_EQ(flow_bad, 0); + ASSERT_GT(flow_own, 0); + ASSERT_GT(none, 0); + ASSERT_EQ(none_bad, 0); + ASSERT_EQ(none_own, 0); + PASS(); +} + /* Lines of a CBM_SEM_DOC_SIGNALS dump whose section file, kind, position, * home and target file equal the given values (NULL: any). */ static int doc_signal_count(const char *dump, const char *sec_file, const char *kind, @@ -20312,6 +20446,7 @@ SUITE(pipeline) { RUN_TEST(pipeline_doc_candidates_published_and_kept_by_delta); RUN_TEST(pipeline_doc_candidates_skip_exact_links_and_use_format_curves); RUN_TEST(pipeline_doc_candidates_home_position_and_kinds); + RUN_TEST(pipeline_doc_candidates_diagram_family_curves); RUN_TEST(pipeline_cpp_static_factory_pointer_receiver_issue1153); RUN_TEST(pipeline_cpp_static_factory_reference_receiver_issue1153); RUN_TEST(pipeline_cpp_static_factory_unique_ptr_receiver_issue1153); diff --git a/tests/test_semantic.c b/tests/test_semantic.c index f1ebd09f7..654c1b536 100644 --- a/tests/test_semantic.c +++ b/tests/test_semantic.c @@ -611,6 +611,86 @@ TEST(sem_doc_calibrated_p_bands) { PASS(); } +/* A section's diagram family (DIAG-C): Mermaid and PlantUML structural kinds, + * any other Mermaid or PlantUML kind is prose, any other format a graph. */ +TEST(sem_diag_family_by_format_and_kind) { + ASSERT_EQ(cbm_sem_diag_family("mermaid", "sequence"), CBM_SEM_DIAG_STRUCTURAL); + ASSERT_EQ(cbm_sem_diag_family("mermaid", "class"), CBM_SEM_DIAG_STRUCTURAL); + ASSERT_EQ(cbm_sem_diag_family("mermaid", "er"), CBM_SEM_DIAG_STRUCTURAL); + ASSERT_EQ(cbm_sem_diag_family("mermaid", "c4_deployment"), CBM_SEM_DIAG_STRUCTURAL); + ASSERT_EQ(cbm_sem_diag_family("mermaid", "architecture"), CBM_SEM_DIAG_STRUCTURAL); + ASSERT_EQ(cbm_sem_diag_family("mermaid", "flowchart"), CBM_SEM_DIAG_PROSE); + ASSERT_EQ(cbm_sem_diag_family("mermaid", "state"), CBM_SEM_DIAG_PROSE); + ASSERT_EQ(cbm_sem_diag_family("mermaid", "usecase"), CBM_SEM_DIAG_PROSE); + ASSERT_EQ(cbm_sem_diag_family("mermaid", NULL), CBM_SEM_DIAG_PROSE); + ASSERT_EQ(cbm_sem_diag_family("plantuml", "usecase"), CBM_SEM_DIAG_STRUCTURAL); + ASSERT_EQ(cbm_sem_diag_family("plantuml", "component"), CBM_SEM_DIAG_STRUCTURAL); + ASSERT_EQ(cbm_sem_diag_family("plantuml", "c4"), CBM_SEM_DIAG_STRUCTURAL); + ASSERT_EQ(cbm_sem_diag_family("plantuml", "activity"), CBM_SEM_DIAG_PROSE); + ASSERT_EQ(cbm_sem_diag_family("plantuml", "er"), CBM_SEM_DIAG_PROSE); + ASSERT_EQ(cbm_sem_diag_family("dot", "digraph"), CBM_SEM_DIAG_GRAPH); + ASSERT_EQ(cbm_sem_diag_family("d2", "d2"), CBM_SEM_DIAG_GRAPH); + ASSERT_EQ(cbm_sem_diag_family("", "sequence"), CBM_SEM_DIAG_NONE); + ASSERT_EQ(cbm_sem_diag_family(NULL, NULL), CBM_SEM_DIAG_NONE); + PASS(); +} + +/* A Markdown section that embeds a diagram (DIAG-C): function candidates at + * or inside the home and local extras carry the family's judged curve, from + * the first band; outside functions, files, other formats, graph diagrams + * and sections without a diagram keep cbm_sem_doc_calibrated_p. */ +TEST(sem_doc_section_p_diagram_families) { + const cbm_sem_doc_format_t md = CBM_SEM_DOC_FMT_MARKDOWN; + const cbm_sem_doc_kind_t fn = CBM_SEM_DOC_KIND_FUNCTION; + const cbm_sem_doc_kind_t lx = CBM_SEM_DOC_KIND_LOCAL; + const cbm_sem_doc_pos_t global = CBM_SEM_DOC_POS_GLOBAL; + const cbm_sem_doc_pos_t local = CBM_SEM_DOC_POS_LOCAL; + const cbm_sem_doc_pos_t outside = CBM_SEM_DOC_POS_OUTSIDE; + const cbm_sem_diag_family_t prose = CBM_SEM_DIAG_PROSE; + const cbm_sem_diag_family_t st = CBM_SEM_DIAG_STRUCTURAL; + ASSERT_FLOAT_EQ(cbm_sem_doc_section_p(md, prose, fn, global, 0.199F), 0.0F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_doc_section_p(md, prose, fn, global, 0.20F), 0.24F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_doc_section_p(md, prose, fn, local, 0.30F), 0.28F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_doc_section_p(md, prose, fn, global, 0.40F), 0.45F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_doc_section_p(md, prose, lx, local, 0.20F), 0.275F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_doc_section_p(md, st, fn, local, 0.25F), 0.32F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_doc_section_p(md, st, fn, global, 0.35F), 0.464F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_doc_section_p(md, st, fn, local, 0.90F), 0.567F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_doc_section_p(md, st, lx, local, 0.50F), 0.778F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_doc_section_p(md, st, lx, local, 0.199F), 0.0F, 0.0); + /* stored as two decimals: 0.275 -> 0.28, 0.464 -> 0.46, 0.567 -> 0.57 */ + ASSERT_FLOAT_EQ(cbm_sem_p_2dp(cbm_sem_doc_section_p(md, prose, lx, local, 0.5F)), 0.28, 1e-12); + ASSERT_FLOAT_EQ(cbm_sem_p_2dp(cbm_sem_doc_section_p(md, st, fn, local, 0.35F)), 0.46, 1e-12); + static const cbm_sem_doc_kind_t KINDS[] = {CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_KIND_LOCAL, + CBM_SEM_DOC_KIND_FILE}; + for (int fam = 0; fam < CBM_SEM_DIAG_COUNT; fam++) { + for (size_t k = 0; k < sizeof(KINDS) / sizeof(KINDS[0]); k++) { + for (int pos = 0; pos < CBM_SEM_DOC_POS_COUNT; pos++) { + float last = 0.0F; + for (int milli = 0; milli <= 1000; milli++) { + float s = (float)milli / 1000.0F; + cbm_sem_doc_pos_t ps = (cbm_sem_doc_pos_t)pos; + float p = + cbm_sem_doc_section_p(md, (cbm_sem_diag_family_t)fam, KINDS[k], ps, s); + ASSERT_TRUE(p >= last); + last = p; + bool own = (fam == prose || fam == st) && ((KINDS[k] == fn && pos != outside) || + (KINDS[k] == lx && pos == local)); + if (!own) { + ASSERT_FLOAT_EQ(p, cbm_sem_doc_calibrated_p(md, KINDS[k], ps, s), 0.0); + } + ASSERT_FLOAT_EQ( + cbm_sem_doc_section_p(CBM_SEM_DOC_FMT_RST, (cbm_sem_diag_family_t)fam, + KINDS[k], ps, s), + cbm_sem_doc_calibrated_p(CBM_SEM_DOC_FMT_RST, KINDS[k], ps, s), 0.0); + } + } + } + } + ASSERT_FLOAT_EQ(cbm_sem_doc_section_p(md, CBM_SEM_DIAG_COUNT, fn, global, 0.25F), 0.0F, 0.0); + PASS(); +} + /* p is stored and shown at two decimals, half up, whatever the float * representation of the judged value (0.225F is 0.22499999...). */ TEST(sem_p_two_decimals_half_up) { @@ -841,6 +921,8 @@ SUITE(semantic) { RUN_TEST(sem_admit_best_first_keeps_the_best); RUN_TEST(sem_calibrated_p_bands); RUN_TEST(sem_doc_calibrated_p_bands); + RUN_TEST(sem_diag_family_by_format_and_kind); + RUN_TEST(sem_doc_section_p_diagram_families); RUN_TEST(sem_doc_format_by_extension); RUN_TEST(sem_p_two_decimals_half_up); RUN_TEST(sem_json_str_honours_escapes); From 65d365ef13da280572691dea6e98be90bb039358 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 11 Oct 2026 16:46:52 +0200 Subject: [PATCH 17/17] test(doc-links jvm): a variable named far does not compile on Windows The Windows headers define far and near as empty macros, so 'const char *far = ...' became 'const char * = ...' on the CLANG64 leg. The variable is far_src now. Signed-off-by: Martin Vogel --- tests/test_doc_mentions_jvm.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_doc_mentions_jvm.c b/tests/test_doc_mentions_jvm.c index baa713001..3fa1bc459 100644 --- a/tests/test_doc_mentions_jvm.c +++ b/tests/test_doc_mentions_jvm.c @@ -3520,8 +3520,8 @@ TEST(doc_mentions_jvm_java_field_outer_outside) { TEST(doc_mentions_jvm_java_field_own_far_outside) { jt_repo_t r; ASSERT(jt_begin(&r, "jfown")); - const char *far = "src/main/java/gb/Far.java"; - jt_write(&r, far, + const char *far_src = "src/main/java/gb/Far.java"; + jt_write(&r, far_src, "package gb;\n" "public class Far extends java.io.InputStream {\n" " public int read() { return 0; }\n" @@ -3542,7 +3542,7 @@ TEST(doc_mentions_jvm_java_field_own_far_outside) { "}\n"); ASSERT_EQ(dm_index(r.repo, r.db, NULL), 0); const char *db = r.db; - ASSERT_EQ(jt_rows(db, far, "#level", "external"), 1); + ASSERT_EQ(jt_rows(db, far_src, "#level", "external"), 1); ASSERT_EQ(dm_mentions_from(db, "gb.Far.Near.m"), 0); /* decoy: only the type's own supertype leaves the repository */ ASSERT_EQ(jt_edge(db, "gb.Far2.Near2.m", "gb.Far2.Near2.level"), 1);