diff --git a/.sisyphus/plans/2026-09-08-issue-165-169-column-analysis-surface.md b/.sisyphus/plans/2026-09-08-issue-165-169-column-analysis-surface.md new file mode 100644 index 0000000..889078b --- /dev/null +++ b/.sisyphus/plans/2026-09-08-issue-165-169-column-analysis-surface.md @@ -0,0 +1,475 @@ +# Issue #165–169 列级分析查询面与解析增强（columns / predicates / transform / 跨表键 / 文档） + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** 打通「列级分析 → 造数/mock 机器可读入口」主线，覆盖五个 issue： +1. **#169** — 函数包裹列的字面量过滤纳入 `HardFilter`（substr/nvl/trim 白名单 + `transform` 字段）； +2. **方案A（用户已拍板）** — `merge_table_access_edges` 合并全部诊断字段（当前只合并 `column_mappings`/`read_tables`，其余「保留第一条」，导致同过程多语句同表时 hard_filters/join_conditions 丢失）+ `STORE_VERSION` 9→10； +3. **#165 P0** — `codeweb columns --procedure X --format json` 按过程聚合导出 ColumnAnalysis； +4. **#168** — WHERE/JOIN ON/SELECT INTO 中 `%ROWTYPE` 记录字段解析为跨表等值键； +5. **#165 P1** — MCP `codeweb_column_analysis`/`codeweb_lineage` + HTTP `GET /api/v1/columns`/`GET /api/v1/lineage`； +6. **#167** — PL IF/CASE 条件解析为表列谓词（置信度分级 + param_table_hint）； +7. **#166** — 文档补齐（lineage CLI、ColumnAnalysis 字段、新命令）。 + +**Architecture:** 全部改动在单 crate 内，无新外部依赖、无新 feature flag： +- **解析层** `src/parser/extractor.rs`：T1 白名单 transform、T4 记录字段等值键、T6 新谓词提取 pass； +- **图构建层** `src/graph/builder.rs`：T2 合并诊断字段（`merge_table_access_edges` L3330-3421）； +- **查询层** `src/graph/`（新增聚合函数）+ `src/main.rs`（新 CLI 子命令）+ `src/mcp/tools.rs` + `src/server/handlers.rs`； +- **存储** `src/graph/store.rs`：`STORE_VERSION` 9→10（T2，含 D4 合并 bump）；T6 再 bump 至 11（D6 已锁定存储方案）。 + +**Tech Stack:** Rust stable、ogsql-parser v0.10.0（git 依赖，checkout `~/.cargo/git/checkouts/ogsql-parser-9b270b8f87a071f2/28b5b4b`）、现有测试 harness（extractor.rs `#[cfg(test)]` 单测 + `tests/regress_*.rs` 端到端 + `tests/serve_api.rs` + `tests/mcp_test.rs`）。 + +--- + +## 决策记录（全部锁定：D1 用户拍板；D2–D6 依用户委托由 Momus 审核裁决） + +> **裁决说明**：Momus 第一轮审核（2026-09-08）确认 D2–D6 实质方向无异议、要求正式锁定以免实施阻塞。以下裁决即为最终结论，Task 4/6 按此执行，不再保留「建议」状态。 + +| # | 问题 | 选项 | 裁决与理由 | +|---|---|---|---| +| **D1** | #165 store 合并丢数据 | A: 合并诊断字段+bump / B: 接受少报 | **✅ 用户已拍板：方案A** | +| **D2** | #168 `JoinConditionSource` | 新增 `RecordField` 变体 / 复用 `ImplicitWhere` | **✅ 锁定：新增 `RecordField` 变体**。store 文件兼容由版本门禁隔离（旧二进制读不了新 store，无枚举反序列化问题）；JSON export 是单向输出，codeweb 自己不回读；唯一消费者 fastaas 是共建中的新代码，可同步适配。语义价值：下游需区分「隐式等值 JOIN」与「记录字段推导键」（置信度不同） | +| **D3** | #167 输出形态 | 独立 `codeweb predicates` / 并入 `columns` JSON | **✅ 锁定：独立 `codeweb predicates` 命令**，schema 复用 `FilterOperator`/`FilterValue`（issue 允许）。`columns` 保持聚焦列约束面；两 issue 解耦交付，TDD 分层清晰。MCP/HTTP 的 predicates 入口**暂缓**（issue 验收未强制，YAGNI） | +| **D4** | #169 是否 bump 版本 | 单独 bump / 与 T2 合并一次 | **✅ 锁定：与 T2 合并为一次 v9→10**。`transform` 是 serde-default 新字段，技术上无需 bump，但按 v7→v8 先例（加 `read_tables` 即 bump）+ 借 `store_is_current()` 促使用户重跑 analyze | +| **D5** | #169 白名单边界 | 仅逗号语法 FunctionCall / 双变体；白名单集合 | **✅ 锁定：同时处理 `FunctionCall` + `SpecialFunction`**（ogsql 文档明言 dual-variant：`SUBSTR(x FROM 1 FOR 2)` 走 SpecialFunction，只处理逗号语法会留下「换写法就漏」的坑）；白名单 **{substr, substring, nvl, trim, upper, lower}**（lower 与 upper 对称，成本≈0）。封闭白名单，不开放任意函数 | +| **D6** | #167 谓词存哪 | (a) analyze 期存入 GraphStore（`procedure_predicates` 侧表，serde default，bump v11）/ (b) 查询期重解析源文件 | **✅ 锁定：(a) 存储方案**。PL IF/CASE 分支结构在 `extract_body_sql` 摊平后即丢失，查询期重解析依赖源文件未变，脆弱且与「分析结果进 store」的既有架构一致。代价是多一次 bump（v11） | + +--- + +## 关键代码位置（当前实现，改动点） + +`src/parser/extractor.rs`： + +```rust +// L1930 — HardFilter（T1 加 transform 字段） +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct HardFilter { + pub table: Option, + pub column: String, + pub operator: FilterOperator, + pub value: FilterValue, +} + +// L2355-2505 — process_expr_for_joins_and_filters（T1 六个比较分支加白名单 arm；T4 等值分支加记录字段解析） +"=" => { + if let (Some(l), Some(r)) = (as_column_ref(left), as_column_ref(right)) { /* equi-join L2370 */ } + else if let Some(col) = as_column_ref(left) { /* col = literal → HardFilter L2385 */ } + else if let Some(col) = as_column_ref(right) { /* literal = col → HardFilter L2389 */ } + // ← FunctionCall/SpecialFunction 包裹列目前三条路全不匹配，静默丢弃 +} + +// L2561 — add_hard_filter（table 只经 resolve_alias 解析，无 transform） +// L3149 — column_source()（T4 复用其记录字段解析规则：精确 output_name 匹配 → 游标源列； +// catch-all（SELECT */动态SQL）→ 游标锚表+字段名；表锚定 %ROWTYPE → 锚表+字段名） +// L3149 所在 impl 已持有 record_cursors / cursor_sources（ProcedureVarContext，L2000） +``` + +`src/graph/builder.rs`： + +```rust +// L3330-3421 — merge_table_access_edges（T2：除 column_mappings/read_tables 外， +// 其余诊断字段 join_conditions/hard_filters/enum_mappings/select_into/ +// insert_columns/update_columns/column_refs/alias_map 当前「保留第一条」，改为集合并集去重） +``` + +`src/graph/store.rs`：L22 `STORE_VERSION: u32 = 9`（T2 → 10；T6 若走存储方案 → 11）。 +`src/graph/lineage.rs`：L1480 `mappings_of_routine`（T3 聚合函数的范本——HashSet 去重、扫入边+出边）。 +`src/main.rs`：L379-411 `Lineage` variant（T3/T6 新子命令的克隆范本）；L1558-1565 v7 软提示范本；L148-179 `ImpactResult`（`schema_version` 字段房屋风格）。 +`src/mcp/tools.rs`：L124-532 六工具注册（`#[tool(description=...)]`）；L539-549 `tool_handler` instructions。 +`src/server/handlers.rs`：L24-41 `router()`；L435-479 `trace` handler（Query-struct GET 范本）。 +`tests/mcp_test.rs`：L185-199 `test_mcp_tools_list` 硬编码 6 工具名，加工具必改。 + +**AST 事实（ogsql-parser v0.10.0，已核实）**： +- `Expr::FunctionCall { name: ObjectName, args: Vec, ... }`（ast/mod.rs:1221，逗号语法）； +- `Expr::SpecialFunction { name, args, ... }`（ast/mod.rs:1384，关键字语法——`SUBSTRING(x FROM 1 FOR 3)`、`TRIM(LEADING ... FROM ...)`）。文档要求 dual-variant 处理； +- `PlIfStmt { condition: Expr, then_stmts, elsifs: Vec, else_stmts }`（ast/plpgsql.rs:237）、`PlCaseStmt { expression, whens: Vec, else_stmts }`（L251）；`walk_pl_statement` 自动递归条件+分支体； +- WHERE 表达式：`Expr::BinaryOp{left,op:String,right}`、`Between`、`InList`、`Like`、`Case`、`FieldAccess{object,field}`（L1302）、`PlVariable`（L1415）； +- 记录字段在 SQL 中解析为多段 `ColumnRef`（`r.security_id` → 2 Idents），`split_alias_column`（extractor.rs L3872）已按此形状处理。 + +**测试基础设施（现有）**：`column_mappings_of(sql)` 等 helper（extractor.rs tests，L4924 起）；`ColumnAccessExtractor::new_with_context(&ProcedureVarContext)`（L2078，单测接缝）；`tests/regress_column_lineage.rs` 的 `project_with_sql` + `lineage()` harness；`run_codeweb_in`（tests/regress_lineage_table_upstream.rs L28）。**注意**：`par_sys_purchase`/`r_get_purchase`/STEP3 样例仓内不存在，T3/T4/T6 需自建 fixture。 + +--- + +## Task 1 (T2): 方案A — merge_table_access_edges 合并全部诊断字段 + STORE_VERSION 10 + +**Files:** +- Modify: `src/graph/builder.rs`（`merge_table_access_edges` L3330-3421） +- Modify: `src/graph/store.rs`（L22 `STORE_VERSION` 9→10；版本注释） +- Modify: `src/parser/extractor.rs`（若 `JoinCondition`/`HardFilter`/`EnumMapping`/`SelectIntoMapping`/`InsertColumnInfo`/`UpdateColumnInfo`/`ColumnRef` 缺 `Hash`，补 derive——所有字段均为 String/枚举/Vec，可哈希） +- Test: `src/graph/builder.rs` `#[cfg(test)]`（若无测试模块则在 store.rs 或新建 `tests/regress_column_analysis_merge.rs`） + +**Step 1: 写失败测试（Red）** + +单测：同一过程两条语句写同一张表、各带不同 `hard_filters` 与 `join_conditions`，经 builder 构建后该 `(proc, table)` 边的 `column_analysis` 应为并集： + +```rust +/// 方案A (issue #165): merged TableAccess edges must UNION diagnostic fields, +/// not keep only the first edge's. Two statements → same proc/table pair with +/// distinct hard filters must both survive. +#[test] +fn merge_table_access_unions_hard_filters_and_joins() { + // 构建：CREATE TABLE t(a NUMBER, b NUMBER); CREATE PROCEDURE p AS BEGIN + // INSERT INTO t SELECT x.a FROM s x WHERE x.a = 1; + // INSERT INTO t SELECT y.b FROM s y JOIN u z ON y.id = z.id WHERE y.b = 2; + // END; + // 断言：该 proc→t 边 column_analysis.hard_filters 同时含 a=1 与 b=2； + // join_conditions 含 s.id = u.id；column_mappings 仍正确去重。 +} +``` + +（实现时按 builder 现有测试范式落位；若 builder 无 `#[cfg(test)]`，用 `tests/regress_column_analysis_merge.rs` 端到端 + `export --format json` 断言。） + +**Step 2: 运行确认失败** + +Run: `cargo test --features full merge_table_access_unions_hard_filters_and_joins` +Expected: FAIL — 只有第一条语句的 hard_filters 幸存。 + +**Step 3: 最小实现（Green）** + +- 为上述类型补 `Hash` derive（`FilterValue::Float(String)` 可哈希，无 f64 阻碍）； +- `merge_table_access_edges`：仿照 `column_mappings` 的 HashSet 去重模式，对 `join_conditions`、`hard_filters`、`enum_mappings`、`select_into`、`insert_columns`、`update_columns`、`column_refs` 做集合并集；`alias_map` 做 BTreeMap extend（同 key 首见优先）；删除/改写「remaining diagnostic fields keep the first」注释（L3377-3379）； +- 读边（`AccessMode::Write` 不含）继续清空 `column_mappings` 的既有行为不变； +- `STORE_VERSION` 9→10，更新邻近注释（v10 = merge 诊断字段并集 + HardFilter.transform 预留，关联 #165/#169）。 + +**Step 4: 验证** + +Run: `cargo test --features full` + `cargo clippy --features full -- -D warnings` + `cargo fmt --all -- --check` +Expected: 新测试绿；store.rs 版本拒绝测试（`load_bincode_rejects_previous_layout_version` L2403、`load_bincode_rejects_pre_issue_159_version` L2425）依旧绿（它们写旧版本文件断言被拒，不受新版本号影响）；既有 full 套件除已知环境跳过项（`test_path_mapping_applied`、`test_serve_*`）外全绿。 + +--- + +## Task 2 (T1): #169 — 函数包裹列的字面量过滤纳入 HardFilter（白名单 + transform） + +**Files:** +- Modify: `src/parser/extractor.rs`（`HardFilter` L1930 加字段；新 struct `FilterTransform`；新 helper `column_transform_of`；`process_expr_for_joins_and_filters` 六个比较分支各加 arm；新 `add_hard_filter_with_transform`） +- Test: `src/parser/extractor.rs` tests 模块（filter 测试群 L4814-5038 旁） + +**Step 1: 写失败测试（Red）** + +```rust +/// #169: a whitelisted pure column transform compared against a literal yields a +/// HardFilter on the underlying column, with a transform descriptor. +#[test] +fn substr_wrapped_column_literal_becomes_hard_filter_with_transform() { + // WHERE substr(qs.stock_kind, 1, 2) = '05' (qs 为表别名) + // 断言：hard_filters 含 { table: Some(..), column: "stock_kind", Eq, String("05"), + // transform: Some(FilterTransform { fn_: "substr", args: [Integer(1), Integer(2)] }) } +} + +/// #169: the STEP3 mixed-cursor case — transformed and plain filters coexist. +#[test] +fn step3_cursor_mixed_filters_all_captured() { + // WHERE substr(qs.stock_kind,1,2)='05' AND qs.stock_kind <> '0509' + // AND qs.scdm = '001' AND qs.cjsl > 0 + // 断言：4 条 HardFilter，第一条带 transform，后三条 transform == None +} + +/// #169: non-literal extra args exclude the filter (PL variable in args). +#[test] +fn substr_with_variable_length_arg_is_excluded() { + // WHERE substr(col, 1, v_len) = '05' → 不产出 +} + +/// #169: non-whitelisted function or func-vs-func comparisons stay excluded. +#[test] +fn non_whitelisted_or_double_sided_function_is_excluded() { + // WHERE fnc_x(col) = '1' → 不产出；WHERE nvl(a,1) = nvl(b,2) → 不产出 +} + +/// #169: SpecialFunction (keyword syntax) is covered too. +#[test] +fn substr_keyword_syntax_produces_transform() { + // WHERE substring(col FROM 1 FOR 2) = '05' → 产出（D5 双变体） +} +``` + +**Step 2: 运行确认失败** + +Run: `cargo test --features full substr_wrapped_column_literal_becomes_hard_filter_with_transform step3_cursor_mixed_filters_all_captured` +Expected: FAIL — 现在什么都不产出。 + +**Step 3: 最小实现（Green）** + +```rust +/// #169: descriptor of a whitelisted pure column transform in a filter. +/// Serialized as {"fn": "substr", "args": [1, 2]} per issue schema. +#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +pub struct FilterTransform { + #[serde(rename = "fn")] + pub fn_name: String, // 小写规范化 + pub args: Vec, // 除目标列外的全部实参（均为字面量） +} +``` + +- `HardFilter` 增加 `#[serde(default, skip_serializing_if = "Option::is_none")] pub transform: Option`（满足验收「JSON 无 transform 或 null」；旧 store 反序列化得 None）； +- helper `column_transform_of(expr) -> Option<(&[Ident] /*列*/, FilterTransform)>`： + - 匹配 `Expr::FunctionCall` 与 `Expr::SpecialFunction`（D5 双变体），name 小写 ∈ {substr, substring, nvl, trim, upper, lower}； + - args 中恰好一个 `Expr::ColumnRef`，其余全部 `literal_to_filter_value` 成功（PL 变量→None→自动排除，天然满足「substr(col,1,v_len) 不产出」）； + - `substring` 与 `substr` 归一化为 `"substr"`； +- 六个比较分支（`=` `<>` `!=` `>` `>=` `<` `<=`）在 col-vs-literal 判断后各加对称 arm：一侧 `column_transform_of` 命中且另一侧 `literal_to_filter_value` 命中 → `add_hard_filter_with_transform`； +- `Like/Between/InList/IsNull` 侧不处理函数包裹（范围外）； +- 既有 `add_hard_filter` 保持签名，内部 `transform: None`（10 个调用点零改动）。 + +**Step 4: 验证** + +Run: `cargo test --features full` （重点回归 `test_join_with_alias_and_hard_filter` L4814、`test_pl_variable_not_hard_filter` L4895）+ clippy + fmt。 +Expected: 新旧全绿；既有 `col='x'` filter 的 `transform` 序列化后不出现（skip_serializing_if）。 + +--- + +## Task 3 (T3): #165 P0 — `codeweb columns` CLI（按过程聚合 ColumnAnalysis） + +**Files:** +- Add: `src/graph/columns.rs`（聚合函数 `pub fn column_analysis_of_routine(...) -> AggregatedColumnAnalysis`；模块注册 `src/graph/mod.rs`） +- Modify: `src/main.rs`（`Commands::Columns` variant + dispatch + `cmd_columns`；旧 store 软提示） +- Test: 新增 `tests/regress_columns.rs`（harness 仿 `regress_column_lineage.rs` 的 `project_with_sql`）+ graph 层单测 + +**Step 1: 写失败测试（Red）** + +```rust +/// #165: per-procedure column analysis export aggregates all TableAccess edges. +#[test] +fn columns_json_lists_hard_filters_and_joins_without_duplicates() { + // fixture（仿 STEP3 驱动游标 + 维表）： + // CREATE TABLE mid_yjqs_detail(...); CREATE TABLE par_fund_partner(...); + // CREATE PROCEDURE prc_trd_hz_byfund AS BEGIN + // -- 两条语句写同一张输出表，各带不同 hard_filter / join_condition + // INSERT INTO mid_yjqs_detail SELECT f.partner_no FROM par_fund_partner f + // WHERE f.fund_code = c.fund_code AND c.scdm = '001' ...; + // END; + // 断言 `codeweb columns --procedure prc_trd_hz_byfund --format json`： + // - schema_version == 1；procedure/package 字段正确 + // - hard_filters 含 scdm='001'；join_conditions 含 par_fund_partner.fund_code ↔ ... + // - 同一 filter/join 不重复出现（多边聚合去重） +} + +/// #165: --table narrows to one table's constraints. +#[test] +fn columns_json_table_filter_narrows_output() { /* --table mid_yjqs_detail 只出该表相关 */ } + +/// #165: unknown procedure → clear error, exit != 0. +#[test] +fn columns_unknown_procedure_errors_cleanly() { /* 不静默空数组 */ } +``` + +graph 层单测：聚合函数对合成边去重（两条边各含相同 `scdm='001'` → 只出现一次）。 + +**Step 2: 运行确认失败** + +Run: `cargo test --features full --test regress_columns` +Expected: FAIL — 子命令不存在（编译失败即为合法 Red）。 + +**Step 3: 最小实现（Green）** + +- `AggregatedColumnAnalysis`（serde struct，首字段 `schema_version: u32 = 1`，房屋风格仿 `ImpactResult` main.rs:148-179）： + `{ schema_version, procedure, package: Option, tables: Vec, join_conditions, hard_filters, select_into, enum_mappings, column_mappings, insert_columns, update_columns, read_tables }`——字段名与 `ColumnAnalysis` 1:1（issue 要求「不要再包一层展示用树」）； +- `column_analysis_of_routine`：仿 `mappings_of_routine`（lineage.rs:1480）——扫该 routine 节点入边+出边的 `Edge::TableAccess.column_analysis`，逐字段 HashSet 去重；`read_tables` 合并；`--table` 过滤在聚合层做（保留与目标表相关的边；join/filter 若涉及其它表仍保留——语句级隔离需要 read_tables）； +- CLI：`Commands::Columns { #[arg(long)] procedure: Option, #[arg(long)] package: Option, #[arg(long)] table: Option, #[arg(long, default_value="json", value_parser=["json"])] format: String, #[arg(short, long, default_value=".")] project: PathBuf }`；procedure/package 二选一必填（clap `group.required = true` + `conflicts_with`）； +- 旧 store 软提示：`store.version < 10` → `eprintln!("note: store version {} predates full column-analysis diagnostics (v10) — run `codeweb analyze` to rebuild.", ...)`（仿 main.rs:1560 范本）； +- 过程定位复用 `store.resolve_single_node(name, MatchMode::Substring, ...)` + 校验 Procedure/Function 节点（仿 cmd_lineage L1670-1696）。 + +**Step 4: 验证** + +Run: `cargo test --features full --test regress_columns` + 全套门禁。README/user-guide 文档在 T7 统一补。 + +--- + +## Task 4 (T4): #168 — WHERE/JOIN ON 记录字段解析为跨表等值键 + +**Files:** +- Modify: `src/parser/extractor.rs`（新 helper `resolve_record_field(&self, names) -> Option<(String, String)>` 复用 `column_source` 的三段规则；`extract_join_condition` 增加记录字段对侧路径；`JoinConditionSource` **新增 `RecordField` 变体【D2 已锁定】**，serde 序列化为 `"RecordField"`） +- Test: extractor.rs tests + `tests/regress_column_lineage.rs`（新 e2e） + +**Step 1: 写失败测试（Red）** + +```rust +/// #168: record field on one side of an equi-comparison resolves to the cursor's +/// source column, producing a cross-table JoinCondition. +#[test] +fn record_field_in_where_resolves_to_cross_table_join() { + // 上下文：CURSOR c_get_data IS SELECT security_id, fund_code FROM mid_yjqs_detail ...; + // r_get_purchase c_get_data%ROWTYPE; + // SQL: SELECT t.purchase_days INTO v_purchase_days FROM par_sys_purchase t + // WHERE t.security_id = r_get_purchase.security_id + // 断言：join_conditions 含 par_sys_purchase.security_id ↔ mid_yjqs_detail.security_id， + // source == RecordField【D2 已锁定】 +} + +/// #168: plain equi-joins regress unchanged. +#[test] +fn plain_on_equi_join_unchanged() { /* ON a.id = b.id → ImplicitWhere/ExplicitOn 如旧 */ } + +/// #168: record-vs-procedure-param and unregistered records produce nothing. +#[test] +fn record_vs_param_or_unregistered_produces_no_join() { + // WHERE r.col = p_i_date（参数侧）→ 不产出；未注册记录变量 → 不产出（不猜表名） +} + +/// #168: table-anchored %ROWTYPE and SELECT * cursor catch-all follow #142 rules. +#[test] +fn table_anchored_rowtype_and_star_cursor_resolve() { /* 两种锚定形态各一断言 */ } +``` + +e2e：`tests/regress_column_lineage.rs` 新增「STEP3 维表 JOIN」用例（fixture 自建 `par_sys_purchase` 风格）。 + +**Step 2: 运行确认失败** → **Step 3: 最小实现（Green）** + +- `resolve_record_field`：抽取 `column_source`（L3149）中「记录字段 → 游标源列」分支为独立函数（精确 output_name 匹配 → `(source_table, source_col)`；catch-all → `(cursor锚表, 字段名)`；表锚定 → `(锚表, 字段名)`），`column_source` 改为调用它（消除重复，Refactor 步骤内聚）； +- `process_expr_for_joins_and_filters` 的 `=` 分支：两侧 `as_column_ref` 双成功 → 现路径；**一侧列、一侧记录字段** → `extract_record_field_join`，产出 `JoinCondition { left/right 表列, source: RecordField }`【D2 已锁定】；去重逻辑复用现有反向查重（L2550-2555）； +- 记录字段一侧同时 `add_column_ref(..., JoinCondition)`（与现路径对齐）； +- `p_i_date` 参数经 `record_cursors` 查不到 → None → 不产出（负例免费）。 + +**Step 4: 验证**：全套门禁；`test_join_with_alias_and_hard_filter` 等既有 join 单测全绿。 + +--- + +## Task 5 (T5): #165 P1 — MCP `codeweb_column_analysis`/`codeweb_lineage` + HTTP `/api/v1/columns`/`/lineage` + +**Files:** +- Modify: `src/mcp/tools.rs`（两个新 `#[tool]` 方法 + 参数结构；`tool_handler` instructions 补两句）；`tests/mcp_test.rs`（tools list 断言 6→8） +- Modify: `src/server/handlers.rs`（router 两条 route + 两个 handler，仿 `trace` L435-479） +- Modify: `docs/serve-api-guide.md`、README 两表（亦可留 T7，此处至少改代码侧） +- 共享后端：T3 的 `graph::columns::column_analysis_of_routine` 与 lineage 既有函数，三个面共用同一 serde 结构，**不另发明 schema** + +**Step 1: 写失败测试（Red）** + +- `tests/mcp_test.rs`：`test_mcp_tools_list` 改为断言 8 个工具名（含 `codeweb_column_analysis`、`codeweb_lineage`）；新增 `test_mcp_call_column_analysis`（仿 `test_mcp_call_stats`，断言返回 JSON 与 CLI `columns --format json` 字段一致）； +- `tests/serve_api.rs`：`test_serve_columns_endpoint`、`test_serve_lineage_endpoint`（启动 serve、请求 `/api/v1/columns?procedure=...`、断言 200 + JSON 字段；404 场景）。 + +**Step 2: 运行确认失败** → **Step 3: 最小实现（Green）** + +- MCP `codeweb_column_analysis`：`ColumnAnalysisParams { procedure: Option, package: Option, table: Option }`；空图守卫复用 `graph_empty()`；返回 T3 同一 JSON 字符串； +- MCP `codeweb_lineage`：`LineageParams { target: String, direction: Option, depth: Option }`；复用 lineage_table/lineage_column + `format_lineage_json`/`format_column_lineage_json`，direction 缺省 both（与 CLI 一致）； +- HTTP `GET /api/v1/columns`：`ColumnsQuery { procedure: Option, package: Option, table: Option }`；`GET /api/v1/lineage`：`LineageQuery { target, direction: Option, depth: Option }`；错误约定与现有一致（缺参/未命中 → 400/404，无 envelope）； +- instructions 字符串（tools.rs:539）追加两工具用途说明。 + +**Step 4: 验证**：`cargo test --features full`（含 serve/mcp 集成测试；CI 跳过项除外）+ clippy + fmt。 + +--- + +## Task 6 (T6): #167 — PL IF/CASE 条件解析为表列谓词 + +**Files:** +- Add: `src/parser/predicates.rs`（`PredicateExtractor`：branch-aware Visitor pass + 谓词 AST） +- Modify: `src/graph/builder.rs`（过程构建期调用新 pass，产出挂入 store）；`src/graph/store.rs`（**加 `procedure_predicates` 侧表 + bump v11【D6 已锁定：存储方案】**） +- Modify: `src/main.rs`（`Commands::Predicates` + `cmd_predicates`，**独立命令【D3 已锁定】**） +- Test: `src/parser/predicates.rs` tests + `tests/regress_predicates.rs` + +**设计要点（D3/D6 均已锁定，直接按此实施）**： + +- 新 AST（全部 serde，schema 复用 `FilterOperator`/`FilterValue`）： + `PlPredicate { id: String /* B001… */, line: usize, origin: String, kind: PredicateKind(If|CaseWhen), confidence: Confidence(High|Medium|Low), table_predicate: Option, needs_review: Option, param_table_hint: Option }`； + `TablePredicate { table, clauses: Vec }`； + `ParamTableHint { table, filters: Vec, set: Vec<(String, FilterValue)> }`； +- pass 形态仿 `CallExtractor` 的 PL 走树（L441-799 证可行）：`impl Visitor for PredicateExtractor`，拦截 `PlStatement::If`/`Case`（读 `condition`/`whens[].condition`），条件表达式经「条件→clauses 转换器」解析——该转换器**复用 T1 的 `column_transform_of` + T4 的 `resolve_record_field` + `ProcedureVarContext`**； +- 置信度规则（issue 表格逐条落地，单测各锁一条）： + | 模式 | confidence | + |---|---| + | `r.field` 且 `record_cursors` 命中，比较字面量 | high | + | 裸列且 `scope_sole_table` 唯一 | high | + | `SELECT col INTO v` 后 `IF v = literal`，col 来自主表 | medium（主表谓词） | + | 同上但 col 来自维表 | low + `param_table_hint` | + | 函数调用/动态 SQL/GOTO | low / skip，保留 `origin` | +- 过程内 `SELECT INTO` 变量源追踪：pass 内自建 `HashMap`（走 `PlStatement::SqlStatement` 的 into_targets + targets，游标源解析复用 `ProcedureVarContext`）； +- IF 分支下语句归属：`then_stmts`/`else_stmts` 递归时携带当前条件上下文（分支内语句不重复产出谓词，谓词只来自条件本身）。 + +**Step 1: 写失败测试（Red）** + +```rust +/// #167: STEP3 star_market IF resolves to high-confidence table predicate. +#[test] +fn star_market_if_resolves_high_confidence() { + // IF r_get_data.stock_kind = '0100' AND r_get_data.zqdm BETWEEN '609100' AND '609999' + // → predicate { confidence: High, table: mid_yjqs_detail, + // clauses: [stock_kind eq '0100', zqdm between [609100,609999]] } +} + +/// #167: SELECT-INTO-derived var yields low confidence + param_table_hint. +#[test] +fn select_into_var_condition_yields_param_table_hint() { + // SELECT kind_id INTO v_kind FROM swh_all_kind WHERE operation_kind='COMMISSION_SWITCH'; + // IF v_kind = '1' → low + hint{ swh_all_kind, filters:[operation_kind eq ...], set:{kind_id:'1'} } + // 断言：不误写成主表谓词 +} + +/// #167: cursor WHERE hard filters do NOT leak into the IF predicate list. +#[test] +fn cursor_hard_filters_not_in_predicates() { /* 游标 WHERE 的 HardFilter 不出现在 predicates */ } + +/// #167: function-call conditions keep origin, low/skip confidence. +#[test] +fn function_condition_degrades_confidence() { /* IF fnc_x(a) = 1 → low + origin 保留 */ } +``` + +**Step 2: 运行确认失败** → **Step 3: 最小实现（Green）** → **Step 4: 验证** + +- CLI：`codeweb predicates --procedure X --format json`，输出 `{ schema_version: 1, procedure, predicates: [...] }`； +- store 增加 `procedure_predicates: HashMap>`（`#[serde(default)]`）【D6 已锁定】，`STORE_VERSION` → 11，`cmd_predicates` 直接读 store；旧 store < 11 软提示重跑 analyze； +- 全套门禁。 + +--- + +## Task 7 (T7): #166 — 文档补齐 + +**Files（纯文档，无代码）:** +- `README.md`（中英两份表格）：CLI 表加 `lineage`、`columns`、`predicates`；HTTP 表加 `/columns`、`/lineage`；MCP 工具表加两个新工具 +- `docs/user-guide.md`：§6 新增 `lineage` 子节（table vs table.column、--direction/--view/--flow-only、store v7+ 提示、与 trace 的区别）+ `columns`/`predicates` 子节 +- `docs/DeveloperGuide.md`：`ColumnAnalysis` 字段表（join/hard_filter/select_into/mapping kind/transform）+ 消费场景（mock 造数）+ MCP/HTTP 表更新 +- `docs/getting-started.md` + `_zh`：10 行 INSERT..SELECT 的 `codeweb lineage t_out.amt --direction upstream` 示例 +- `docs/serve-api-guide.md`：`/columns`、`/lineage` 端点文档（若 T5 未覆盖） + +**Step 1: 可执行 QA 场景（文档的「失败测试」——先跑通核对清单再动笔，列出当前缺失项）** + +```bash +# QA-1 README 命令表与 --help 一致性（中英两份表都要核对） +codeweb --help +# 预期缺失（写文档前应确认 grep 全部落空， documenting 后应 ≥2：英文表 + 中文表各一行）： +grep -c '| `codeweb lineage' README.md # 现在 0 → 目标 ≥ 2 +grep -c '| `codeweb columns' README.md # 现在 0 → 目标 ≥ 2 +grep -c '| `codeweb predicates' README.md # 现在 0 → 目标 ≥ 2 + +# QA-2 user-guide 出现可照跑的小节（写前 0 命中，写后各 ≥1 个 §6.x 标题） +grep -n '^#\{2,3\} .*lineage' docs/user-guide.md +grep -n '^#\{2,3\} .*columns' docs/user-guide.md +grep -n '^#\{2,3\} .*predicates' docs/user-guide.md + +# QA-3 serve-api-guide 端点存在且字段与实际输出一致 +grep -n 'api/v1/columns\|api/v1/lineage' docs/serve-api-guide.md # 目标 ≥ 1 处/端点 +# 字段一致性核对：文档响应示例顶层键 == 实际输出顶层键（对 T3 fixture 项目执行） +codeweb columns --procedure prc_trd_hz_byfund --format json | jq -S 'keys' +codeweb serve & curl -s 'http://127.0.0.1:3000/api/v1/columns?procedure=prc_trd_hz_byfund' | jq -S 'keys' +# 两次 jq keys 输出必须相同，且与 serve-api-guide 文档示例逐键一致 + +# QA-4 DeveloperGuide ColumnAnalysis 字段说明 +grep -n 'ColumnAnalysis' docs/DeveloperGuide.md # 目标：字段表出现（含 transform 行） +grep -n 'codeweb_column_analysis\|codeweb_lineage' docs/DeveloperGuide.md # MCP 表 8 工具 + +# QA-5 getting-started 示例可照跑（10 行 INSERT..SELECT fixture） +# 按文档步骤在 /tmp 临时项目逐字执行，预期输出含： +codeweb lineage t_out.amt --direction upstream +# → 树中出现源列 t_src.amt（或 fixture 对应源表列），非 "No column lineage" +``` + +**Step 2: 依清单撰写/修订文档**（上面每条 grep 由 0 → 目标值；QA-3/QA-5 的实际命令输出与文档示例逐字一致） + +**验收（照 issue #166 + Momus 要求的可执行核对）**：QA-1~QA-5 全部通过；`codeweb --help` 的每个子命令在 README 两份 CLI 表各有且仅有一行；serve-api-guide 响应示例键集与 `jq keys` 实测一致。 + +--- + +## 执行顺序与门禁 + +``` +T2（方案A合并+bump v10） → T1（#169 transform） → T3（#165 P0 CLI） +→ T4（#168 跨表键） → T5（#165 P1 MCP/HTTP） → T6（#167 谓词，bump v11【D6 已锁定】） +→ T7（#166 文档） → 全量门禁 +``` + +每个 Task 独立 Red→Green→Refactor 循环，完成即跑： +```bash +cargo test --features full -- --skip test_path_mapping_applied --skip test_serve_ +cargo clippy --features full -- -D warnings +cargo fmt --all -- --check +``` +最终门禁另跑 `cargo build --features full` + `cargo test --features full`。 + +**Never 红线（AGENTS.md）**：不删/跳过/改写人类已有测试断言；`test_join_with_alias_and_hard_filter`、`test_pl_variable_not_hard_filter`、`test_mcp_tools_list`（改 6→8 属新增工具的必要同步，在汇报中显式说明）、store 版本拒绝测试为只读基线；每个行为先有失败测试；不引入新依赖/feature flag。 diff --git a/.sisyphus/plans/2026-09-08-pr170-review-fixes.md b/.sisyphus/plans/2026-09-08-pr170-review-fixes.md new file mode 100644 index 0000000..0fc9c47 --- /dev/null +++ b/.sisyphus/plans/2026-09-08-pr170-review-fixes.md @@ -0,0 +1,105 @@ +# PR #170 评审修复计划（#165–#169 跟随修复） + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** 修复 PR #170 六条评审意见（4 bug + 2 suggestion，全部已对照代码核实成立）。核心目标：#167 谓词在 ELSIF/简单 CASE/函数包裹裸列/SELECT INTO 变量形态下不漏报不错报；`columns` 与 `predicates` 的过程身份可 join；机器入口不静默错配。 + +**Architecture:** 全部改动位于 `src/parser/predicates.rs`、`src/main.rs`、`src/mcp/tools.rs`、`src/server/handlers.rs`、`src/graph/lineage.rs`、`src/parser/extractor.rs`（仅注释）。无 store 布局变更——**不需要 bump `STORE_VERSION`（保持 12）**：F1/F2 改的是提取逻辑而非已序列化结构；F3 只改 JSON 输出字段来源；F4/F5 是解析参数与错误语义。 + +**已核实的评审发现（全部接受，无争议项）：** + +| # | 发现 | 核实位置 | +|---|---|---| +| F1 | If 臂漏 `elsifs`；简单 CASE（`expression: Some`）把 WHEN 值当裸条件 | predicates.rs:277-287 | +| F2 | `condition_operand` 的 `expr_name(expr)?` 对 FunctionCall 断链，跳过 var_sources 与 sole-table fallback；Derived 臂硬编码 `transform: None` | predicates.rs:402, 409 | +| F3 | `cmd_predicates` 的 `procedure` 取自 NodeKey 展示串（包内过程得 `pkg.prc`），与 columns 的 `id.name`+`package` 不一致；无 `package` 字段 | main.rs:2026-2029 | +| F4 | 四处新调用点 `resolve_single_node(..., false, false)` → `Ambiguous` 臂不可达，多匹配静默取首个 | main.rs:1894/1990, tools.rs:715, handlers.rs:487 | +| F5 | resolved-but-empty 谓词非零退出，应返回 `predicates: []` | main.rs:2021-2025 | +| F6 | 注释复述控制流/带 issue 叙事；`cmd_lineage` 保留内联解析双份 | extractor.rs 多处, main.rs, lineage.rs | + +--- + +## Fix 1 (F1): ELSIF 采集 + 简单 CASE 合成比较 + +**Files:** `src/parser/predicates.rs`（visitor 的 `PlStatement::If`/`PlStatement::Case` 臂）；测试同文件 tests 模块 + `tests/regress_predicates.rs`。 + +**Step 1 (Red):** +- 单测 `elsif_conditions_collected_as_predicates`：`IF r.x = '1' THEN ... ELSIF r.x = '2' THEN ... ELSIF r.x = '3' THEN ...`（record ctx）→ 3 条谓词，全部 `PredicateKind::If`、High、同表 clauses，id 递增；ELSIF 的 line 取各自 span（若 span 可得，否则 0——与现行主条件取法一致）。 +- 单测 `simple_case_synthesizes_expression_comparison`：`CASE r.x WHEN '1' THEN ... WHEN '2' THEN ...`（`expression: Some`）→ 每条 WHEN 产出 `column: x, op: Eq, value: '1'/'2'` 的 High 谓词（合成 `expression = when.condition`），而非裸字面量 Low。 +- e2e：`tests/regress_predicates.rs` 增补 fixture 断言 ELSIF 数量与简单 CASE 的 clauses。 + +**Step 2 (Green):** +- If 臂：主条件 push 后遍历 `spanned.elsifs`，逐个 `push_condition(&elsif.condition, PredicateKind::If, elsif 行号)`。 +- Case 臂：`spanned.expression` 为 `Some` 时，对每个 when 合成比较表达式（构造 `Expr::BinaryOp { left: expression.clone(), op: "=".into(), right: when.condition.clone() }` 或等价内部表示——以 `push_condition` 现有输入类型为准，必要时新增 `push_equality(expression, when_value)` 内部路径），`expression: None`（搜索型 CASE）保持现行为。 +- 跑 F1 既有测试确认不回归（搜索型 CASE 测试 `case_when_yields_predicates` 必须保持绿、语义不变）。 + +## Fix 2 (F2): condition_operand 断链修复 + Derived 携带 transform + +**Files:** `src/parser/predicates.rs`。 + +**Step 1 (Red):** +- `naked_column_substr_resolves_via_sole_table`：单游标表 ctx + `IF substr(stock_kind,1,2) = '05'` → High 谓词，clause 带 `transform: Some(substr[1,2])`（当前实际：Low 无谓词）。 +- `select_into_var_substr_resolves_via_var_source`：`SELECT kind_id INTO v_kind FROM swh_all_kind ...; IF substr(v_kind,1,2) = '05'` → Derived clause 指向 `swh_all_kind.kind_id` 且 **transform 携带**（当前：断链 Low）。 + +**Step 2 (Green):** +- `expr_name(expr)?` 改为可失败但不提前中断：将 `var_sources` 查找的键改为 `expr_name(expr)` **或** `column_transform_of(expr)` 的目标列名（裸列名，小写）；两键都查不到才落入 sole-table fallback（`names.len()==1 && tables.len()==1` 分支，现有 transform 透传已就绪）。 +- Derived 臂的 `PredicateClause` 携带与 Direct 臂相同的 `transform`（删除硬编码 `None`；var_sources 命中的是变量名包裹形态时 transform 语义同样成立）。 +- 注意 fallback 顺序保持：记录字段（`resolved_clause`）→ var_sources → sole-table；不改变记录字段路径的既有行为（`transformed_condition_clause_carries_transform` 等测试保持绿）。 + +## Fix 3 (F3): predicates 输出身份对齐 columns + +**Files:** `src/main.rs`（`cmd_predicates` + `PredicatesResult`）；`tests/regress_predicates.rs`。 + +**Step 1 (Red):** e2e `predicates_identity_matches_columns_for_packaged_procedure`：包内过程 fixture → `codeweb predicates --format json` 的 `procedure` == `columns` 的 `procedure`（均为裸名），且 predicates JSON 新增 `package` 字段 == 包名（columns 同名字段一致）。当前实际：`procedure == "pkg.prc"` 且无 package 字段 → FAIL。 + +**Step 2 (Green):** +- `PredicatesResult` 增 `#[serde(default, skip_serializing_if = "Option::is_none")] package: Option`（纯 JSON 输出结构，非 bincode 持久化——skip 安全；仿 `AggregatedColumnAnalysis` 的 package 字段风格）。 +- `cmd_predicates` 不再从 NodeKey 展示串 split：从图节点 `RoutineId` 取 `name` 与 `package`（对齐 `column_analysis_of_routine` 的取法）。 +- 独立过程 `package: None`（JSON 省略），schema_version 不变。 + +## Fix 4 (F4): 歧义显式失败，消灭静默首匹配 + +**Files:** `src/main.rs`（`cmd_columns`/`cmd_predicates` 两处）、`src/mcp/tools.rs`（`resolve_node`）、`src/server/handlers.rs`（`resolve_node`）；测试 `tests/regress_columns.rs`、`tests/regress_predicates.rs`、`tests/mcp_test.rs`、`tests/serve_api.rs`。 + +**Step 1 (Red):** +- e2e：同前缀双过程 fixture（如 `prc_order` / `prc_order_header`）→ `codeweb columns --procedure prc_order` 非零退出且 stderr 提示歧义（当前实际：静默返回首个 + exit 0）；`codeweb predicates` 同理。 +- serve_api：`GET /api/v1/columns?procedure=prc_order` → 409 或 400（按 handlers 既有错误约定选一个，报告所选）；mcp_test：`codeweb_column_analysis` 歧义名返回 error JSON（区分 Empty 的 "No nodes matching" 文案）。 + +**Step 2 (Green):** +- 四处调用第 4 参 `fail_on_multiple` 改 `true`；`cmd_*` 的 `ResolveResult::Ambiguous` 臂从死代码变为可达（保留现有非零错误路径）。 +- MCP `resolve_node` 返回区分 `Empty`（"No nodes matching ..."）与 `Ambiguous`（"Ambiguous match: N candidates ..."）；HTTP 对应 404 vs 400（报告所选映射）。 +- 不动 `trace`/`detail`/`impact` 等既有调用点的语义（它们本就交互式，首匹配+stderr 提示是既有契约）。 + +## Fix 5 (F5): resolved-empty 返回空数组 + +**Files:** `src/main.rs`；`tests/regress_predicates.rs`。 + +**Step 1 (Red):** `predicates_empty_branches_return_empty_array`：存在但无 IF/CASE 的过程 → exit 0、stdout 为 `{schema_version, procedure, predicates: []}`（当前实际：非零 + "No PL predicates found"）。 + +**Step 2 (Green):** `cmd_predicates` 中 store 侧表 miss/resolved-empty 不再 `?` 报错，改输出空 `predicates`；非零保留给：名称未解析（Empty）、歧义（F4 后可达）。注意与 F4 的歧义错误路径不冲突。 + +## Fix 6 (F6): 注释卫生 + cmd_lineage 共享 parse_lineage_target + +**Files:** `src/parser/extractor.rs`（仅注释）、`src/parser/predicates.rs`（仅注释）、`src/graph/lineage.rs`、`src/main.rs`。 + +**内容：** +- 精简复述控制流的注释；保留并压缩非显性 WHY（HardFilter/PredicateClause 的 bincode 固定字段数约束一句话足够）；删除 issue 编号/评审轮次/"intentionally left untouched" 类叙事。 +- `cmd_lineage` 改为调用 `graph::lineage::parse_lineage_target`（消除 T5 留下的内联双份及其叙事注释）；行为必须逐字不变——`tests/regress_lineage_table_upstream.rs`、`tests/regress_column_lineage.rs`、`tests/regress_issue_154_lineage_targets.rs` 全套保持绿不动即验证。 + +--- + +## 执行顺序与门禁 + +``` +F1 → F2（同文件连续 Red→Green） → F3 → F4 → F5 → F6（纯清理收尾） +``` + +每项独立 Red→Green；每完成两项跑一次： +```bash +cargo test --features full -- --skip test_path_mapping_applied --skip test_serve_ +cargo clippy --features full -- -D warnings +cargo fmt --all -- --check +``` +最终全量门禁 + `cargo build --features full`。 + +**Never 红线（AGENTS.md）**：不删/跳过/改写既有测试（F3/F4/F5 的新行为一律新增测试表达；若既有测试因 F4/F5 语义变化失败——如某测试断言了旧的静默首匹配——STOP 并报告，不得擅改）；不引入依赖/feature/unsafe/`#[allow]`；不动 `STORE_VERSION`；F6 不改任何行为语义（仅注释与等价重构，行为守护靠既有套件全绿）。 diff --git a/README.md b/README.md index 84353e2..2213b30 100644 --- a/README.md +++ b/README.md @@ -163,6 +163,9 @@ codeweb merge -o full-graph.bincode my-project.bincode erp-store.bincode | `codeweb files` | List analyzed files with node counts | | `codeweb nodes` | List graph nodes with filtering | | `codeweb trace-sql ` | Search by SQL fragment and trace to Java methods | +| `codeweb lineage ` | Table-level and column-level lineage analysis | +| `codeweb columns --procedure ` | Aggregate column-analysis for a procedure/package (JSON) | +| `codeweb predicates --procedure ` | PL IF/CASE predicates resolved to table columns (JSON) | | `codeweb query` | Execute declarative JSON QuerySpec | | `codeweb import` | Import CGEF JSON graph file | | `codeweb merge` | Merge multiple graph stores | @@ -219,6 +222,8 @@ When built with `--features serve`, codeweb provides a RESTful API: | GET | `/api/v1/nodes/:id/callees` | Downstream callees | | GET | `/api/v1/nodes/search-sql` | Search nodes by SQL fragment | | GET | `/api/v1/trace` | Bidirectional call chain tracing | +| GET | `/api/v1/lineage` | Table-level and column-level lineage | +| GET | `/api/v1/columns` | Aggregate column-analysis for a procedure/package | | POST | `/api/v1/query` | Execute declarative QuerySpec | | GET | `/api/v1/export` | Export graph (DOT/JSON/Mermaid) | @@ -263,6 +268,8 @@ Add to `claude_desktop_config.json`: | `codeweb_trace` | Bidirectional call chain tracing from a node name | | `codeweb_search_sql` | Search nodes by SQL text content with scoring | | `codeweb_query` | Execute declarative JSON QuerySpec for complex traversals | +| `codeweb_column_analysis` | Aggregate column-analysis for a procedure/package | +| `codeweb_lineage` | Table-level and column-level lineage analysis | ## Project Structure @@ -501,6 +508,9 @@ codeweb merge -o full-graph.bincode my-project.bincode erp-store.bincode | `codeweb files` | 列出已分析文件及节点数 | | `codeweb nodes` | 列出图节点（支持过滤） | | `codeweb trace-sql ` | 按 SQL 片段搜索并追踪到 Java 方法 | +| `codeweb lineage ` | 表级与列级血缘分析 | +| `codeweb columns --procedure ` | 按过程/包聚合列级分析结果 (JSON) | +| `codeweb predicates --procedure ` | 解析 PL IF/CASE 条件为表列谓词 (JSON) | | `codeweb query` | 执行声明式 JSON QuerySpec | | `codeweb import` | 导入 CGEF JSON 图谱文件 | | `codeweb merge` | 合并多个图谱存储 | @@ -557,6 +567,8 @@ codeweb merge -o full-graph.bincode my-project.bincode erp-store.bincode | GET | `/api/v1/nodes/:id/callees` | 下游被调用方 | | GET | `/api/v1/nodes/search-sql` | 按 SQL 文本搜索节点 | | GET | `/api/v1/trace` | 双向调用链追踪 | +| GET | `/api/v1/lineage` | 表级与列级血缘分析 | +| GET | `/api/v1/columns` | 按过程/包聚合列级分析结果 | | POST | `/api/v1/query` | 执行声明式 QuerySpec | | GET | `/api/v1/export` | 导出图谱（DOT/JSON/Mermaid） | @@ -601,6 +613,8 @@ codeweb mcp --project /path/to/your/project | `codeweb_trace` | 从节点名双向追踪调用链 | | `codeweb_search_sql` | 按 SQL 文本搜索节点（含相关性评分） | | `codeweb_query` | 执行声明式 JSON QuerySpec，支持复杂多步遍历 | +| `codeweb_column_analysis` | 按过程/包聚合列级分析结果 | +| `codeweb_lineage` | 表级与列级血缘分析 | ## 项目结构 diff --git a/docs/DeveloperGuide.md b/docs/DeveloperGuide.md index 05fe17e..219537e 100644 --- a/docs/DeveloperGuide.md +++ b/docs/DeveloperGuide.md @@ -140,6 +140,31 @@ impl CodeGraph { } ``` +### ColumnAnalysis（列级分析模型） + +`ColumnAnalysis` 结构体（以及通过 `codeweb columns` 导出的 `AggregatedColumnAnalysis`）承载了过程或语句级别的详细列约束面，其核心字段如下： + +| 字段名 | 类型 | 说明 | +|--------|------|------| +| `alias_map` | `BTreeMap` | 表别名到实际表名的映射关系 | +| `column_refs` | `HashSet` | 语句中出现的所有列引用集合 | +| `join_conditions` | `Vec` | 等值关联条件。`JoinCondition` 包含左右表列、关联类型及来源 `source`（`ImplicitWhere` \| `ExplicitOn` \| `RecordField` — 其中 `RecordField` 表示由 `%ROWTYPE` 记录字段推导出的跨表等值键） | +| `hard_filters` | `Vec` | 字面量过滤条件。`HardFilter` 包含表、列、操作符（Eq, Neq, Gt, Gte, Lt, Lte, Like, NotLike, In, Between, IsNull, IsNotNull）、字面量值，以及可选的 `transform`（函数包裹描述，如 `{"fn": "substr", "args": [...]}`） | +| `enum_mappings` | `Vec` | 基于 `CASE` / `DECODE` 的离散值枚举转换映射 | +| `select_into` | `Vec` | `SELECT INTO` 赋值到 PL 变量的映射关系 | +| `column_mappings` | `Vec` | 目标表列到源表列的血缘映射，区分 `Direct`（直接赋值）、`Derived`（表达式派生）、`Aggregated`（聚合函数）等种类 | +| `insert_columns` | `Vec` | `INSERT` 语句写入的目标列集合 | +| `update_columns` | `Vec` | `UPDATE` 语句更新的目标列集合 | +| `read_tables` | `Vec` | 语句或过程读取的源表列表 | + +#### 自动化造数与 Mock 消费场景 + +`ColumnAnalysis` 的结构化输出是自动化测试数据生成（Mock 数据生成）的核心输入源： +1. **跨表键关联**：利用 `join_conditions`（特别是 `RecordField` 隐式推导键）可以自动构建跨表的主外键关联池，确保生成的 Mock 数据在多表 JOIN 时不会因关联落空而变成空结果。 +2. **边界约束提取**：通过 `hard_filters` 提取出各表各列必须满足的字面量强约束（如 `status = '05'`），并结合 `transform` 逆向推导原始列的取值范围（如 `substr(kind,1,2)='05'` 要求 `kind` 前两位必须是 `'05'`）。 +3. **有效值集合播种**：从 `enum_mappings` 中收集列的离散有效值边界，避免生成非法的业务状态码。 +4. **靶向分支覆盖**：结合 `predicates`（PL 谓词解析）的条件约束与置信度，可以逆向推导触发特定 PL 分支（如特定的 `IF` 逻辑块）所需的数据特征，实现面向代码分支覆盖的靶向数据播种。 + --- ## GraphStore 存储层 @@ -205,6 +230,8 @@ HTTP API 通过 `axum` 框架提供，所有端点以 `/api/v1/` 为前缀，启 | GET | `/api/v1/nodes/:id/callees` | 节点下游被调用方（分页） | | GET | `/api/v1/nodes/search-sql` | 按 SQL 文本搜索（`q` 参数） | | GET | `/api/v1/trace` | 双向调用链追踪（`from`, `depth`, `max_nodes`） | +| GET | `/api/v1/lineage` | 表级与列级血缘分析（`target`, `direction`, `depth`） | +| GET | `/api/v1/columns` | 按过程/包聚合列级分析结果（`procedure`, `package`, `table`） | | POST | `/api/v1/query` | 执行 QuerySpec 声明式查询 | | GET | `/api/v1/export` | 导出图谱（`format` 参数：dot/json/mermaid） | | GET | `/api/v1/graph` | 完整图谱 JSON 数据 | @@ -331,6 +358,8 @@ codeweb 提供四种 MCP/外部集成方式： | `codeweb_nodes` | `search`, `node_type`, `limit`, `offset` | 节点列表（搜索、类型过滤、分页） | | `codeweb_node_detail` | `id` (usize) | 节点详情：属性 + callers + callees | | `codeweb_trace` | `from`, `depth`, `max_nodes` | 双向调用链追踪 | +| `codeweb_column_analysis` | `procedure`, `package`, `table` | 按过程/包聚合列级分析结果 | +| `codeweb_lineage` | `target`, `direction`, `depth` | 表级与列级血缘分析 | | `codeweb_search_sql` | `sql` | SQL 片段搜索 | | `codeweb_query` | `spec` (QuerySpec JSON) | 声明式复杂遍历 | diff --git a/docs/getting-started.md b/docs/getting-started.md index b4d2951..f138faf 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -364,6 +364,36 @@ Key options: ✅ **Success**: Output shows the path(s) between your nodes with hop count and edge types. +### 4.6 Data lineage — "Where does this column come from?" + +`lineage` traces data flow between tables and columns. It "looks inside" procedures to see how data moves. + +```sql +-- Add this to your myapp.sql +CREATE TABLE t_src (id INT, amt NUMBER); +CREATE TABLE t_out (id INT, amt NUMBER); + +CREATE OR REPLACE PROCEDURE proc_copy_amt AS +BEGIN + INSERT INTO t_out (id, amt) + SELECT id, amt FROM t_src; +END; +/ +``` + +After `codeweb analyze`, run: + +```bash +codeweb lineage t_out.amt --direction upstream +``` + +``` +t_out.amt + ← t_src.amt [direct] via proc:proc_copy_amt +``` + +✅ **Success**: codeweb correctly identified that `t_out.amt` is populated from `t_src.amt` via the `proc_copy_amt` procedure. + --- ## 5. Visual exploration (going further) diff --git a/docs/getting-started_zh.md b/docs/getting-started_zh.md index c05fe16..72adc50 100644 --- a/docs/getting-started_zh.md +++ b/docs/getting-started_zh.md @@ -360,6 +360,36 @@ codeweb inspect proc_main proc_helper --style tree ✅ **验证成功**：输出展示节点间的路径、跳数和边类型。 +### 4.6 数据血缘分析 — "这个列的数据从哪来？" + +`lineage` 命令追踪表与表、列与列之间的数据流转。它能“看穿”存储过程内部逻辑，识别数据搬运路径。 + +```sql +-- 在 myapp.sql 中追加以下内容 +CREATE TABLE t_src (id INT, amt NUMBER); +CREATE TABLE t_out (id INT, amt NUMBER); + +CREATE OR REPLACE PROCEDURE proc_copy_amt AS +BEGIN + INSERT INTO t_out (id, amt) + SELECT id, amt FROM t_src; +END; +/ +``` + +执行 `codeweb analyze` 后，运行： + +```bash +codeweb lineage t_out.amt --direction upstream +``` + +``` +t_out.amt + ← t_src.amt [direct] via proc:proc_copy_amt +``` + +✅ **验证成功**：codeweb 准确识别出 `t_out.amt` 的数据来源于 `t_src.amt`，且流转路径经过了 `proc_copy_amt` 存储过程。 + --- ## 5. 可视化探索（进阶） diff --git a/docs/plans/2026-09-08-issue-165-169-column-analysis-surface.md b/docs/plans/2026-09-08-issue-165-169-column-analysis-surface.md new file mode 100644 index 0000000..889078b --- /dev/null +++ b/docs/plans/2026-09-08-issue-165-169-column-analysis-surface.md @@ -0,0 +1,475 @@ +# Issue #165–169 列级分析查询面与解析增强（columns / predicates / transform / 跨表键 / 文档） + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** 打通「列级分析 → 造数/mock 机器可读入口」主线，覆盖五个 issue： +1. **#169** — 函数包裹列的字面量过滤纳入 `HardFilter`（substr/nvl/trim 白名单 + `transform` 字段）； +2. **方案A（用户已拍板）** — `merge_table_access_edges` 合并全部诊断字段（当前只合并 `column_mappings`/`read_tables`，其余「保留第一条」，导致同过程多语句同表时 hard_filters/join_conditions 丢失）+ `STORE_VERSION` 9→10； +3. **#165 P0** — `codeweb columns --procedure X --format json` 按过程聚合导出 ColumnAnalysis； +4. **#168** — WHERE/JOIN ON/SELECT INTO 中 `%ROWTYPE` 记录字段解析为跨表等值键； +5. **#165 P1** — MCP `codeweb_column_analysis`/`codeweb_lineage` + HTTP `GET /api/v1/columns`/`GET /api/v1/lineage`； +6. **#167** — PL IF/CASE 条件解析为表列谓词（置信度分级 + param_table_hint）； +7. **#166** — 文档补齐（lineage CLI、ColumnAnalysis 字段、新命令）。 + +**Architecture:** 全部改动在单 crate 内，无新外部依赖、无新 feature flag： +- **解析层** `src/parser/extractor.rs`：T1 白名单 transform、T4 记录字段等值键、T6 新谓词提取 pass； +- **图构建层** `src/graph/builder.rs`：T2 合并诊断字段（`merge_table_access_edges` L3330-3421）； +- **查询层** `src/graph/`（新增聚合函数）+ `src/main.rs`（新 CLI 子命令）+ `src/mcp/tools.rs` + `src/server/handlers.rs`； +- **存储** `src/graph/store.rs`：`STORE_VERSION` 9→10（T2，含 D4 合并 bump）；T6 再 bump 至 11（D6 已锁定存储方案）。 + +**Tech Stack:** Rust stable、ogsql-parser v0.10.0（git 依赖，checkout `~/.cargo/git/checkouts/ogsql-parser-9b270b8f87a071f2/28b5b4b`）、现有测试 harness（extractor.rs `#[cfg(test)]` 单测 + `tests/regress_*.rs` 端到端 + `tests/serve_api.rs` + `tests/mcp_test.rs`）。 + +--- + +## 决策记录（全部锁定：D1 用户拍板；D2–D6 依用户委托由 Momus 审核裁决） + +> **裁决说明**：Momus 第一轮审核（2026-09-08）确认 D2–D6 实质方向无异议、要求正式锁定以免实施阻塞。以下裁决即为最终结论，Task 4/6 按此执行，不再保留「建议」状态。 + +| # | 问题 | 选项 | 裁决与理由 | +|---|---|---|---| +| **D1** | #165 store 合并丢数据 | A: 合并诊断字段+bump / B: 接受少报 | **✅ 用户已拍板：方案A** | +| **D2** | #168 `JoinConditionSource` | 新增 `RecordField` 变体 / 复用 `ImplicitWhere` | **✅ 锁定：新增 `RecordField` 变体**。store 文件兼容由版本门禁隔离（旧二进制读不了新 store，无枚举反序列化问题）；JSON export 是单向输出，codeweb 自己不回读；唯一消费者 fastaas 是共建中的新代码，可同步适配。语义价值：下游需区分「隐式等值 JOIN」与「记录字段推导键」（置信度不同） | +| **D3** | #167 输出形态 | 独立 `codeweb predicates` / 并入 `columns` JSON | **✅ 锁定：独立 `codeweb predicates` 命令**，schema 复用 `FilterOperator`/`FilterValue`（issue 允许）。`columns` 保持聚焦列约束面；两 issue 解耦交付，TDD 分层清晰。MCP/HTTP 的 predicates 入口**暂缓**（issue 验收未强制，YAGNI） | +| **D4** | #169 是否 bump 版本 | 单独 bump / 与 T2 合并一次 | **✅ 锁定：与 T2 合并为一次 v9→10**。`transform` 是 serde-default 新字段，技术上无需 bump，但按 v7→v8 先例（加 `read_tables` 即 bump）+ 借 `store_is_current()` 促使用户重跑 analyze | +| **D5** | #169 白名单边界 | 仅逗号语法 FunctionCall / 双变体；白名单集合 | **✅ 锁定：同时处理 `FunctionCall` + `SpecialFunction`**（ogsql 文档明言 dual-variant：`SUBSTR(x FROM 1 FOR 2)` 走 SpecialFunction，只处理逗号语法会留下「换写法就漏」的坑）；白名单 **{substr, substring, nvl, trim, upper, lower}**（lower 与 upper 对称，成本≈0）。封闭白名单，不开放任意函数 | +| **D6** | #167 谓词存哪 | (a) analyze 期存入 GraphStore（`procedure_predicates` 侧表，serde default，bump v11）/ (b) 查询期重解析源文件 | **✅ 锁定：(a) 存储方案**。PL IF/CASE 分支结构在 `extract_body_sql` 摊平后即丢失，查询期重解析依赖源文件未变，脆弱且与「分析结果进 store」的既有架构一致。代价是多一次 bump（v11） | + +--- + +## 关键代码位置（当前实现，改动点） + +`src/parser/extractor.rs`： + +```rust +// L1930 — HardFilter（T1 加 transform 字段） +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct HardFilter { + pub table: Option, + pub column: String, + pub operator: FilterOperator, + pub value: FilterValue, +} + +// L2355-2505 — process_expr_for_joins_and_filters（T1 六个比较分支加白名单 arm；T4 等值分支加记录字段解析） +"=" => { + if let (Some(l), Some(r)) = (as_column_ref(left), as_column_ref(right)) { /* equi-join L2370 */ } + else if let Some(col) = as_column_ref(left) { /* col = literal → HardFilter L2385 */ } + else if let Some(col) = as_column_ref(right) { /* literal = col → HardFilter L2389 */ } + // ← FunctionCall/SpecialFunction 包裹列目前三条路全不匹配，静默丢弃 +} + +// L2561 — add_hard_filter（table 只经 resolve_alias 解析，无 transform） +// L3149 — column_source()（T4 复用其记录字段解析规则：精确 output_name 匹配 → 游标源列； +// catch-all（SELECT */动态SQL）→ 游标锚表+字段名；表锚定 %ROWTYPE → 锚表+字段名） +// L3149 所在 impl 已持有 record_cursors / cursor_sources（ProcedureVarContext，L2000） +``` + +`src/graph/builder.rs`： + +```rust +// L3330-3421 — merge_table_access_edges（T2：除 column_mappings/read_tables 外， +// 其余诊断字段 join_conditions/hard_filters/enum_mappings/select_into/ +// insert_columns/update_columns/column_refs/alias_map 当前「保留第一条」，改为集合并集去重） +``` + +`src/graph/store.rs`：L22 `STORE_VERSION: u32 = 9`（T2 → 10；T6 若走存储方案 → 11）。 +`src/graph/lineage.rs`：L1480 `mappings_of_routine`（T3 聚合函数的范本——HashSet 去重、扫入边+出边）。 +`src/main.rs`：L379-411 `Lineage` variant（T3/T6 新子命令的克隆范本）；L1558-1565 v7 软提示范本；L148-179 `ImpactResult`（`schema_version` 字段房屋风格）。 +`src/mcp/tools.rs`：L124-532 六工具注册（`#[tool(description=...)]`）；L539-549 `tool_handler` instructions。 +`src/server/handlers.rs`：L24-41 `router()`；L435-479 `trace` handler（Query-struct GET 范本）。 +`tests/mcp_test.rs`：L185-199 `test_mcp_tools_list` 硬编码 6 工具名，加工具必改。 + +**AST 事实（ogsql-parser v0.10.0，已核实）**： +- `Expr::FunctionCall { name: ObjectName, args: Vec, ... }`（ast/mod.rs:1221，逗号语法）； +- `Expr::SpecialFunction { name, args, ... }`（ast/mod.rs:1384，关键字语法——`SUBSTRING(x FROM 1 FOR 3)`、`TRIM(LEADING ... FROM ...)`）。文档要求 dual-variant 处理； +- `PlIfStmt { condition: Expr, then_stmts, elsifs: Vec, else_stmts }`（ast/plpgsql.rs:237）、`PlCaseStmt { expression, whens: Vec, else_stmts }`（L251）；`walk_pl_statement` 自动递归条件+分支体； +- WHERE 表达式：`Expr::BinaryOp{left,op:String,right}`、`Between`、`InList`、`Like`、`Case`、`FieldAccess{object,field}`（L1302）、`PlVariable`（L1415）； +- 记录字段在 SQL 中解析为多段 `ColumnRef`（`r.security_id` → 2 Idents），`split_alias_column`（extractor.rs L3872）已按此形状处理。 + +**测试基础设施（现有）**：`column_mappings_of(sql)` 等 helper（extractor.rs tests，L4924 起）；`ColumnAccessExtractor::new_with_context(&ProcedureVarContext)`（L2078，单测接缝）；`tests/regress_column_lineage.rs` 的 `project_with_sql` + `lineage()` harness；`run_codeweb_in`（tests/regress_lineage_table_upstream.rs L28）。**注意**：`par_sys_purchase`/`r_get_purchase`/STEP3 样例仓内不存在，T3/T4/T6 需自建 fixture。 + +--- + +## Task 1 (T2): 方案A — merge_table_access_edges 合并全部诊断字段 + STORE_VERSION 10 + +**Files:** +- Modify: `src/graph/builder.rs`（`merge_table_access_edges` L3330-3421） +- Modify: `src/graph/store.rs`（L22 `STORE_VERSION` 9→10；版本注释） +- Modify: `src/parser/extractor.rs`（若 `JoinCondition`/`HardFilter`/`EnumMapping`/`SelectIntoMapping`/`InsertColumnInfo`/`UpdateColumnInfo`/`ColumnRef` 缺 `Hash`，补 derive——所有字段均为 String/枚举/Vec，可哈希） +- Test: `src/graph/builder.rs` `#[cfg(test)]`（若无测试模块则在 store.rs 或新建 `tests/regress_column_analysis_merge.rs`） + +**Step 1: 写失败测试（Red）** + +单测：同一过程两条语句写同一张表、各带不同 `hard_filters` 与 `join_conditions`，经 builder 构建后该 `(proc, table)` 边的 `column_analysis` 应为并集： + +```rust +/// 方案A (issue #165): merged TableAccess edges must UNION diagnostic fields, +/// not keep only the first edge's. Two statements → same proc/table pair with +/// distinct hard filters must both survive. +#[test] +fn merge_table_access_unions_hard_filters_and_joins() { + // 构建：CREATE TABLE t(a NUMBER, b NUMBER); CREATE PROCEDURE p AS BEGIN + // INSERT INTO t SELECT x.a FROM s x WHERE x.a = 1; + // INSERT INTO t SELECT y.b FROM s y JOIN u z ON y.id = z.id WHERE y.b = 2; + // END; + // 断言：该 proc→t 边 column_analysis.hard_filters 同时含 a=1 与 b=2； + // join_conditions 含 s.id = u.id；column_mappings 仍正确去重。 +} +``` + +（实现时按 builder 现有测试范式落位；若 builder 无 `#[cfg(test)]`，用 `tests/regress_column_analysis_merge.rs` 端到端 + `export --format json` 断言。） + +**Step 2: 运行确认失败** + +Run: `cargo test --features full merge_table_access_unions_hard_filters_and_joins` +Expected: FAIL — 只有第一条语句的 hard_filters 幸存。 + +**Step 3: 最小实现（Green）** + +- 为上述类型补 `Hash` derive（`FilterValue::Float(String)` 可哈希，无 f64 阻碍）； +- `merge_table_access_edges`：仿照 `column_mappings` 的 HashSet 去重模式，对 `join_conditions`、`hard_filters`、`enum_mappings`、`select_into`、`insert_columns`、`update_columns`、`column_refs` 做集合并集；`alias_map` 做 BTreeMap extend（同 key 首见优先）；删除/改写「remaining diagnostic fields keep the first」注释（L3377-3379）； +- 读边（`AccessMode::Write` 不含）继续清空 `column_mappings` 的既有行为不变； +- `STORE_VERSION` 9→10，更新邻近注释（v10 = merge 诊断字段并集 + HardFilter.transform 预留，关联 #165/#169）。 + +**Step 4: 验证** + +Run: `cargo test --features full` + `cargo clippy --features full -- -D warnings` + `cargo fmt --all -- --check` +Expected: 新测试绿；store.rs 版本拒绝测试（`load_bincode_rejects_previous_layout_version` L2403、`load_bincode_rejects_pre_issue_159_version` L2425）依旧绿（它们写旧版本文件断言被拒，不受新版本号影响）；既有 full 套件除已知环境跳过项（`test_path_mapping_applied`、`test_serve_*`）外全绿。 + +--- + +## Task 2 (T1): #169 — 函数包裹列的字面量过滤纳入 HardFilter（白名单 + transform） + +**Files:** +- Modify: `src/parser/extractor.rs`（`HardFilter` L1930 加字段；新 struct `FilterTransform`；新 helper `column_transform_of`；`process_expr_for_joins_and_filters` 六个比较分支各加 arm；新 `add_hard_filter_with_transform`） +- Test: `src/parser/extractor.rs` tests 模块（filter 测试群 L4814-5038 旁） + +**Step 1: 写失败测试（Red）** + +```rust +/// #169: a whitelisted pure column transform compared against a literal yields a +/// HardFilter on the underlying column, with a transform descriptor. +#[test] +fn substr_wrapped_column_literal_becomes_hard_filter_with_transform() { + // WHERE substr(qs.stock_kind, 1, 2) = '05' (qs 为表别名) + // 断言：hard_filters 含 { table: Some(..), column: "stock_kind", Eq, String("05"), + // transform: Some(FilterTransform { fn_: "substr", args: [Integer(1), Integer(2)] }) } +} + +/// #169: the STEP3 mixed-cursor case — transformed and plain filters coexist. +#[test] +fn step3_cursor_mixed_filters_all_captured() { + // WHERE substr(qs.stock_kind,1,2)='05' AND qs.stock_kind <> '0509' + // AND qs.scdm = '001' AND qs.cjsl > 0 + // 断言：4 条 HardFilter，第一条带 transform，后三条 transform == None +} + +/// #169: non-literal extra args exclude the filter (PL variable in args). +#[test] +fn substr_with_variable_length_arg_is_excluded() { + // WHERE substr(col, 1, v_len) = '05' → 不产出 +} + +/// #169: non-whitelisted function or func-vs-func comparisons stay excluded. +#[test] +fn non_whitelisted_or_double_sided_function_is_excluded() { + // WHERE fnc_x(col) = '1' → 不产出；WHERE nvl(a,1) = nvl(b,2) → 不产出 +} + +/// #169: SpecialFunction (keyword syntax) is covered too. +#[test] +fn substr_keyword_syntax_produces_transform() { + // WHERE substring(col FROM 1 FOR 2) = '05' → 产出（D5 双变体） +} +``` + +**Step 2: 运行确认失败** + +Run: `cargo test --features full substr_wrapped_column_literal_becomes_hard_filter_with_transform step3_cursor_mixed_filters_all_captured` +Expected: FAIL — 现在什么都不产出。 + +**Step 3: 最小实现（Green）** + +```rust +/// #169: descriptor of a whitelisted pure column transform in a filter. +/// Serialized as {"fn": "substr", "args": [1, 2]} per issue schema. +#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +pub struct FilterTransform { + #[serde(rename = "fn")] + pub fn_name: String, // 小写规范化 + pub args: Vec, // 除目标列外的全部实参（均为字面量） +} +``` + +- `HardFilter` 增加 `#[serde(default, skip_serializing_if = "Option::is_none")] pub transform: Option`（满足验收「JSON 无 transform 或 null」；旧 store 反序列化得 None）； +- helper `column_transform_of(expr) -> Option<(&[Ident] /*列*/, FilterTransform)>`： + - 匹配 `Expr::FunctionCall` 与 `Expr::SpecialFunction`（D5 双变体），name 小写 ∈ {substr, substring, nvl, trim, upper, lower}； + - args 中恰好一个 `Expr::ColumnRef`，其余全部 `literal_to_filter_value` 成功（PL 变量→None→自动排除，天然满足「substr(col,1,v_len) 不产出」）； + - `substring` 与 `substr` 归一化为 `"substr"`； +- 六个比较分支（`=` `<>` `!=` `>` `>=` `<` `<=`）在 col-vs-literal 判断后各加对称 arm：一侧 `column_transform_of` 命中且另一侧 `literal_to_filter_value` 命中 → `add_hard_filter_with_transform`； +- `Like/Between/InList/IsNull` 侧不处理函数包裹（范围外）； +- 既有 `add_hard_filter` 保持签名，内部 `transform: None`（10 个调用点零改动）。 + +**Step 4: 验证** + +Run: `cargo test --features full` （重点回归 `test_join_with_alias_and_hard_filter` L4814、`test_pl_variable_not_hard_filter` L4895）+ clippy + fmt。 +Expected: 新旧全绿；既有 `col='x'` filter 的 `transform` 序列化后不出现（skip_serializing_if）。 + +--- + +## Task 3 (T3): #165 P0 — `codeweb columns` CLI（按过程聚合 ColumnAnalysis） + +**Files:** +- Add: `src/graph/columns.rs`（聚合函数 `pub fn column_analysis_of_routine(...) -> AggregatedColumnAnalysis`；模块注册 `src/graph/mod.rs`） +- Modify: `src/main.rs`（`Commands::Columns` variant + dispatch + `cmd_columns`；旧 store 软提示） +- Test: 新增 `tests/regress_columns.rs`（harness 仿 `regress_column_lineage.rs` 的 `project_with_sql`）+ graph 层单测 + +**Step 1: 写失败测试（Red）** + +```rust +/// #165: per-procedure column analysis export aggregates all TableAccess edges. +#[test] +fn columns_json_lists_hard_filters_and_joins_without_duplicates() { + // fixture（仿 STEP3 驱动游标 + 维表）： + // CREATE TABLE mid_yjqs_detail(...); CREATE TABLE par_fund_partner(...); + // CREATE PROCEDURE prc_trd_hz_byfund AS BEGIN + // -- 两条语句写同一张输出表，各带不同 hard_filter / join_condition + // INSERT INTO mid_yjqs_detail SELECT f.partner_no FROM par_fund_partner f + // WHERE f.fund_code = c.fund_code AND c.scdm = '001' ...; + // END; + // 断言 `codeweb columns --procedure prc_trd_hz_byfund --format json`： + // - schema_version == 1；procedure/package 字段正确 + // - hard_filters 含 scdm='001'；join_conditions 含 par_fund_partner.fund_code ↔ ... + // - 同一 filter/join 不重复出现（多边聚合去重） +} + +/// #165: --table narrows to one table's constraints. +#[test] +fn columns_json_table_filter_narrows_output() { /* --table mid_yjqs_detail 只出该表相关 */ } + +/// #165: unknown procedure → clear error, exit != 0. +#[test] +fn columns_unknown_procedure_errors_cleanly() { /* 不静默空数组 */ } +``` + +graph 层单测：聚合函数对合成边去重（两条边各含相同 `scdm='001'` → 只出现一次）。 + +**Step 2: 运行确认失败** + +Run: `cargo test --features full --test regress_columns` +Expected: FAIL — 子命令不存在（编译失败即为合法 Red）。 + +**Step 3: 最小实现（Green）** + +- `AggregatedColumnAnalysis`（serde struct，首字段 `schema_version: u32 = 1`，房屋风格仿 `ImpactResult` main.rs:148-179）： + `{ schema_version, procedure, package: Option, tables: Vec, join_conditions, hard_filters, select_into, enum_mappings, column_mappings, insert_columns, update_columns, read_tables }`——字段名与 `ColumnAnalysis` 1:1（issue 要求「不要再包一层展示用树」）； +- `column_analysis_of_routine`：仿 `mappings_of_routine`（lineage.rs:1480）——扫该 routine 节点入边+出边的 `Edge::TableAccess.column_analysis`，逐字段 HashSet 去重；`read_tables` 合并；`--table` 过滤在聚合层做（保留与目标表相关的边；join/filter 若涉及其它表仍保留——语句级隔离需要 read_tables）； +- CLI：`Commands::Columns { #[arg(long)] procedure: Option, #[arg(long)] package: Option, #[arg(long)] table: Option, #[arg(long, default_value="json", value_parser=["json"])] format: String, #[arg(short, long, default_value=".")] project: PathBuf }`；procedure/package 二选一必填（clap `group.required = true` + `conflicts_with`）； +- 旧 store 软提示：`store.version < 10` → `eprintln!("note: store version {} predates full column-analysis diagnostics (v10) — run `codeweb analyze` to rebuild.", ...)`（仿 main.rs:1560 范本）； +- 过程定位复用 `store.resolve_single_node(name, MatchMode::Substring, ...)` + 校验 Procedure/Function 节点（仿 cmd_lineage L1670-1696）。 + +**Step 4: 验证** + +Run: `cargo test --features full --test regress_columns` + 全套门禁。README/user-guide 文档在 T7 统一补。 + +--- + +## Task 4 (T4): #168 — WHERE/JOIN ON 记录字段解析为跨表等值键 + +**Files:** +- Modify: `src/parser/extractor.rs`（新 helper `resolve_record_field(&self, names) -> Option<(String, String)>` 复用 `column_source` 的三段规则；`extract_join_condition` 增加记录字段对侧路径；`JoinConditionSource` **新增 `RecordField` 变体【D2 已锁定】**，serde 序列化为 `"RecordField"`） +- Test: extractor.rs tests + `tests/regress_column_lineage.rs`（新 e2e） + +**Step 1: 写失败测试（Red）** + +```rust +/// #168: record field on one side of an equi-comparison resolves to the cursor's +/// source column, producing a cross-table JoinCondition. +#[test] +fn record_field_in_where_resolves_to_cross_table_join() { + // 上下文：CURSOR c_get_data IS SELECT security_id, fund_code FROM mid_yjqs_detail ...; + // r_get_purchase c_get_data%ROWTYPE; + // SQL: SELECT t.purchase_days INTO v_purchase_days FROM par_sys_purchase t + // WHERE t.security_id = r_get_purchase.security_id + // 断言：join_conditions 含 par_sys_purchase.security_id ↔ mid_yjqs_detail.security_id， + // source == RecordField【D2 已锁定】 +} + +/// #168: plain equi-joins regress unchanged. +#[test] +fn plain_on_equi_join_unchanged() { /* ON a.id = b.id → ImplicitWhere/ExplicitOn 如旧 */ } + +/// #168: record-vs-procedure-param and unregistered records produce nothing. +#[test] +fn record_vs_param_or_unregistered_produces_no_join() { + // WHERE r.col = p_i_date（参数侧）→ 不产出；未注册记录变量 → 不产出（不猜表名） +} + +/// #168: table-anchored %ROWTYPE and SELECT * cursor catch-all follow #142 rules. +#[test] +fn table_anchored_rowtype_and_star_cursor_resolve() { /* 两种锚定形态各一断言 */ } +``` + +e2e：`tests/regress_column_lineage.rs` 新增「STEP3 维表 JOIN」用例（fixture 自建 `par_sys_purchase` 风格）。 + +**Step 2: 运行确认失败** → **Step 3: 最小实现（Green）** + +- `resolve_record_field`：抽取 `column_source`（L3149）中「记录字段 → 游标源列」分支为独立函数（精确 output_name 匹配 → `(source_table, source_col)`；catch-all → `(cursor锚表, 字段名)`；表锚定 → `(锚表, 字段名)`），`column_source` 改为调用它（消除重复，Refactor 步骤内聚）； +- `process_expr_for_joins_and_filters` 的 `=` 分支：两侧 `as_column_ref` 双成功 → 现路径；**一侧列、一侧记录字段** → `extract_record_field_join`，产出 `JoinCondition { left/right 表列, source: RecordField }`【D2 已锁定】；去重逻辑复用现有反向查重（L2550-2555）； +- 记录字段一侧同时 `add_column_ref(..., JoinCondition)`（与现路径对齐）； +- `p_i_date` 参数经 `record_cursors` 查不到 → None → 不产出（负例免费）。 + +**Step 4: 验证**：全套门禁；`test_join_with_alias_and_hard_filter` 等既有 join 单测全绿。 + +--- + +## Task 5 (T5): #165 P1 — MCP `codeweb_column_analysis`/`codeweb_lineage` + HTTP `/api/v1/columns`/`/lineage` + +**Files:** +- Modify: `src/mcp/tools.rs`（两个新 `#[tool]` 方法 + 参数结构；`tool_handler` instructions 补两句）；`tests/mcp_test.rs`（tools list 断言 6→8） +- Modify: `src/server/handlers.rs`（router 两条 route + 两个 handler，仿 `trace` L435-479） +- Modify: `docs/serve-api-guide.md`、README 两表（亦可留 T7，此处至少改代码侧） +- 共享后端：T3 的 `graph::columns::column_analysis_of_routine` 与 lineage 既有函数，三个面共用同一 serde 结构，**不另发明 schema** + +**Step 1: 写失败测试（Red）** + +- `tests/mcp_test.rs`：`test_mcp_tools_list` 改为断言 8 个工具名（含 `codeweb_column_analysis`、`codeweb_lineage`）；新增 `test_mcp_call_column_analysis`（仿 `test_mcp_call_stats`，断言返回 JSON 与 CLI `columns --format json` 字段一致）； +- `tests/serve_api.rs`：`test_serve_columns_endpoint`、`test_serve_lineage_endpoint`（启动 serve、请求 `/api/v1/columns?procedure=...`、断言 200 + JSON 字段；404 场景）。 + +**Step 2: 运行确认失败** → **Step 3: 最小实现（Green）** + +- MCP `codeweb_column_analysis`：`ColumnAnalysisParams { procedure: Option, package: Option, table: Option }`；空图守卫复用 `graph_empty()`；返回 T3 同一 JSON 字符串； +- MCP `codeweb_lineage`：`LineageParams { target: String, direction: Option, depth: Option }`；复用 lineage_table/lineage_column + `format_lineage_json`/`format_column_lineage_json`，direction 缺省 both（与 CLI 一致）； +- HTTP `GET /api/v1/columns`：`ColumnsQuery { procedure: Option, package: Option, table: Option }`；`GET /api/v1/lineage`：`LineageQuery { target, direction: Option, depth: Option }`；错误约定与现有一致（缺参/未命中 → 400/404，无 envelope）； +- instructions 字符串（tools.rs:539）追加两工具用途说明。 + +**Step 4: 验证**：`cargo test --features full`（含 serve/mcp 集成测试；CI 跳过项除外）+ clippy + fmt。 + +--- + +## Task 6 (T6): #167 — PL IF/CASE 条件解析为表列谓词 + +**Files:** +- Add: `src/parser/predicates.rs`（`PredicateExtractor`：branch-aware Visitor pass + 谓词 AST） +- Modify: `src/graph/builder.rs`（过程构建期调用新 pass，产出挂入 store）；`src/graph/store.rs`（**加 `procedure_predicates` 侧表 + bump v11【D6 已锁定：存储方案】**） +- Modify: `src/main.rs`（`Commands::Predicates` + `cmd_predicates`，**独立命令【D3 已锁定】**） +- Test: `src/parser/predicates.rs` tests + `tests/regress_predicates.rs` + +**设计要点（D3/D6 均已锁定，直接按此实施）**： + +- 新 AST（全部 serde，schema 复用 `FilterOperator`/`FilterValue`）： + `PlPredicate { id: String /* B001… */, line: usize, origin: String, kind: PredicateKind(If|CaseWhen), confidence: Confidence(High|Medium|Low), table_predicate: Option, needs_review: Option, param_table_hint: Option }`； + `TablePredicate { table, clauses: Vec }`； + `ParamTableHint { table, filters: Vec, set: Vec<(String, FilterValue)> }`； +- pass 形态仿 `CallExtractor` 的 PL 走树（L441-799 证可行）：`impl Visitor for PredicateExtractor`，拦截 `PlStatement::If`/`Case`（读 `condition`/`whens[].condition`），条件表达式经「条件→clauses 转换器」解析——该转换器**复用 T1 的 `column_transform_of` + T4 的 `resolve_record_field` + `ProcedureVarContext`**； +- 置信度规则（issue 表格逐条落地，单测各锁一条）： + | 模式 | confidence | + |---|---| + | `r.field` 且 `record_cursors` 命中，比较字面量 | high | + | 裸列且 `scope_sole_table` 唯一 | high | + | `SELECT col INTO v` 后 `IF v = literal`，col 来自主表 | medium（主表谓词） | + | 同上但 col 来自维表 | low + `param_table_hint` | + | 函数调用/动态 SQL/GOTO | low / skip，保留 `origin` | +- 过程内 `SELECT INTO` 变量源追踪：pass 内自建 `HashMap`（走 `PlStatement::SqlStatement` 的 into_targets + targets，游标源解析复用 `ProcedureVarContext`）； +- IF 分支下语句归属：`then_stmts`/`else_stmts` 递归时携带当前条件上下文（分支内语句不重复产出谓词，谓词只来自条件本身）。 + +**Step 1: 写失败测试（Red）** + +```rust +/// #167: STEP3 star_market IF resolves to high-confidence table predicate. +#[test] +fn star_market_if_resolves_high_confidence() { + // IF r_get_data.stock_kind = '0100' AND r_get_data.zqdm BETWEEN '609100' AND '609999' + // → predicate { confidence: High, table: mid_yjqs_detail, + // clauses: [stock_kind eq '0100', zqdm between [609100,609999]] } +} + +/// #167: SELECT-INTO-derived var yields low confidence + param_table_hint. +#[test] +fn select_into_var_condition_yields_param_table_hint() { + // SELECT kind_id INTO v_kind FROM swh_all_kind WHERE operation_kind='COMMISSION_SWITCH'; + // IF v_kind = '1' → low + hint{ swh_all_kind, filters:[operation_kind eq ...], set:{kind_id:'1'} } + // 断言：不误写成主表谓词 +} + +/// #167: cursor WHERE hard filters do NOT leak into the IF predicate list. +#[test] +fn cursor_hard_filters_not_in_predicates() { /* 游标 WHERE 的 HardFilter 不出现在 predicates */ } + +/// #167: function-call conditions keep origin, low/skip confidence. +#[test] +fn function_condition_degrades_confidence() { /* IF fnc_x(a) = 1 → low + origin 保留 */ } +``` + +**Step 2: 运行确认失败** → **Step 3: 最小实现（Green）** → **Step 4: 验证** + +- CLI：`codeweb predicates --procedure X --format json`，输出 `{ schema_version: 1, procedure, predicates: [...] }`； +- store 增加 `procedure_predicates: HashMap>`（`#[serde(default)]`）【D6 已锁定】，`STORE_VERSION` → 11，`cmd_predicates` 直接读 store；旧 store < 11 软提示重跑 analyze； +- 全套门禁。 + +--- + +## Task 7 (T7): #166 — 文档补齐 + +**Files（纯文档，无代码）:** +- `README.md`（中英两份表格）：CLI 表加 `lineage`、`columns`、`predicates`；HTTP 表加 `/columns`、`/lineage`；MCP 工具表加两个新工具 +- `docs/user-guide.md`：§6 新增 `lineage` 子节（table vs table.column、--direction/--view/--flow-only、store v7+ 提示、与 trace 的区别）+ `columns`/`predicates` 子节 +- `docs/DeveloperGuide.md`：`ColumnAnalysis` 字段表（join/hard_filter/select_into/mapping kind/transform）+ 消费场景（mock 造数）+ MCP/HTTP 表更新 +- `docs/getting-started.md` + `_zh`：10 行 INSERT..SELECT 的 `codeweb lineage t_out.amt --direction upstream` 示例 +- `docs/serve-api-guide.md`：`/columns`、`/lineage` 端点文档（若 T5 未覆盖） + +**Step 1: 可执行 QA 场景（文档的「失败测试」——先跑通核对清单再动笔，列出当前缺失项）** + +```bash +# QA-1 README 命令表与 --help 一致性（中英两份表都要核对） +codeweb --help +# 预期缺失（写文档前应确认 grep 全部落空， documenting 后应 ≥2：英文表 + 中文表各一行）： +grep -c '| `codeweb lineage' README.md # 现在 0 → 目标 ≥ 2 +grep -c '| `codeweb columns' README.md # 现在 0 → 目标 ≥ 2 +grep -c '| `codeweb predicates' README.md # 现在 0 → 目标 ≥ 2 + +# QA-2 user-guide 出现可照跑的小节（写前 0 命中，写后各 ≥1 个 §6.x 标题） +grep -n '^#\{2,3\} .*lineage' docs/user-guide.md +grep -n '^#\{2,3\} .*columns' docs/user-guide.md +grep -n '^#\{2,3\} .*predicates' docs/user-guide.md + +# QA-3 serve-api-guide 端点存在且字段与实际输出一致 +grep -n 'api/v1/columns\|api/v1/lineage' docs/serve-api-guide.md # 目标 ≥ 1 处/端点 +# 字段一致性核对：文档响应示例顶层键 == 实际输出顶层键（对 T3 fixture 项目执行） +codeweb columns --procedure prc_trd_hz_byfund --format json | jq -S 'keys' +codeweb serve & curl -s 'http://127.0.0.1:3000/api/v1/columns?procedure=prc_trd_hz_byfund' | jq -S 'keys' +# 两次 jq keys 输出必须相同，且与 serve-api-guide 文档示例逐键一致 + +# QA-4 DeveloperGuide ColumnAnalysis 字段说明 +grep -n 'ColumnAnalysis' docs/DeveloperGuide.md # 目标：字段表出现（含 transform 行） +grep -n 'codeweb_column_analysis\|codeweb_lineage' docs/DeveloperGuide.md # MCP 表 8 工具 + +# QA-5 getting-started 示例可照跑（10 行 INSERT..SELECT fixture） +# 按文档步骤在 /tmp 临时项目逐字执行，预期输出含： +codeweb lineage t_out.amt --direction upstream +# → 树中出现源列 t_src.amt（或 fixture 对应源表列），非 "No column lineage" +``` + +**Step 2: 依清单撰写/修订文档**（上面每条 grep 由 0 → 目标值；QA-3/QA-5 的实际命令输出与文档示例逐字一致） + +**验收（照 issue #166 + Momus 要求的可执行核对）**：QA-1~QA-5 全部通过；`codeweb --help` 的每个子命令在 README 两份 CLI 表各有且仅有一行；serve-api-guide 响应示例键集与 `jq keys` 实测一致。 + +--- + +## 执行顺序与门禁 + +``` +T2（方案A合并+bump v10） → T1（#169 transform） → T3（#165 P0 CLI） +→ T4（#168 跨表键） → T5（#165 P1 MCP/HTTP） → T6（#167 谓词，bump v11【D6 已锁定】） +→ T7（#166 文档） → 全量门禁 +``` + +每个 Task 独立 Red→Green→Refactor 循环，完成即跑： +```bash +cargo test --features full -- --skip test_path_mapping_applied --skip test_serve_ +cargo clippy --features full -- -D warnings +cargo fmt --all -- --check +``` +最终门禁另跑 `cargo build --features full` + `cargo test --features full`。 + +**Never 红线（AGENTS.md）**：不删/跳过/改写人类已有测试断言；`test_join_with_alias_and_hard_filter`、`test_pl_variable_not_hard_filter`、`test_mcp_tools_list`（改 6→8 属新增工具的必要同步，在汇报中显式说明）、store 版本拒绝测试为只读基线；每个行为先有失败测试；不引入新依赖/feature flag。 diff --git a/docs/plans/2026-09-08-pr170-review-fixes.md b/docs/plans/2026-09-08-pr170-review-fixes.md new file mode 100644 index 0000000..0fc9c47 --- /dev/null +++ b/docs/plans/2026-09-08-pr170-review-fixes.md @@ -0,0 +1,105 @@ +# PR #170 评审修复计划（#165–#169 跟随修复） + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** 修复 PR #170 六条评审意见（4 bug + 2 suggestion，全部已对照代码核实成立）。核心目标：#167 谓词在 ELSIF/简单 CASE/函数包裹裸列/SELECT INTO 变量形态下不漏报不错报；`columns` 与 `predicates` 的过程身份可 join；机器入口不静默错配。 + +**Architecture:** 全部改动位于 `src/parser/predicates.rs`、`src/main.rs`、`src/mcp/tools.rs`、`src/server/handlers.rs`、`src/graph/lineage.rs`、`src/parser/extractor.rs`（仅注释）。无 store 布局变更——**不需要 bump `STORE_VERSION`（保持 12）**：F1/F2 改的是提取逻辑而非已序列化结构；F3 只改 JSON 输出字段来源；F4/F5 是解析参数与错误语义。 + +**已核实的评审发现（全部接受，无争议项）：** + +| # | 发现 | 核实位置 | +|---|---|---| +| F1 | If 臂漏 `elsifs`；简单 CASE（`expression: Some`）把 WHEN 值当裸条件 | predicates.rs:277-287 | +| F2 | `condition_operand` 的 `expr_name(expr)?` 对 FunctionCall 断链，跳过 var_sources 与 sole-table fallback；Derived 臂硬编码 `transform: None` | predicates.rs:402, 409 | +| F3 | `cmd_predicates` 的 `procedure` 取自 NodeKey 展示串（包内过程得 `pkg.prc`），与 columns 的 `id.name`+`package` 不一致；无 `package` 字段 | main.rs:2026-2029 | +| F4 | 四处新调用点 `resolve_single_node(..., false, false)` → `Ambiguous` 臂不可达，多匹配静默取首个 | main.rs:1894/1990, tools.rs:715, handlers.rs:487 | +| F5 | resolved-but-empty 谓词非零退出，应返回 `predicates: []` | main.rs:2021-2025 | +| F6 | 注释复述控制流/带 issue 叙事；`cmd_lineage` 保留内联解析双份 | extractor.rs 多处, main.rs, lineage.rs | + +--- + +## Fix 1 (F1): ELSIF 采集 + 简单 CASE 合成比较 + +**Files:** `src/parser/predicates.rs`（visitor 的 `PlStatement::If`/`PlStatement::Case` 臂）；测试同文件 tests 模块 + `tests/regress_predicates.rs`。 + +**Step 1 (Red):** +- 单测 `elsif_conditions_collected_as_predicates`：`IF r.x = '1' THEN ... ELSIF r.x = '2' THEN ... ELSIF r.x = '3' THEN ...`（record ctx）→ 3 条谓词，全部 `PredicateKind::If`、High、同表 clauses，id 递增；ELSIF 的 line 取各自 span（若 span 可得，否则 0——与现行主条件取法一致）。 +- 单测 `simple_case_synthesizes_expression_comparison`：`CASE r.x WHEN '1' THEN ... WHEN '2' THEN ...`（`expression: Some`）→ 每条 WHEN 产出 `column: x, op: Eq, value: '1'/'2'` 的 High 谓词（合成 `expression = when.condition`），而非裸字面量 Low。 +- e2e：`tests/regress_predicates.rs` 增补 fixture 断言 ELSIF 数量与简单 CASE 的 clauses。 + +**Step 2 (Green):** +- If 臂：主条件 push 后遍历 `spanned.elsifs`，逐个 `push_condition(&elsif.condition, PredicateKind::If, elsif 行号)`。 +- Case 臂：`spanned.expression` 为 `Some` 时，对每个 when 合成比较表达式（构造 `Expr::BinaryOp { left: expression.clone(), op: "=".into(), right: when.condition.clone() }` 或等价内部表示——以 `push_condition` 现有输入类型为准，必要时新增 `push_equality(expression, when_value)` 内部路径），`expression: None`（搜索型 CASE）保持现行为。 +- 跑 F1 既有测试确认不回归（搜索型 CASE 测试 `case_when_yields_predicates` 必须保持绿、语义不变）。 + +## Fix 2 (F2): condition_operand 断链修复 + Derived 携带 transform + +**Files:** `src/parser/predicates.rs`。 + +**Step 1 (Red):** +- `naked_column_substr_resolves_via_sole_table`：单游标表 ctx + `IF substr(stock_kind,1,2) = '05'` → High 谓词，clause 带 `transform: Some(substr[1,2])`（当前实际：Low 无谓词）。 +- `select_into_var_substr_resolves_via_var_source`：`SELECT kind_id INTO v_kind FROM swh_all_kind ...; IF substr(v_kind,1,2) = '05'` → Derived clause 指向 `swh_all_kind.kind_id` 且 **transform 携带**（当前：断链 Low）。 + +**Step 2 (Green):** +- `expr_name(expr)?` 改为可失败但不提前中断：将 `var_sources` 查找的键改为 `expr_name(expr)` **或** `column_transform_of(expr)` 的目标列名（裸列名，小写）；两键都查不到才落入 sole-table fallback（`names.len()==1 && tables.len()==1` 分支，现有 transform 透传已就绪）。 +- Derived 臂的 `PredicateClause` 携带与 Direct 臂相同的 `transform`（删除硬编码 `None`；var_sources 命中的是变量名包裹形态时 transform 语义同样成立）。 +- 注意 fallback 顺序保持：记录字段（`resolved_clause`）→ var_sources → sole-table；不改变记录字段路径的既有行为（`transformed_condition_clause_carries_transform` 等测试保持绿）。 + +## Fix 3 (F3): predicates 输出身份对齐 columns + +**Files:** `src/main.rs`（`cmd_predicates` + `PredicatesResult`）；`tests/regress_predicates.rs`。 + +**Step 1 (Red):** e2e `predicates_identity_matches_columns_for_packaged_procedure`：包内过程 fixture → `codeweb predicates --format json` 的 `procedure` == `columns` 的 `procedure`（均为裸名），且 predicates JSON 新增 `package` 字段 == 包名（columns 同名字段一致）。当前实际：`procedure == "pkg.prc"` 且无 package 字段 → FAIL。 + +**Step 2 (Green):** +- `PredicatesResult` 增 `#[serde(default, skip_serializing_if = "Option::is_none")] package: Option`（纯 JSON 输出结构，非 bincode 持久化——skip 安全；仿 `AggregatedColumnAnalysis` 的 package 字段风格）。 +- `cmd_predicates` 不再从 NodeKey 展示串 split：从图节点 `RoutineId` 取 `name` 与 `package`（对齐 `column_analysis_of_routine` 的取法）。 +- 独立过程 `package: None`（JSON 省略），schema_version 不变。 + +## Fix 4 (F4): 歧义显式失败，消灭静默首匹配 + +**Files:** `src/main.rs`（`cmd_columns`/`cmd_predicates` 两处）、`src/mcp/tools.rs`（`resolve_node`）、`src/server/handlers.rs`（`resolve_node`）；测试 `tests/regress_columns.rs`、`tests/regress_predicates.rs`、`tests/mcp_test.rs`、`tests/serve_api.rs`。 + +**Step 1 (Red):** +- e2e：同前缀双过程 fixture（如 `prc_order` / `prc_order_header`）→ `codeweb columns --procedure prc_order` 非零退出且 stderr 提示歧义（当前实际：静默返回首个 + exit 0）；`codeweb predicates` 同理。 +- serve_api：`GET /api/v1/columns?procedure=prc_order` → 409 或 400（按 handlers 既有错误约定选一个，报告所选）；mcp_test：`codeweb_column_analysis` 歧义名返回 error JSON（区分 Empty 的 "No nodes matching" 文案）。 + +**Step 2 (Green):** +- 四处调用第 4 参 `fail_on_multiple` 改 `true`；`cmd_*` 的 `ResolveResult::Ambiguous` 臂从死代码变为可达（保留现有非零错误路径）。 +- MCP `resolve_node` 返回区分 `Empty`（"No nodes matching ..."）与 `Ambiguous`（"Ambiguous match: N candidates ..."）；HTTP 对应 404 vs 400（报告所选映射）。 +- 不动 `trace`/`detail`/`impact` 等既有调用点的语义（它们本就交互式，首匹配+stderr 提示是既有契约）。 + +## Fix 5 (F5): resolved-empty 返回空数组 + +**Files:** `src/main.rs`；`tests/regress_predicates.rs`。 + +**Step 1 (Red):** `predicates_empty_branches_return_empty_array`：存在但无 IF/CASE 的过程 → exit 0、stdout 为 `{schema_version, procedure, predicates: []}`（当前实际：非零 + "No PL predicates found"）。 + +**Step 2 (Green):** `cmd_predicates` 中 store 侧表 miss/resolved-empty 不再 `?` 报错，改输出空 `predicates`；非零保留给：名称未解析（Empty）、歧义（F4 后可达）。注意与 F4 的歧义错误路径不冲突。 + +## Fix 6 (F6): 注释卫生 + cmd_lineage 共享 parse_lineage_target + +**Files:** `src/parser/extractor.rs`（仅注释）、`src/parser/predicates.rs`（仅注释）、`src/graph/lineage.rs`、`src/main.rs`。 + +**内容：** +- 精简复述控制流的注释；保留并压缩非显性 WHY（HardFilter/PredicateClause 的 bincode 固定字段数约束一句话足够）；删除 issue 编号/评审轮次/"intentionally left untouched" 类叙事。 +- `cmd_lineage` 改为调用 `graph::lineage::parse_lineage_target`（消除 T5 留下的内联双份及其叙事注释）；行为必须逐字不变——`tests/regress_lineage_table_upstream.rs`、`tests/regress_column_lineage.rs`、`tests/regress_issue_154_lineage_targets.rs` 全套保持绿不动即验证。 + +--- + +## 执行顺序与门禁 + +``` +F1 → F2（同文件连续 Red→Green） → F3 → F4 → F5 → F6（纯清理收尾） +``` + +每项独立 Red→Green；每完成两项跑一次： +```bash +cargo test --features full -- --skip test_path_mapping_applied --skip test_serve_ +cargo clippy --features full -- -D warnings +cargo fmt --all -- --check +``` +最终全量门禁 + `cargo build --features full`。 + +**Never 红线（AGENTS.md）**：不删/跳过/改写既有测试（F3/F4/F5 的新行为一律新增测试表达；若既有测试因 F4/F5 语义变化失败——如某测试断言了旧的静默首匹配——STOP 并报告，不得擅改）；不引入依赖/feature/unsafe/`#[allow]`；不动 `STORE_VERSION`；F6 不改任何行为语义（仅注释与等价重构，行为守护靠既有套件全绿）。 diff --git a/docs/serve-api-guide.md b/docs/serve-api-guide.md index ca09c76..c64b620 100644 --- a/docs/serve-api-guide.md +++ b/docs/serve-api-guide.md @@ -36,6 +36,8 @@ cargo run --features serve -- serve --open | GET | `/api/v1/nodes/:id/callees` | 节点的下游被调用方 | | GET | `/api/v1/nodes/search-sql` | 按 SQL 文本内容搜索节点 | | GET | `/api/v1/trace` | 双向调用链追踪 | +| GET | `/api/v1/columns` | 按过程/包聚合列级分析（hard filters、joins 等，与 `codeweb columns --format json` 同构） | +| GET | `/api/v1/lineage` | 表级/列级血缘（与 `codeweb lineage --format json` 同构） | | POST | `/api/v1/query` | 执行声明式查询（QuerySpec） | | GET | `/api/v1/export` | 导出图谱（DOT/JSON/Mermaid） | | GET | `/api/v1/graph` | 完整图谱数据（JSON） | @@ -670,6 +672,110 @@ curl http://127.0.0.1:3000/api/v1/graph --- +## 12. GET `/api/v1/columns` — 列级分析聚合 + +按存储过程或包聚合导出详细的列级分析结果（Hard Filter、Join 条件、SELECT INTO 映射等）。 + +### 请求参数 + +| 参数 | 类型 | 必填 | 说明 | +|------|------|------|------| +| `procedure` | string | 否 | 存储过程或函数名（子串匹配） | +| `package` | string | 否 | 包名（导出该包下所有过程的并集） | +| `table` | string | 否 | 仅输出与指定表相关的诊断信息 | + +`procedure` 与 `package` 必须提供其中之一。 +名称不存在时返回 `404`；子串匹配到多个候选时返回 `400`，不会静默选择首个结果。 + +### 请求示例 + +```bash +curl "http://127.0.0.1:3000/api/v1/columns?procedure=p_test_hard" +``` + +### 响应 + +```json +{ + "schema_version": 1, + "procedure": "p_test_hard", + "package": null, + "tables": ["t_out", "t_src"], + "join_conditions": [], + "hard_filters": [ + { + "table": null, + "column": "kind", + "operator": "Eq", + "value": { "String": "05" }, + "transform": { + "fn": "substr", + "args": [{ "Integer": 1 }, { "Integer": 2 }] + } + } + ], + "select_into": [], + "enum_mappings": [], + "column_mappings": [ + { + "target_table": "t_out", + "target_column": "amt", + "position": 1, + "sources": [{ "Column": { "table": "t_src", "column": "amt" } }], + "kind": "Direct", + "expression": null + } + ], + "insert_columns": [{ "table": "t_out", "columns": ["id", "amt", "kind"] }], + "update_columns": [], + "read_tables": ["t_out", "t_src"] +} +``` + +--- + +## 13. GET `/api/v1/lineage` — 表级/列级血缘分析 + +执行表级或列级的血缘分析，追踪数据的来源或去向。 + +### 请求参数 + +| 参数 | 类型 | 必填 | 默认值 | 说明 | +|------|------|------|--------|------| +| `target` | string | 是 | — | 分析目标：`table` 或 `table.column` | +| `direction` | string | 否 | `both` | 方向：`upstream`、`downstream`、`both` | +| `depth` | number | 否 | `5` | 递归深度 | + +### 请求示例 + +```bash +curl "http://127.0.0.1:3000/api/v1/lineage?target=t_out.amt&direction=upstream" +``` + +### 响应 + +```json +{ + "table": "t_out", + "column": "amt", + "steps": [ + { + "source": "t_src.amt", + "via": "proc:p_test_hard", + "kind": "direct", + "expression": null, + "next": { + "table": "t_src", + "column": "amt", + "steps": [] + } + } + ] +} +``` + +--- + ## 典型使用场景 ### 场景 1：查找某个存储过程的所有调用方 @@ -742,6 +848,6 @@ curl "http://127.0.0.1:3000/api/v1/trace?from=sp_calc_risk&depth=5&max_nodes=100 | HTTP 状态码 | 说明 | |-------------|------| | `200` | 成功 | -| `400` | 请求参数错误（如 QuerySpec JSON 格式错误、不支持的导出格式） | -| `404` | 节点不存在（`node_detail`、`node_callers`、`node_callees`、`trace`） | +| `400` | 请求参数错误或 `/api/v1/columns` 名称匹配存在歧义 | +| `404` | 节点不存在（包括 `/api/v1/columns` 无匹配） | | `500` | 服务器内部错误 | diff --git a/docs/user-guide.md b/docs/user-guide.md index 7213372..fd3e6aa 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -22,15 +22,18 @@ - [6.8 节点列表：`nodes`](#68-节点列表nodes) - [6.9 SQL 搜索与追踪：`trace-sql`](#69-sql-搜索与追踪trace-sql) - [6.10 影响分析：`impact`](#610-影响分析impact) - - [6.11 图谱导出：`export`](#611-图谱导出export) - - [6.12 声明式查询：`query`](#612-声明式查询query) - - [6.13 图谱去重：`dedup`](#613-图谱去重dedup) - - [6.14 系统分解：`partition`](#614-系统分解partition) - - [6.15 CGEF 导入：`import`](#615-cgef-导入import) - - [6.16 多项目合并：`merge`](#616-多项目合并merge) - - [6.17 交互式终端：`tui`](#617-交互式终端tui) - - [6.18 HTTP 服务：`serve`](#618-http-服务serve) - - [6.19 MCP 服务：`mcp`](#619-mcp-服务mcp) + - [6.11 血缘分析：`lineage`](#611-血缘分析lineage) + - [6.12 列级分析聚合：`columns`](#612-列级分析聚合columns) + - [6.13 PL 谓词解析：`predicates`](#613-pl-谓词解析predicates) + - [6.14 图谱导出：`export`](#614-图谱导出export) + - [6.15 声明式查询：`query`](#615-声明式查询query) + - [6.16 图谱去重：`dedup`](#616-图谱去重dedup) + - [6.17 系统分解：`partition`](#617-系统分解partition) + - [6.18 CGEF 导入：`import`](#618-cgef-导入import) + - [6.19 多项目合并：`merge`](#619-多项目合并merge) + - [6.20 交互式终端：`tui`](#620-交互式终端tui) + - [6.21 HTTP 服务：`serve`](#621-http-服务serve) + - [6.22 MCP 服务：`mcp`](#622-mcp-服务mcp) - [7. 典型使用场景](#7-典型使用场景) - [8. 常见问题](#8-常见问题) - [附录：节点类型与边类型](#附录节点类型与边类型) @@ -662,7 +665,98 @@ done --- -### 6.11 图谱导出：`export` +### 6.11 血缘分析：`lineage` + +执行表级或列级的血缘分析，追踪数据的来源（上游）或去向（下游）。 + +```bash +codeweb lineage <目标> [OPTIONS] +``` + +| 参数 | 说明 | +|------|------| +| `<目标>` | 血缘分析目标：`table_name`（表级）、`table.column`（列级）或节点标识如 `table:schema.table` | +| `--direction <方向>` | 血缘方向：`upstream`（溯源）、`downstream`（去向）或 `both`（双向，默认） | +| `--depth <深度>` | 递归深度（默认 5） | +| `--format <格式>` | 输出格式：`tree`（树形，默认）、`json`、`dot`、`mermaid` | +| `--view <视图>` | 渲染视图：`tree`（默认）、`entity`（隐藏过程节点）、`relation`（显式关系线）、`grouped`（按过程分组） | +| `--flow-only` | 仅显示流转源，隐藏参考源（受配置阈值影响） | + +**与 `trace` 的区别**： +- `trace` 侧重于**调用链**（谁调用了谁），展示过程间的控制流。 +- `lineage` 侧重于**数据流**（数据从哪张表的哪个列流向了哪张表的哪个列），会自动穿透存储过程内部的赋值和 `INSERT..SELECT` 逻辑。 + +**注意**： +- 列级血缘需要存储版本 ≥ v7。如果 store 版本过低，会提示 "No column lineage" 或建议重新运行 `analyze`。 + +**示例**： +```bash +# 追踪 t_out 表 amt 列的来源 +codeweb lineage t_out.amt --direction upstream + +# 导出表级血缘为 Mermaid 流程图 +codeweb lineage t_orders --format mermaid +``` + +--- + +### 6.12 列级分析聚合：`columns` + +按存储过程或包聚合导出详细的列级分析结果（包括 Hard Filter、Join 条件、SELECT INTO 映射、枚举映射等）。这是为自动化造数或 Mock 环境提供的机器可读入口。 + +```bash +codeweb columns <--procedure <名称>|--package <名称>> [OPTIONS] +``` + +| 参数 | 说明 | +|------|------| +| `--procedure <名称>` | 要聚合的存储过程或函数名（支持子串匹配） | +| `--package <名称>` | 要聚合的包名（导出该包下所有过程的并集） | +| `--table <表名>` | 仅输出与指定表相关的诊断信息（不区分大小写） | +| `--format <格式>` | 输出格式（目前仅支持 `json`） | + +**注意**： +- 该命令需要存储版本 ≥ v10。旧版本 store 仅包含基础血缘，缺少详细的过滤和关联诊断。 + +**示例**： +```bash +# 导出过程 prc_trd_hz 的列级分析 JSON +codeweb columns --procedure prc_trd_hz --format json +``` + +--- + +### 6.13 PL 谓词解析：`predicates` + +解析 PL/SQL 内部的 `IF` 和 `CASE` 分支条件，将其转化为针对表列的谓词约束。 + +```bash +codeweb predicates --procedure <名称> [OPTIONS] +``` + +| 参数 | 说明 | +|------|------| +| `--procedure <名称>` | 存储过程或函数名（支持子串匹配） | +| `--format <格式>` | 输出格式（目前仅支持 `json`） | + +**输出包含**： +- **过程身份**：`procedure` 始终为裸过程/函数名；包内例程另有 `package` 字段，便于与 `columns` 输出关联。 +- **置信度 (Confidence)**：High（直接列比较）、Medium（经变量传递）、Low（复杂表达式或维表关联）。 +- **Param Table Hint**：如果谓词涉及维表开关，会产出造数建议。 +- **Needs Review**：对于无法自动解析的复杂逻辑，保留原始代码片段供人工审计。 + +**注意**： +- 该命令需要存储版本 ≥ v12。 +- 已解析到过程但没有 `IF`/`CASE` 谓词时命令仍成功，并返回 `"predicates": []`；仅名称不存在或存在歧义时失败。 + +**示例**： +```bash +codeweb predicates --procedure prc_calc_fee --format json +``` + +--- + +### 6.14 图谱导出：`export` 将代码图谱导出为不同格式，用于可视化或集成。 diff --git a/src/graph/builder.rs b/src/graph/builder.rs index 7feb32c..bbd48f8 100644 --- a/src/graph/builder.rs +++ b/src/graph/builder.rs @@ -134,6 +134,9 @@ pub struct GraphBuildContext { /// Threaded through SQL-proc / XML-mapper / Java / JSP paths so the same /// builtin called from multiple paths is a single graph node. pub builtin_index: HashMap, + /// Branch predicates keyed by the routine's serialized [`NodeKey`]. Kept outside + /// the graph because predicates are an analysis side-table, not traversable edges. + pub procedure_predicates: HashMap>, /// Deferred column comments from `COMMENT ON COLUMN` statements. /// Collected during `create_sql_nodes` and applied in `finalize_graph` /// after all table columns are populated. @@ -152,6 +155,7 @@ impl GraphBuildContext { sequence_index: HashMap::new(), inferred_sequence_index: HashMap::new(), builtin_index: HashMap::new(), + procedure_predicates: HashMap::new(), deferred_column_comments: Vec::new(), } } @@ -189,13 +193,35 @@ impl GraphBuilder { #[allow(dead_code)] pub fn build_store(&self, all: &AllParsedFiles, project_name: &str) -> GraphStore { - let graph = Self::build_graph_internal( - &all.sql_files, + let mut ctx = GraphBuildContext::new(); + Self::build_sql_chunk(&mut ctx, &all.sql_files); + Self::add_ibatis_nodes_from_parsed( &all.ibatis_files, + &mut ctx.graph, + &mut ctx.proc_index, + &mut ctx.mapper_index, + &mut ctx.table_index, + &mut ctx.builtin_index, + ); + Self::add_java_nodes_from_parsed( &all.java_files, + &mut ctx.graph, + &mut ctx.proc_index, + &ctx.mapper_index, + &mut ctx.table_index, + &mut ctx.builtin_index, + ); + Self::add_java_method_nodes_from_parsed( &all.java_method_results, + &mut ctx.graph, + &mut ctx.proc_index, + &ctx.mapper_index, ); - GraphStore::from_graph(project_name, graph) + Self::finalize_graph(&mut ctx); + let predicates = std::mem::take(&mut ctx.procedure_predicates); + let mut store = GraphStore::from_graph(project_name, ctx.graph); + store.set_procedure_predicates(predicates); + store } fn build_graph_internal( @@ -308,6 +334,133 @@ impl GraphBuilder { &ctx.sequence_index, &mut ctx.inferred_sequence_index, ); + Self::collect_procedure_predicates(ctx, sql_files); + } + + fn collect_procedure_predicates(ctx: &mut GraphBuildContext, files: &[ParsedFile]) { + for file in files { + for info in &file.statements { + match &info.statement { + Statement::CreateProcedure(procedure) => { + // Storage key must match the lowercase normalization that + // `NodeKey::from_node` applies (graph nodes are keyed off + // `RoutineId::normalized()`); otherwise `cmd_predicates` + // silently misses raw-case routines (#167 Finding 1). + let id = + RoutineId::from_object_name(&procedure.name, RoutineKind::Procedure) + .normalized(); + if let Some(block) = &procedure.block { + Self::collect_block_predicates( + ctx, + NodeKey::Procedure { + schema: id.schema.clone(), + package: id.package.clone(), + name: id.name.clone(), + } + .to_string(), + block, + ); + } + } + Statement::CreateFunction(function) => { + let id = RoutineId::from_object_name(&function.name, RoutineKind::Function) + .normalized(); + if let Some(block) = &function.block { + Self::collect_block_predicates( + ctx, + NodeKey::Function { + schema: id.schema.clone(), + package: id.package.clone(), + name: id.name.clone(), + } + .to_string(), + block, + ); + } + } + Statement::CreatePackage(package) => { + Self::collect_package_predicates(ctx, &package.name, &package.items) + } + Statement::CreatePackageBody(package) => { + Self::collect_package_predicates(ctx, &package.name, &package.items) + } + _ => {} + } + } + } + } + + fn collect_package_predicates( + ctx: &mut GraphBuildContext, + name: &ogsql_parser::ast::ObjectName, + items: &[PackageItem], + ) { + // Mirrors `create_package_nodes`'s RoutineId construction exactly, then + // normalizes it — this must produce the same key as the actual graph + // node's `NodeKey::from_node` (#167 Finding 1). + let schema = (name.len() > 1).then(|| name[..name.len() - 1].join(".")); + let package_name = name.last().map(ToString::to_string); + for item in items { + match item { + PackageItem::Procedure(procedure) => { + if let Some(block) = &procedure.block { + let id = RoutineId { + schema: schema.clone(), + package: package_name.clone(), + name: procedure.name.join("."), + kind: RoutineKind::Procedure, + } + .normalized(); + Self::collect_block_predicates( + ctx, + NodeKey::Procedure { + schema: id.schema, + package: id.package, + name: id.name, + } + .to_string(), + block, + ); + } + } + PackageItem::Function(function) => { + if let Some(block) = &function.block { + let id = RoutineId { + schema: schema.clone(), + package: package_name.clone(), + name: function.name.join("."), + kind: RoutineKind::Function, + } + .normalized(); + Self::collect_block_predicates( + ctx, + NodeKey::Function { + schema: id.schema, + package: id.package, + name: id.name, + } + .to_string(), + block, + ); + } + } + _ => {} + } + } + } + + fn collect_block_predicates( + ctx: &mut GraphBuildContext, + key: String, + block: &ogsql_parser::ast::plpgsql::PlBlock, + ) { + let mut columns = crate::parser::ColumnAccessExtractor::new(); + walk_pl_block(&mut columns, block); + let procedure_ctx = columns.procedure_context(); + let predicates = crate::parser::extract_predicates(block, &procedure_ctx); + if !predicates.is_empty() { + ctx.procedure_predicates.insert(key, predicates); + } } /// Finalize the graph after all files are processed. @@ -3327,6 +3480,18 @@ impl GraphBuilder { } } + /// Append items from `src` to `dst` that are not already present in `dst` + /// (order-preserving, `Hash + Eq` dedup), used by `merge_table_access_edges` + /// to union `ColumnAnalysis` diagnostic fields across merged edges (issue #165). + fn union_dedup_vec(dst: &mut Vec, src: &[T]) { + let mut seen: std::collections::HashSet = dst.iter().cloned().collect(); + for item in src { + if seen.insert(item.clone()) { + dst.push(item.clone()); + } + } + } + fn merge_table_access_edges(graph: &mut CodeGraph) { let mut merge_targets: HashMap< ( @@ -3374,9 +3539,14 @@ impl GraphBuilder { for wk in write_kinds { merged_kinds.insert(*wk); } - // Union the per-statement analyses: mappings and same-statement - // read tables accumulate across the statements merged into this edge - // (issue #147); the remaining diagnostic fields keep the first. + // Union the per-statement analyses across all edges merged into this + // (src, dst, flow_kind) key (issue #147 introduced column_mappings/ + // read_tables union; issue #165 extends the union to every diagnostic + // Vec field — join_conditions, hard_filters, enum_mappings, + // select_into, insert_columns, update_columns, column_refs — plus an + // alias_map extend (first-wins on key collision), so statements that + // write the same table with distinct filters/joins don't lose all but + // the first statement's diagnostics. if let Some(ca) = column_analysis { match &mut merged_col { Some(m) => { @@ -3396,6 +3566,18 @@ impl GraphBuilder { (Some(rt), None) => m.read_tables = Some(rt.clone()), _ => {} } + Self::union_dedup_vec(&mut m.join_conditions, &ca.join_conditions); + Self::union_dedup_vec(&mut m.hard_filters, &ca.hard_filters); + Self::union_dedup_vec(&mut m.enum_mappings, &ca.enum_mappings); + Self::union_dedup_vec(&mut m.select_into, &ca.select_into); + Self::union_dedup_vec(&mut m.insert_columns, &ca.insert_columns); + Self::union_dedup_vec(&mut m.update_columns, &ca.update_columns); + Self::union_dedup_vec(&mut m.column_refs, &ca.column_refs); + for (alias, table) in &ca.alias_map { + m.alias_map + .entry(alias.clone()) + .or_insert_with(|| table.clone()); + } } None => merged_col = Some(ca.clone()), } @@ -4521,6 +4703,7 @@ impl Default for GraphBuilder { #[cfg(test)] mod tests { use crate::graph::builder::GraphBuilder; + use crate::graph::key::NodeKey; use crate::graph::{Edge, Node}; use crate::parser::ParsedFile; use std::collections::HashMap; @@ -4542,6 +4725,212 @@ mod tests { GraphBuilder::new().build(&parsed) } + #[test] + fn procedure_predicates_are_collected_under_routine_node_key() { + let sql = r#" + CREATE TABLE main_data(x VARCHAR(10)); + CREATE PROCEDURE P AS + CURSOR c IS SELECT x FROM main_data; + r c%ROWTYPE; + BEGIN + IF r.x = '1' THEN NULL; END IF; + END; + "#; + let parsed = vec![ParsedFile { + path: PathBuf::from("test.sql"), + statements: parse_sql(sql), + content_hash: String::new(), + }]; + let mut ctx = crate::graph::builder::GraphBuildContext::new(); + + GraphBuilder::build_sql_chunk(&mut ctx, &parsed); + + // #167 fix: storage keys are lowercase-normalized (matching `NodeKey::from_node`), + // so the raw-case "P" from the source is stored as "proc:p". + let predicates = ctx + .procedure_predicates + .get("proc:p") + .expect("predicates stored by routine node key"); + assert_eq!(predicates.len(), 1); + assert_eq!(predicates[0].confidence, crate::parser::Confidence::High); + } + + /// Regression for the review Finding 1: `collect_procedure_predicates` / + /// `collect_package_predicates` must build storage keys using the SAME + /// lowercase normalization that `NodeKey::from_node` applies when + /// `cmd_predicates` looks the key back up. Fixture procedures are declared + /// UPPERCASE (both standalone and inside a package body) to prove the + /// storage key isn't accidentally case-matching by coincidence. + #[test] + fn predicates_stored_under_lowercase_node_key() { + let sql = r#" + CREATE TABLE main_data(x VARCHAR(10)); + + CREATE PROCEDURE PRC_STAR_MARKET AS + CURSOR c IS SELECT x FROM main_data; + r c%ROWTYPE; + BEGIN + IF r.x = '1' THEN NULL; END IF; + END; + + CREATE OR REPLACE PACKAGE BODY PKG_MAIN AS + PROCEDURE PRC_IN_PKG IS + CURSOR c IS SELECT x FROM main_data; + r c%ROWTYPE; + BEGIN + IF r.x = '2' THEN NULL; END IF; + END; + END PKG_MAIN; + "#; + let parsed = vec![ParsedFile { + path: PathBuf::from("test.sql"), + statements: parse_sql(sql), + content_hash: String::new(), + }]; + let mut ctx = crate::graph::builder::GraphBuildContext::new(); + + GraphBuilder::build_sql_chunk(&mut ctx, &parsed); + + let standalone_idx = ctx + .graph + .node_indices() + .find(|&idx| { + matches!(&ctx.graph[idx], Node::Procedure { id, .. } if id.name == "prc_star_market") + }) + .expect("standalone procedure node exists"); + let standalone_key = NodeKey::from_node(&ctx.graph[standalone_idx]).to_string(); + let predicates = ctx + .procedure_predicates + .get(&standalone_key) + .unwrap_or_else(|| { + panic!( + "predicates stored under key matching NodeKey::from_node ({standalone_key}); \ + available keys: {:?}", + ctx.procedure_predicates.keys().collect::>() + ) + }); + assert_eq!(predicates.len(), 1); + + let package_idx = ctx + .graph + .node_indices() + .find(|&idx| { + matches!(&ctx.graph[idx], Node::Procedure { id, .. } if id.name == "prc_in_pkg") + }) + .expect("package procedure node exists"); + let package_key = NodeKey::from_node(&ctx.graph[package_idx]).to_string(); + let predicates = ctx + .procedure_predicates + .get(&package_key) + .unwrap_or_else(|| { + panic!( + "package predicates stored under key matching NodeKey::from_node ({package_key}); \ + available keys: {:?}", + ctx.procedure_predicates.keys().collect::>() + ) + }); + assert_eq!(predicates.len(), 1); + } + + /// 方案A (issue #165): merged TableAccess edges must UNION diagnostic fields, + /// not keep only the first edge's. Two statements → same proc/table pair with + /// distinct hard filters (and only one carrying a join condition) must both + /// survive the merge, not just the first statement's. + #[test] + fn merge_table_access_unions_hard_filters_and_joins() { + let sql = r#" + CREATE TABLE s (col NUMBER, id NUMBER); + CREATE TABLE u (id NUMBER); + CREATE TABLE t (col NUMBER); + + CREATE OR REPLACE PROCEDURE p AS + BEGIN + INSERT INTO t(col) SELECT x.col FROM s x WHERE x.col = 1; + INSERT INTO t(col) SELECT y.col FROM s y JOIN u z ON y.id = z.id WHERE y.col = 2; + END; + "#; + let graph = build_from_sql(sql); + + let proc_idx = graph + .node_indices() + .find(|i| matches!(&graph[*i], Node::Procedure { id, .. } if id.name == "p")) + .expect("procedure p should exist"); + let table_t_idx = graph + .node_indices() + .find(|i| matches!(&graph[*i], Node::Table { name, .. } if name == "t")) + .expect("table t should exist"); + + let write_edges: Vec<_> = graph + .edge_indices() + .filter(|e| { + let (src, dst) = graph.edge_endpoints(*e).unwrap(); + src == proc_idx + && dst == table_t_idx + && matches!(&graph[*e], Edge::TableAccess { modes, .. } if modes.contains(crate::graph::AccessMode::Write)) + }) + .collect(); + assert_eq!( + write_edges.len(), + 1, + "the two INSERT statements into t should merge into exactly 1 TableAccess edge, got {}", + write_edges.len() + ); + + let ca = match &graph[write_edges[0]] { + Edge::TableAccess { + column_analysis: Some(ca), + .. + } => ca, + other => panic!("expected TableAccess edge with column_analysis, got {other:?}"), + }; + + use crate::parser::{FilterOperator, FilterValue}; + + let has_filter_1 = ca.hard_filters.iter().any(|hf| { + hf.column == "col" + && hf.operator == FilterOperator::Eq + && hf.value == FilterValue::Integer(1) + }); + let has_filter_2 = ca.hard_filters.iter().any(|hf| { + hf.column == "col" + && hf.operator == FilterOperator::Eq + && hf.value == FilterValue::Integer(2) + }); + assert!( + has_filter_1, + "merged hard_filters should still contain the first statement's col=1 filter, got: {:?}", + ca.hard_filters + ); + assert!( + has_filter_2, + "merged hard_filters should contain the second statement's col=2 filter \ + (this is the union bug: pre-fix code only kept the first edge's hard_filters), got: {:?}", + ca.hard_filters + ); + + assert_eq!( + ca.join_conditions.len(), + 1, + "merged join_conditions should contain the join from the second statement \ + (pre-fix code kept only the first edge's join_conditions, which had none), got: {:?}", + ca.join_conditions + ); + let jc = &ca.join_conditions[0]; + assert_eq!(jc.left_table, "s"); + assert_eq!(jc.left_column, "id"); + assert_eq!(jc.right_table, "u"); + assert_eq!(jc.right_column, "id"); + + // column_mappings must still be deduplicated (both statements produce the + // identical mapping t.col ← s.col), not doubled by the union. + assert_eq!( + ca.column_mappings.len(), + 1, + "identical column_mappings across merged statements should stay deduplicated, got: {:?}", + ca.column_mappings + ); + } + #[test] fn package_body_creates_package_and_procedure_nodes() { let sql = r#" diff --git a/src/graph/columns.rs b/src/graph/columns.rs new file mode 100644 index 0000000..daaa0c8 --- /dev/null +++ b/src/graph/columns.rs @@ -0,0 +1,398 @@ +//! #165 (P0): per-procedure/per-package aggregated column-analysis query surface. +//! +//! A routine that writes the same table across multiple statements (e.g. one `INSERT` +//! per branch of a legacy PL/SQL procedure) attaches a distinct [`ColumnAnalysis`] to +//! each statement's `TableAccess` edges. Without aggregation, a caller wanting "every +//! hard filter and join this procedure relies on" would have to walk the graph itself +//! and reconcile duplicates across edges. [`column_analysis_of_routine`] (and its +//! package-scoped sibling [`column_analysis_of_package`]) does that walk once and +//! unions every diagnostic field with `HashSet`-backed dedup, so the same filter/join +//! attached to two statements is reported once, not twice. +//! +//! The output schema ([`AggregatedColumnAnalysis`]) mirrors [`ColumnAnalysis`] field +//! names 1:1 — issue #165 explicitly rules out an extra display-tree wrapper layer — +//! so the MCP/HTTP surfaces planned for a later task can reuse this struct unchanged. + +use std::collections::HashSet; + +use petgraph::graph::NodeIndex; +use petgraph::visit::EdgeRef; +use petgraph::Direction; + +use crate::graph::{CodeGraph, Edge, Node}; +use crate::parser::{ + ColumnMapping, EnumMapping, HardFilter, InsertColumnInfo, JoinCondition, SelectIntoMapping, + UpdateColumnInfo, +}; + +/// `codeweb columns` JSON output schema (schema_version=1). +/// +/// Field names mirror [`ColumnAnalysis`](crate::parser::ColumnAnalysis) 1:1 by design +/// (issue #165). `procedure` names the queried entity (the resolved procedure/function +/// name in `--procedure` mode, or the resolved package name in `--package` mode, since +/// a package query has no single "the" procedure). `package` is `Some` whenever the +/// query's scope is known to belong to a package: the routine's own +/// [`RoutineId::package`](crate::graph::RoutineId) in `--procedure` mode, or the queried +/// package's own name in `--package` mode. +#[derive(Debug, Clone, serde::Serialize)] +pub struct AggregatedColumnAnalysis { + pub schema_version: u32, + pub procedure: String, + pub package: Option, + pub tables: Vec, + pub join_conditions: Vec, + pub hard_filters: Vec, + pub select_into: Vec, + pub enum_mappings: Vec, + pub column_mappings: Vec, + pub insert_columns: Vec, + pub update_columns: Vec, + pub read_tables: Vec, +} + +/// Working accumulator for [`collect_diagnostics`] — same fields as +/// [`AggregatedColumnAnalysis`] minus the identity fields (`schema_version`, +/// `procedure`, `package`), which only the public entry points know how to fill in. +#[derive(Default)] +struct Diagnostics { + tables: Vec, + join_conditions: Vec, + hard_filters: Vec, + select_into: Vec, + enum_mappings: Vec, + column_mappings: Vec, + insert_columns: Vec, + update_columns: Vec, + read_tables: Vec, +} + +/// Scan every `TableAccess` edge of `routines` (both directions, mirroring +/// [`super::lineage::mappings_of_routine`]'s defensive both-direction walk — today's +/// builder always emits routine→table edges outgoing, but scanning both keeps this +/// correct if that ever changes) and union each `ColumnAnalysis` diagnostic field with +/// `HashSet`-based dedup. +/// +/// `table_filter`, when set, narrows which edges are scanned to those whose *other* +/// endpoint (the table) matches case-insensitively — this is what shrinks `tables` +/// (and `read_tables`) to one table. Row-level fields (`join_conditions`, +/// `hard_filters`, `select_into`, ...) are NOT filtered by column value: the same +/// `ColumnAnalysis` is attached to every edge of one statement (the write edge and +/// every read edge), so an edge to the filtered table already carries exactly that +/// table's statements' constraints — including constraints that reference other +/// tables (e.g. a join partner, or a filter on a dimension table used by the same +/// statement). Dropping rows that merely *mention* another table would throw away +/// the join/filter context the caller is asking for; statement-level isolation for a +/// single table is what `read_tables` is for. +fn collect_diagnostics( + graph: &CodeGraph, + routines: &[NodeIndex], + table_filter: Option<&str>, +) -> Diagnostics { + let filter_lower = table_filter.map(|t| t.to_lowercase()); + + let mut tables: HashSet = HashSet::new(); + let mut read_tables: HashSet = HashSet::new(); + + let mut join_conditions: Vec = Vec::new(); + let mut jc_seen: HashSet = HashSet::new(); + let mut hard_filters: Vec = Vec::new(); + let mut hf_seen: HashSet = HashSet::new(); + let mut select_into: Vec = Vec::new(); + let mut si_seen: HashSet = HashSet::new(); + let mut enum_mappings: Vec = Vec::new(); + let mut em_seen: HashSet = HashSet::new(); + let mut column_mappings: Vec = Vec::new(); + let mut cm_seen: HashSet = HashSet::new(); + let mut insert_columns: Vec = Vec::new(); + let mut ic_seen: HashSet = HashSet::new(); + let mut update_columns: Vec = Vec::new(); + let mut uc_seen: HashSet = HashSet::new(); + + for &routine in routines { + for dir in [Direction::Outgoing, Direction::Incoming] { + for edge_ref in graph.edges_directed(routine, dir) { + let Edge::TableAccess { + column_analysis: Some(analysis), + .. + } = edge_ref.weight() + else { + continue; + }; + + let other = if dir == Direction::Outgoing { + edge_ref.target() + } else { + edge_ref.source() + }; + let table_name = match &graph[other] { + Node::Table { name, .. } | Node::View { name, .. } => name.clone(), + _ => continue, + }; + + if let Some(filt) = &filter_lower { + if table_name.to_lowercase() != *filt { + continue; + } + } + + tables.insert(table_name); + + for jc in &analysis.join_conditions { + if jc_seen.insert(jc.clone()) { + join_conditions.push(jc.clone()); + } + } + for hf in &analysis.hard_filters { + if hf_seen.insert(hf.clone()) { + hard_filters.push(hf.clone()); + } + } + for si in &analysis.select_into { + if si_seen.insert(si.clone()) { + select_into.push(si.clone()); + } + } + for em in &analysis.enum_mappings { + if em_seen.insert(em.clone()) { + enum_mappings.push(em.clone()); + } + } + for cm in &analysis.column_mappings { + if cm_seen.insert(cm.clone()) { + column_mappings.push(cm.clone()); + } + } + for ic in &analysis.insert_columns { + if ic_seen.insert(ic.clone()) { + insert_columns.push(ic.clone()); + } + } + for uc in &analysis.update_columns { + if uc_seen.insert(uc.clone()) { + update_columns.push(uc.clone()); + } + } + if let Some(rt) = &analysis.read_tables { + for t in rt { + read_tables.insert(t.clone()); + } + } + } + } + } + + let mut tables: Vec = tables.into_iter().collect(); + tables.sort(); + let mut read_tables: Vec = read_tables.into_iter().collect(); + read_tables.sort(); + + Diagnostics { + tables, + join_conditions, + hard_filters, + select_into, + enum_mappings, + column_mappings, + insert_columns, + update_columns, + read_tables, + } +} + +/// Aggregate every `TableAccess` diagnostic for one routine (procedure or function) +/// into a single [`AggregatedColumnAnalysis`]. Returns `None` when `routine` is not a +/// `Node::Procedure`/`Node::Function` — callers should verify the node type up front +/// (as `codeweb columns`'s CLI handler does) so this is a defensive fallback, not the +/// primary error path. +pub fn column_analysis_of_routine( + graph: &CodeGraph, + routine: NodeIndex, + table_filter: Option<&str>, +) -> Option { + let (name, package) = match &graph[routine] { + Node::Procedure { id, .. } | Node::Function { id, .. } => { + (id.name.clone(), id.package.clone()) + } + _ => return None, + }; + + let diag = collect_diagnostics(graph, &[routine], table_filter); + + Some(AggregatedColumnAnalysis { + schema_version: 1, + procedure: name, + package, + tables: diag.tables, + join_conditions: diag.join_conditions, + hard_filters: diag.hard_filters, + select_into: diag.select_into, + enum_mappings: diag.enum_mappings, + column_mappings: diag.column_mappings, + insert_columns: diag.insert_columns, + update_columns: diag.update_columns, + read_tables: diag.read_tables, + }) +} + +/// Aggregate every `TableAccess` diagnostic across all procedures/functions a package +/// contains (via `Edge::ContainsRoutine`, the same edge `codeweb detail`'s package +/// table-access summary uses — see `print_table_summary` in `src/main.rs`) into a +/// single [`AggregatedColumnAnalysis`]. Returns `None` when `package` is not a +/// `Node::Package`. +/// +/// A package with zero `ContainsRoutine` children (e.g. an empty or partially-parsed +/// package) yields an aggregation with empty diagnostic vectors rather than `None` — +/// the package itself was found, so this is not the "unknown name" error case the CLI +/// guards against. +pub fn column_analysis_of_package( + graph: &CodeGraph, + package: NodeIndex, + table_filter: Option<&str>, +) -> Option { + let pkg_name = match &graph[package] { + Node::Package { name, .. } => name.clone(), + _ => return None, + }; + + let children: Vec = graph + .edges_directed(package, Direction::Outgoing) + .filter(|e| matches!(e.weight(), Edge::ContainsRoutine)) + .map(|e| e.target()) + .collect(); + + let diag = collect_diagnostics(graph, &children, table_filter); + + Some(AggregatedColumnAnalysis { + schema_version: 1, + procedure: pkg_name.clone(), + package: Some(pkg_name), + tables: diag.tables, + join_conditions: diag.join_conditions, + hard_filters: diag.hard_filters, + select_into: diag.select_into, + enum_mappings: diag.enum_mappings, + column_mappings: diag.column_mappings, + insert_columns: diag.insert_columns, + update_columns: diag.update_columns, + read_tables: diag.read_tables, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::graph::{AccessMode, DataFlowKind, RoutineId, RoutineKind, SourceLocation}; + use crate::parser::{ColumnAnalysis, FilterOperator, FilterValue}; + use std::collections::BTreeMap; + use std::path::PathBuf; + use std::sync::Arc; + + fn loc() -> SourceLocation { + SourceLocation { + file: Arc::new(PathBuf::from("t.sql")), + line: 1, + } + } + + fn empty_analysis() -> ColumnAnalysis { + ColumnAnalysis { + alias_map: BTreeMap::new(), + column_refs: Vec::new(), + join_conditions: Vec::new(), + hard_filters: Vec::new(), + enum_mappings: Vec::new(), + select_into: Vec::new(), + insert_columns: Vec::new(), + update_columns: Vec::new(), + column_mappings: Vec::new(), + read_tables: None, + } + } + + /// Two separate `TableAccess` edges to the same table, each carrying an + /// analysis with the same `HardFilter` (as if two statements wrote the same + /// table with an identical filter): aggregation must dedup to one occurrence. + #[test] + fn dedups_identical_hard_filter_across_two_edges() { + let mut graph = CodeGraph::new(); + let routine = graph.add_node(Node::Procedure { + id: RoutineId { + schema: None, + package: None, + name: "prc_test".to_string(), + kind: RoutineKind::Procedure, + }, + location: loc(), + partial: false, + body_sql: Vec::new(), + }); + let table = graph.add_node(Node::Table { + schema: None, + name: "s1_src".to_string(), + explicit: true, + system: false, + location: None, + columns: Box::new(Vec::new()), + partition_by: None, + distribute_by: None, + tablespace: None, + temporary: false, + unlogged: false, + ddl_source: None, + }); + + let same_filter = HardFilter { + table: Some("s1_src".to_string()), + column: "scdm".to_string(), + operator: FilterOperator::Eq, + value: FilterValue::String("001".to_string()), + transform: None, + }; + + for _ in 0..2 { + let mut analysis = empty_analysis(); + analysis.hard_filters.push(same_filter.clone()); + graph.add_edge( + routine, + table, + Edge::TableAccess { + flow_kind: DataFlowKind::DmlAccess, + modes: AccessMode::Read, + write_kinds: Default::default(), + location: loc(), + column_analysis: Some(Box::new(analysis)), + }, + ); + } + + let result = column_analysis_of_routine(&graph, routine, None).expect("aggregation"); + assert_eq!( + result.hard_filters.len(), + 1, + "identical hard_filter from two edges should dedup to one, got: {:?}", + result.hard_filters + ); + assert_eq!(result.hard_filters[0], same_filter); + assert_eq!(result.tables, vec!["s1_src".to_string()]); + } + + #[test] + fn non_routine_node_returns_none() { + let mut graph = CodeGraph::new(); + let table = graph.add_node(Node::Table { + schema: None, + name: "t".to_string(), + explicit: true, + system: false, + location: None, + columns: Box::new(Vec::new()), + partition_by: None, + distribute_by: None, + tablespace: None, + temporary: false, + unlogged: false, + ddl_source: None, + }); + assert!(column_analysis_of_routine(&graph, table, None).is_none()); + } +} diff --git a/src/graph/lineage.rs b/src/graph/lineage.rs index 9d94cb5..9b45446 100644 --- a/src/graph/lineage.rs +++ b/src/graph/lineage.rs @@ -1900,6 +1900,59 @@ pub fn format_column_lineage_json( }) } +/// Outcome of [`parse_lineage_target`]: a target string resolves to either a bare table +/// (table-level lineage) or a `table.column` pair (column-level lineage). +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum ParsedLineageTarget { + Table(String), + Column(String, String), +} + +/// Handles the same three grammars as the CLI: a node-key (`table:schema.table`, left +/// alone — table-level), `table.column` (column-level, but only once the table half is +/// confirmed to exist unambiguously), and a bare table name (table-level). +pub(crate) fn parse_lineage_target( + graph: &CodeGraph, + target: &str, +) -> Result { + // Node keys (`table:schema.table`) must not be last-dot-split — the final dot is + // part of the key, not a `table.column` separator. + let (table_name, column_name) = if crate::graph::key::split_type_prefix(target).is_some() { + (target, None) + } else { + match target.rsplit_once('.') { + Some((table, column)) if !table.is_empty() && !column.is_empty() => { + (table, Some(column)) + } + Some(_) => { + return Err(format!( + "Invalid target format: {}. Use 'table', 'table.column', or a node key like 'table:schema.table'", + target + )); + } + None => (target, None), + } + }; + + // A missing table half means the split was probably `schema.table`: reinterpret the + // whole target as a table reference. An ambiguous half stops with a qualifier hint. + match column_name { + Some(column) => match lookup_table_node(graph, table_name) { + TableLookup::Found(_) => Ok(ParsedLineageTarget::Column( + table_name.to_string(), + column.to_string(), + )), + TableLookup::Ambiguous => Err(format!( + "table '{table_name}' is ambiguous across schemas — qualify it as \ + 'schema.{table_name}' for table-level, or 'schema.{table_name}.{column}' \ + for column-level lineage" + )), + TableLookup::Missing => Ok(ParsedLineageTarget::Table(target.to_string())), + }, + None => Ok(ParsedLineageTarget::Table(table_name.to_string())), + } +} + #[cfg(test)] mod tests { use super::*; @@ -1909,6 +1962,105 @@ mod tests { use std::path::PathBuf; use std::sync::Arc; + /// #165 P1: a bare table name with no dot parses as a table-level target. + #[test] + fn parse_lineage_target_bare_table_is_table() { + let graph = CodeGraph::new(); + let result = parse_lineage_target(&graph, "orders").expect("should parse"); + assert_eq!(result, ParsedLineageTarget::Table("orders".to_string())); + } + + /// #165 P1: a node-key (`table:schema.table`) is left alone as a table target, not + /// split on its dot. + #[test] + fn parse_lineage_target_node_key_is_table() { + let graph = CodeGraph::new(); + let result = parse_lineage_target(&graph, "table:public.orders").expect("should parse"); + assert_eq!( + result, + ParsedLineageTarget::Table("table:public.orders".to_string()) + ); + } + + /// #165 P1: `table.column` where `table` resolves uniquely parses as column-level. + #[test] + fn parse_lineage_target_table_dot_column_when_table_exists_is_column() { + let mut graph = CodeGraph::new(); + graph.add_node(table_node("orders", &["id", "amt"])); + let result = parse_lineage_target(&graph, "orders.amt").expect("should parse"); + assert_eq!( + result, + ParsedLineageTarget::Column("orders".to_string(), "amt".to_string()) + ); + } + + /// #165 P1: when the table half doesn't exist, the recent fix (#154) reinterprets the + /// whole string as a table reference instead of guessing a column split — e.g. + /// `schema.table` with no such table gets treated as one table-level target. + #[test] + fn parse_lineage_target_missing_table_reinterprets_whole_string_as_table() { + let graph = CodeGraph::new(); + let result = parse_lineage_target(&graph, "public.orders").expect("should parse"); + assert_eq!( + result, + ParsedLineageTarget::Table("public.orders".to_string()) + ); + } + + /// #165 P1: an ambiguous bare table name (same name in 2+ schemas) errors instead of + /// guessing. + #[test] + fn parse_lineage_target_ambiguous_table_errors() { + let mut graph = CodeGraph::new(); + graph.add_node(Node::Table { + schema: Some("s1".to_string()), + name: "orders".to_string(), + explicit: true, + system: false, + location: None, + columns: Box::new(Vec::new()), + partition_by: None, + distribute_by: None, + tablespace: None, + temporary: false, + unlogged: false, + ddl_source: None, + }); + graph.add_node(Node::Table { + schema: Some("s2".to_string()), + name: "orders".to_string(), + explicit: true, + system: false, + location: None, + columns: Box::new(Vec::new()), + partition_by: None, + distribute_by: None, + tablespace: None, + temporary: false, + unlogged: false, + ddl_source: None, + }); + let err = + parse_lineage_target(&graph, "orders.amt").expect_err("ambiguous table should error"); + assert!( + err.contains("ambiguous"), + "error should mention ambiguity, got: {err}" + ); + } + + /// #165 P1: an empty column half (`table.`) is an invalid-format error, not silently + /// treated as a table. + #[test] + fn parse_lineage_target_trailing_dot_errors() { + let graph = CodeGraph::new(); + let err = parse_lineage_target(&graph, "orders.") + .expect_err("trailing dot with empty column should error"); + assert!( + err.contains("Invalid target format"), + "error should mention invalid format, got: {err}" + ); + } + fn table_node(name: &str, cols: &[&str]) -> Node { Node::Table { schema: None, diff --git a/src/graph/mod.rs b/src/graph/mod.rs index 3452f5f..906e70c 100644 --- a/src/graph/mod.rs +++ b/src/graph/mod.rs @@ -2,6 +2,7 @@ use crate::parser::ColumnAnalysis; pub mod builder; pub mod cluster; +pub mod columns; pub mod conflict; pub mod format; pub mod inspect; diff --git a/src/graph/store.rs b/src/graph/store.rs index a337399..3e9ab02 100644 --- a/src/graph/store.rs +++ b/src/graph/store.rs @@ -19,7 +19,18 @@ const STORE_MAGIC: [u8; 9] = *b"CWEBSTORE"; /// GraphStore on-disk format version. Bump when the serialized struct layout /// changes. Validated in the file header (post-header era files) and again in /// `GraphStore.version` after deserialize (legacy files + belt-and-suspenders). -const STORE_VERSION: u32 = 9; +/// +/// v10: `merge_table_access_edges` now unions ALL `ColumnAnalysis` diagnostic +/// Vec fields (join_conditions/hard_filters/enum_mappings/select_into/ +/// insert_columns/update_columns/column_refs) instead of keeping only the +/// first merged edge's, plus a reserved (serde-default) `HardFilter.transform` +/// slot for future function-wrapped-column filters. Refs #165, #169. +/// v11: adds the `procedure_predicates` side-table populated by the branch-aware +/// PL/SQL predicate pass (#167). +/// v12: `PredicateClause` gains a reserved (serde-default) `transform` slot for +/// column-transform conditions (e.g. `substr(col,1,2) = 'x'`), mirroring +/// `HardFilter.transform`. Refs #167, #169. +const STORE_VERSION: u32 = 12; /// Pre-computed lightweight summary of a graph node for fast listing/filtering. #[derive(Debug, Clone, Serialize, Deserialize)] @@ -66,6 +77,9 @@ pub struct GraphStore { /// Built from ogsql-parser AST for O(1) lookup of FOR UPDATE / FOR SHARE etc. #[serde(default)] lock_clause_index: HashMap>, + /// Routine NodeKey string (`proc:...` / `func:...`) → PL IF/CASE predicates. + #[serde(default)] + pub procedure_predicates: HashMap>, } #[allow(dead_code)] @@ -90,6 +104,7 @@ impl GraphStore { edge_category_index: HashMap::new(), sql_fingerprint_index: HashMap::new(), lock_clause_index: HashMap::new(), + procedure_predicates: HashMap::new(), } } @@ -321,9 +336,17 @@ impl GraphStore { edge_category_index, sql_fingerprint_index, lock_clause_index, + procedure_predicates: HashMap::new(), } } + pub fn set_procedure_predicates( + &mut self, + predicates: HashMap>, + ) { + self.procedure_predicates = predicates; + } + pub fn graph(&self) -> &CodeGraph { &self.graph } @@ -1319,6 +1342,14 @@ impl GraphStore { let mut merged = GraphStore::new(merged_name); for store in &stores { + for (key, predicates) in &store.procedure_predicates { + let entry = merged.procedure_predicates.entry(key.clone()).or_default(); + for predicate in predicates { + if !entry.contains(predicate) { + entry.push(predicate.clone()); + } + } + } let mut idx_map: HashMap = HashMap::new(); // Build a reverse-relaxed index: maps relaxed(key) → existing index @@ -2396,6 +2427,56 @@ mod tests { assert_eq!(loaded.unwrap().version, STORE_VERSION); } + #[test] + fn procedure_predicates_survive_bincode_round_trip() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("predicates.bincode"); + let mut store = GraphStore::from_graph("roundtrip", CodeGraph::new()); + store.procedure_predicates.insert( + "proc:p".to_string(), + vec![crate::parser::PlPredicate { + id: "B001".to_string(), + line: 3, + origin: "IF r.x = '1'".to_string(), + kind: crate::parser::PredicateKind::If, + confidence: crate::parser::Confidence::High, + table_predicate: Some(crate::parser::TablePredicate { + table: "t".to_string(), + clauses: vec![ + crate::parser::PredicateClause { + column: "x".to_string(), + op: crate::parser::FilterOperator::Eq, + value: crate::parser::FilterValue::String("1".to_string()), + transform: None, + }, + // #167/#169: a transform-carrying clause must also survive the + // bincode round-trip via the hand-written `is_human_readable` + // Serialize impl (skip_serializing_if would corrupt the layout). + crate::parser::PredicateClause { + column: "stock_kind".to_string(), + op: crate::parser::FilterOperator::Eq, + value: crate::parser::FilterValue::String("05".to_string()), + transform: Some(crate::parser::FilterTransform { + fn_name: "substr".to_string(), + args: vec![ + crate::parser::FilterValue::Integer(1), + crate::parser::FilterValue::Integer(2), + ], + }), + }, + ], + }), + needs_review: None, + param_table_hint: None, + }], + ); + + store.save_bincode(&path).unwrap(); + let loaded = GraphStore::load_bincode(&path).unwrap(); + + assert_eq!(loaded.procedure_predicates, store.procedure_predicates); + } + /// A store written by the previous layout (version 7, before `ColumnAnalysis.read_tables`, /// issue #147) must be rejected by the version gate with the friendly message, not fail /// with a raw bincode deserialize error after passing the check. diff --git a/src/main.rs b/src/main.rs index cd4116a..cff365f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -178,6 +178,15 @@ struct ImpactResult { downstream: Vec, } +#[derive(Serialize)] +struct PredicatesResult { + schema_version: u32, + procedure: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + package: Option, + predicates: Vec, +} + const LOGO: &str = r#" ██████╗ ██████╗ ██████╗ ███████╗ ██╗ ██╗ ███████╗ ██████╗ ██╔════╝ ██╔═══██╗ ██╔══██╗ ██╔════╝ ██║ ██║ ██╔════╝ ██╔══██╗ @@ -410,6 +419,47 @@ enum Commands { project: PathBuf, }, + /// #165: per-procedure/per-package aggregated column-analysis export (hard filters, + /// joins, SELECT INTO, enum/column mappings) — the machine-readable entry point for + /// test-data/mock generation. + #[command(group(clap::ArgGroup::new("columns_target").required(true).multiple(false)))] + Columns { + /// Procedure or function name to aggregate (substring match, same as `trace`) + #[arg(long, group = "columns_target")] + procedure: Option, + + /// Package name to aggregate over all of its contained procedures/functions + #[arg(long, group = "columns_target")] + package: Option, + + /// Narrow output to one table's diagnostics (case-insensitive) + #[arg(long)] + table: Option, + + /// Output format (only "json" is supported today) + #[arg(long, default_value = "json", value_parser = ["json"])] + format: String, + + /// Project directory (default: current directory) + #[arg(short, long, default_value = ".")] + project: PathBuf, + }, + + /// #167: PL IF/CASE predicates resolved to table columns. + Predicates { + /// Procedure or function name (substring match, same as `columns`) + #[arg(long)] + procedure: String, + + /// Output format (only "json" is supported today) + #[arg(long, default_value = "json", value_parser = ["json"])] + format: String, + + /// Project directory (default: current directory) + #[arg(short, long, default_value = ".")] + project: PathBuf, + }, + /// Show project statistics Stats { /// Project directory (default: current directory) @@ -944,6 +994,18 @@ fn run() -> Result<()> { }) => cmd_lineage( &target, &direction, depth, &format, &view, flow_only, &project, ), + Some(Commands::Columns { + procedure, + package, + table, + format, + project, + }) => cmd_columns(procedure, package, table, &format, &project), + Some(Commands::Predicates { + procedure, + format, + project, + }) => cmd_predicates(&procedure, &format, &project), Some(Commands::Stats { project }) => cmd_stats(&project), Some(Commands::Files { project }) => cmd_files(&project), Some(Commands::Nodes { @@ -1564,49 +1626,36 @@ fn cmd_lineage( ); } - // Node keys (`table:schema.table`) must not be last-dot-split — the final dot is - // part of the key, not a `table.column` separator. - let (table_name, column_name) = if graph::key::split_type_prefix(target).is_some() { - (target, None) - } else { - match target.rsplit_once('.') { - Some((table, column)) if !table.is_empty() && !column.is_empty() => { - (table, Some(column)) - } - Some(_) => { - eprintln!( - "Invalid target format: {}. Use 'table', 'table.column', or a node key like 'table:schema.table'", - target - ); - return Ok(()); + let parsed_target = match graph::lineage::parse_lineage_target(graph, target) { + Ok(parsed) => parsed, + Err(message) => { + if message.starts_with("table '") { + eprintln!("error: {}", message); + } else { + eprintln!("{}", message); } - None => (target, None), + return Ok(()); } }; - - // A missing table half means the split was probably `schema.table`: reinterpret the - // whole target as a table reference. An ambiguous half stops with a qualifier hint. - let (table_name, column_name) = match column_name { - Some(column) => match graph::lineage::lookup_table_node(graph, table_name) { - graph::lineage::TableLookup::Found(_) => (table_name, Some(column)), - graph::lineage::TableLookup::Ambiguous => { - eprintln!( - "error: table '{}' is ambiguous across schemas — qualify it as \ - 'schema.{table_name}' for table-level, or 'schema.{table_name}.{column}' \ - for column-level lineage", - table_name - ); - return Ok(()); - } - graph::lineage::TableLookup::Missing => { - eprintln!( - "note: no table '{}' found — interpreting '{}' as a table reference (for column-level lineage, the table must exist)", - table_name, target - ); - (target, None) + let (table_name, column_name) = match &parsed_target { + graph::lineage::ParsedLineageTarget::Column(table, column) => { + (table.as_str(), Some(column.as_str())) + } + graph::lineage::ParsedLineageTarget::Table(table) => { + if let Some((prefix, suffix)) = target.rsplit_once('.') { + if !prefix.is_empty() + && !suffix.is_empty() + && graph::key::split_type_prefix(target).is_none() + && table == target + { + eprintln!( + "note: no table '{}' found — interpreting '{}' as a table reference (for column-level lineage, the table must exist)", + prefix, target + ); + } } - }, - None => (table_name, None), + (table.as_str(), None) + } }; // Parse direction up front — both the table and column paths need it. `None` means @@ -1802,6 +1851,194 @@ fn cmd_lineage( Ok(()) } +/// #165: `codeweb columns --procedure X` / `--package Y` — aggregate every `TableAccess` +/// diagnostic a routine (or a whole package's routines) touches. Errors +/// cleanly (non-zero exit, message on stderr) on an unresolved name rather than printing +/// an empty result — silent empty output would be indistinguishable from "this routine +/// really has no constraints". +fn cmd_columns( + procedure: Option, + package: Option, + table: Option, + format: &str, + project: &Path, +) -> Result<()> { + let mut proj = project::Project::find(project)?; + let store = proj.load_store()?; + let graph = store.graph(); + + // Stores built before the diagnostic-field union landed (STORE_VERSION 10, #165) + // under-report hard_filters/join_conditions for routines that touch the same table + // in more than one statement — only the first statement's diagnostics survive. + if store.version < 10 { + eprintln!( + "note: store version {} predates full column-analysis diagnostics (v10) — run `codeweb analyze` to rebuild.", + store.version + ); + } + + let table_filter = table.as_deref(); + + let result = if let Some(name) = procedure { + let resolved = store.resolve_single_node( + &name, + crate::graph::search::MatchMode::Substring, + false, + true, + ); + let idx = match resolved { + crate::graph::search::ResolveResult::Single(idx, _) => idx, + crate::graph::search::ResolveResult::Empty => { + return Err(error::CodeWebError::ExportError { + message: format!("No procedure or function found matching '{}'", name), + }); + } + _ => { + return Err(error::CodeWebError::ExportError { + message: format!("Ambiguous match for '{}'", name), + }); + } + }; + if !matches!( + &graph[idx], + graph::Node::Procedure { .. } | graph::Node::Function { .. } + ) { + return Err(error::CodeWebError::ExportError { + message: format!("'{}' is not a procedure or function", name), + }); + } + graph::columns::column_analysis_of_routine(graph, idx, table_filter).ok_or_else(|| { + error::CodeWebError::ExportError { + message: format!("failed to aggregate column analysis for '{}'", name), + } + })? + } else { + // clap's `columns_target` ArgGroup (required, mutually exclusive) guarantees + // exactly one of `procedure`/`package` is `Some` by the time we get here. + let name = package.expect("clap group guarantees procedure or package is set"); + let resolved = store.resolve_single_node( + &name, + crate::graph::search::MatchMode::Substring, + false, + true, + ); + let idx = match resolved { + crate::graph::search::ResolveResult::Single(idx, _) => idx, + crate::graph::search::ResolveResult::Empty => { + return Err(error::CodeWebError::ExportError { + message: format!("No package found matching '{}'", name), + }); + } + _ => { + return Err(error::CodeWebError::ExportError { + message: format!("Ambiguous match for '{}'", name), + }); + } + }; + if !matches!(&graph[idx], graph::Node::Package { .. }) { + return Err(error::CodeWebError::ExportError { + message: format!("'{}' is not a package", name), + }); + } + graph::columns::column_analysis_of_package(graph, idx, table_filter).ok_or_else(|| { + error::CodeWebError::ExportError { + message: format!("failed to aggregate column analysis for package '{}'", name), + } + })? + }; + + match format { + "json" => { + let json_str = serde_json::to_string_pretty(&result).map_err(|e| { + error::CodeWebError::ExportError { + message: format!("Failed to format JSON: {}", e), + } + })?; + println_stdout!("{}", json_str); + } + other => { + return Err(error::CodeWebError::ExportError { + message: format!("Unknown format: {}. Use 'json'", other), + }); + } + } + + Ok(()) +} + +fn cmd_predicates(procedure: &str, format: &str, project: &Path) -> Result<()> { + let mut proj = project::Project::find(project)?; + let store = proj.load_store()?; + if store.version < 12 { + eprintln!( + "note: store version {} predates PL predicate extraction (v12) — run `codeweb analyze` to rebuild.", + store.version + ); + } + + let resolved = store.resolve_single_node( + procedure, + crate::graph::search::MatchMode::Substring, + false, + true, + ); + let idx = match resolved { + crate::graph::search::ResolveResult::Single(idx, _) => idx, + crate::graph::search::ResolveResult::Empty => { + return Err(error::CodeWebError::ExportError { + message: format!("No procedure or function found matching '{}'", procedure), + }); + } + _ => { + return Err(error::CodeWebError::ExportError { + message: format!("Ambiguous match for '{}'", procedure), + }); + } + }; + if !matches!( + &store.graph()[idx], + graph::Node::Procedure { .. } | graph::Node::Function { .. } + ) { + return Err(error::CodeWebError::ExportError { + message: format!("'{}' is not a procedure or function", procedure), + }); + } + let key = crate::graph::key::NodeKey::from_node(&store.graph()[idx]).to_string(); + let predicates = store + .procedure_predicates + .get(&key) + .cloned() + .unwrap_or_default(); + let (procedure_name, package) = match &store.graph()[idx] { + graph::Node::Procedure { id, .. } | graph::Node::Function { id, .. } => { + (id.name.clone(), id.package.clone()) + } + _ => unreachable!("routine node type checked above"), + }; + let result = PredicatesResult { + schema_version: 1, + procedure: procedure_name, + package, + predicates, + }; + match format { + "json" => println_stdout!( + "{}", + serde_json::to_string_pretty(&result).map_err(|error| { + error::CodeWebError::ExportError { + message: format!("Failed to format JSON: {}", error), + } + })? + ), + other => { + return Err(error::CodeWebError::ExportError { + message: format!("Unknown format: {}. Use 'json'", other), + }); + } + } + Ok(()) +} + fn cmd_stats(project: &Path) -> Result<()> { let mut proj = project::Project::find(project)?; let store = proj.load_store()?; diff --git a/src/mcp/tools.rs b/src/mcp/tools.rs index cc04070..54c3294 100644 --- a/src/mcp/tools.rs +++ b/src/mcp/tools.rs @@ -99,6 +99,25 @@ pub struct QueryParams { pub spec: serde_json::Value, } +#[derive(Deserialize, schemars::JsonSchema)] +pub struct ColumnAnalysisParams { + #[serde(default)] + pub procedure: Option, + #[serde(default)] + pub package: Option, + #[serde(default)] + pub table: Option, +} + +#[derive(Deserialize, schemars::JsonSchema)] +pub struct LineageParams { + pub target: String, + #[serde(default)] + pub direction: Option, + #[serde(default)] + pub depth: Option, +} + // ── Helper functions ── fn tree_nodes_to_json(nodes: &[traverse::TreeNode], graph: &CodeGraph) -> Vec { @@ -529,6 +548,196 @@ impl McpState { } } } + + /// Aggregate per-procedure/per-package column analysis + #[tool( + description = "Aggregate hard filters, join conditions, SELECT INTO mappings, and enum/column mappings for a procedure or package — same JSON schema as `codeweb columns --format json` (issue #165). Provide exactly one of procedure or package; table optionally narrows to one table's diagnostics." + )] + fn codeweb_column_analysis( + &self, + Parameters(params): Parameters, + ) -> String { + if self.graph_empty() { + return self.empty_graph_response(); + } + if params.procedure.is_some() == params.package.is_some() { + let err = serde_json::json!({ + "error": "exactly one of 'procedure' or 'package' is required", + }); + return serde_json::to_string(&err).unwrap_or_default(); + } + + let store = self.store(); + let graph = self.graph(); + let table_filter = params.table.as_deref(); + + let result = if let Some(name) = ¶ms.procedure { + let idx = match resolve_node(store, name, true) { + Ok(idx) => idx, + Err(msg) => return msg, + }; + if !matches!(&graph[idx], Node::Procedure { .. } | Node::Function { .. }) { + let err = serde_json::json!({ + "error": format!("'{}' is not a procedure or function", name), + }); + return serde_json::to_string(&err).unwrap_or_default(); + } + crate::graph::columns::column_analysis_of_routine(graph, idx, table_filter) + } else { + let name = params.package.as_ref().expect("checked exactly-one above"); + let idx = match resolve_node(store, name, true) { + Ok(idx) => idx, + Err(msg) => return msg, + }; + if !matches!(&graph[idx], Node::Package { .. }) { + let err = serde_json::json!({"error": format!("'{}' is not a package", name)}); + return serde_json::to_string(&err).unwrap_or_default(); + } + crate::graph::columns::column_analysis_of_package(graph, idx, table_filter) + }; + + match result { + Some(analysis) => serde_json::to_string(&analysis).unwrap_or_default(), + None => { + let err = serde_json::json!({"error": "failed to aggregate column analysis"}); + serde_json::to_string(&err).unwrap_or_default() + } + } + } + + /// Table-level or column-level lineage for a target + #[tool( + description = "Table-level or column-level lineage for a target ('table', 'table.column', or a node key like 'table:schema.table') — same functions/JSON shape as `codeweb lineage --format json` (issue #165 P1). direction: upstream/downstream/both (default both); depth default 5." + )] + fn codeweb_lineage(&self, Parameters(params): Parameters) -> String { + if self.graph_empty() { + return self.empty_graph_response(); + } + let graph = self.graph(); + let store = self.store(); + let depth = params.depth.unwrap_or(5); + let direction = params.direction.as_deref().unwrap_or("both"); + + let dir_spec = match direction.to_lowercase().as_str() { + "upstream" => Some(crate::graph::lineage::LineageDirection::Upstream), + "downstream" => Some(crate::graph::lineage::LineageDirection::Downstream), + "both" => None, + _ => { + let err = serde_json::json!({ + "error": format!( + "Unknown direction: {}. Use 'upstream', 'downstream' or 'both'", + direction + ), + }); + return serde_json::to_string(&err).unwrap_or_default(); + } + }; + + let parsed = match crate::graph::lineage::parse_lineage_target(graph, ¶ms.target) { + Ok(p) => p, + Err(e) => { + let err = serde_json::json!({"error": e}); + return serde_json::to_string(&err).unwrap_or_default(); + } + }; + + match parsed { + crate::graph::lineage::ParsedLineageTarget::Column(table, column) => { + let render = |dir: crate::graph::lineage::LineageDirection| { + let node = + crate::graph::lineage::lineage_column(graph, &table, &column, dir, depth); + crate::graph::lineage::format_column_lineage_json(&node, graph) + }; + let json = match dir_spec { + Some(dir) => render(dir), + None => serde_json::json!({ + "upstream": render(crate::graph::lineage::LineageDirection::Upstream), + "downstream": render(crate::graph::lineage::LineageDirection::Downstream), + }), + }; + serde_json::to_string(&json).unwrap_or_default() + } + crate::graph::lineage::ParsedLineageTarget::Table(table_name) => { + let table_idx = match resolve_node(store, &table_name, false) { + Ok(idx) => idx, + Err(msg) => return msg, + }; + if !matches!(&graph[table_idx], Node::Table { .. } | Node::View { .. }) { + let err = serde_json::json!({ + "error": format!("'{}' is not a table or view", table_name), + }); + return serde_json::to_string(&err).unwrap_or_default(); + } + + let cfg = crate::graph::lineage::LineageConfig::default(); + let opts = crate::graph::lineage::DisplayOptions::new( + crate::graph::lineage::LineageView::Tree, + false, + ); + let json = match dir_spec { + Some(dir) => { + let node = crate::graph::lineage::lineage_table( + graph, table_idx, dir, depth, &cfg, + ); + crate::graph::lineage::format_lineage_json(&node, graph, &opts) + } + None => { + let up = crate::graph::lineage::lineage_table( + graph, + table_idx, + crate::graph::lineage::LineageDirection::Upstream, + depth, + &cfg, + ); + let down = crate::graph::lineage::lineage_table( + graph, + table_idx, + crate::graph::lineage::LineageDirection::Downstream, + depth, + &cfg, + ); + serde_json::json!({ + "upstream": crate::graph::lineage::format_lineage_json(&up, graph, &opts), + "downstream": crate::graph::lineage::format_lineage_json(&down, graph, &opts), + }) + } + }; + serde_json::to_string(&json).unwrap_or_default() + } + } + } +} + +/// Resolve `name` (substring match, same as `trace`/CLI `columns`/`lineage`) to a single +/// node, or a pre-serialized `{"error": ...}` JSON string on empty/ambiguous match — +/// shared by `codeweb_column_analysis` and `codeweb_lineage`. +fn resolve_node( + store: &GraphStore, + name: &str, + fail_on_multiple: bool, +) -> Result { + match store.resolve_single_node( + name, + crate::graph::search::MatchMode::Substring, + false, + fail_on_multiple, + ) { + crate::graph::search::ResolveResult::Single(idx, _) => Ok(idx), + crate::graph::search::ResolveResult::Empty => { + let err = serde_json::json!({"error": format!("No nodes matching '{}'", name)}); + Err(serde_json::to_string(&err).unwrap_or_default()) + } + crate::graph::search::ResolveResult::Ambiguous => { + let count = store + .search_nodes_with_mode(name, crate::graph::search::MatchMode::Substring) + .len(); + let err = serde_json::json!({ + "error": format!("Ambiguous match: {} candidates for '{}'", count, name) + }); + Err(serde_json::to_string(&err).unwrap_or_default()) + } + crate::graph::search::ResolveResult::Multiple(_) => unreachable!("all_matches is false"), + } } // ── ServerHandler implementation ── @@ -544,6 +753,8 @@ use rmcp::ServerHandler; codeweb_trace to follow call chains bidirectionally, \ codeweb_search_sql to find SQL by text content, \ codeweb_node_detail for properties + callers + callees, \ - codeweb_query for complex multi-step traversals via JSON QuerySpec." + codeweb_query for complex multi-step traversals via JSON QuerySpec, \ + codeweb_column_analysis for per-procedure/per-package hard filters, joins, and column mappings, \ + codeweb_lineage for table/column-level lineage tracing." )] impl ServerHandler for McpState {} diff --git a/src/parser/extractor.rs b/src/parser/extractor.rs index 236aab2..34c6af2 100644 --- a/src/parser/extractor.rs +++ b/src/parser/extractor.rs @@ -17,7 +17,7 @@ pub struct ProcedureBodySql { pub line: Option, /// The parsed statement AST from the ORIGINAL procedure-body parse, when the source /// was a typed SQL statement. Walking this (instead of re-parsing `sql_text`) keeps - /// procedure context such as declared-variable classification (issue #147). + /// procedure context such as declared-variable classification. pub statement: Option, } @@ -1146,7 +1146,7 @@ impl TableAccessExtractor { /// Walk the expression-bearing fields of a `SELECT` (targets, WHERE, /// HAVING, GROUP BY, ORDER BY, …) and extract reads from any subqueries /// found inside them. This captures subquery table references regardless - /// of the outer statement kind (#140) — before this, only UPDATE/DELETE + /// of the outer statement kind; previously only UPDATE/DELETE /// contexts descended into WHERE expressions, so tables referenced in /// SELECT/INSERT subqueries were silently dropped. /// @@ -1654,7 +1654,7 @@ impl Visitor for TableAccessExtractor { // Subqueries in expression positions (WHERE / HAVING / SELECT list / // GROUP BY / ORDER BY / …) must be walked while this select's CTE scope - // is still active, or their table references are silently dropped (#140). + // is still active, or their table references are silently dropped. self.walk_select_expr_subqueries(select); self.pop_cte_scope(); @@ -1690,7 +1690,7 @@ impl Visitor for TableAccessExtractor { // Non-SELECT sources can still carry subqueries in their expressions // (e.g. `INSERT INTO t VALUES ((SELECT …))`); walk them while this - // insert's CTE scope is active (#140). + // insert's CTE scope is active. match &insert.source { ogsql_parser::ast::InsertSource::Values(rows) => { for row in rows { @@ -1786,12 +1786,12 @@ pub struct ColumnAnalysis { /// Per-column data flow: which sources feed each written column. #[serde(default)] pub column_mappings: Vec, - /// Names of the OTHER tables touched by the same statement as this edge (issue #147). + /// Names of the other tables touched by the same statement as this edge. /// Populated by the builder on every TableAccess edge of a statement; it is what lets /// lineage restrict hops to tables read in the same statement as a write, instead of /// connecting all of a routine's reads to all of its writes. /// - /// `None` = not populated (store built before #147) — lineage falls back to + /// `None` means not populated by an older store, so lineage falls back to /// connecting all of a routine's reads/writes. `Some(vec![])` = populated and the /// statement genuinely touches no other table (e.g. a bare `UPDATE t SET ...`) — a /// legitimate empty hop set, NOT a reason to fall back. @@ -1874,7 +1874,7 @@ pub struct CursorColumn { } /// Column reference. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] pub struct ColumnRef { /// Resolved table name (via alias_map). None if unresolvable or unprefixed. pub resolved_table: Option, @@ -1901,7 +1901,7 @@ pub enum ColumnContext { } /// Equi-join condition. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] pub struct JoinCondition { pub left_table: String, pub left_column: String, @@ -1911,7 +1911,7 @@ pub struct JoinCondition { pub source: JoinConditionSource, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] pub enum JoinType { Inner, Left, @@ -1920,22 +1920,60 @@ pub enum JoinType { Cross, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] pub enum JoinConditionSource { ImplicitWhere, ExplicitOn, + /// One side is a `%ROWTYPE` record field resolved to its underlying cursor + /// source column (via `resolve_record_field`), not a plain SQL table alias. Kept + /// distinct from `ImplicitWhere`/`ExplicitOn` so downstream consumers can weigh the + /// confidence of a derived cross-table key differently from a literal equi-join. + RecordField, } /// WHERE clause hard-coded filter. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +/// +/// Serialization branches by format because bincode requires a fixed field count. +#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Deserialize)] pub struct HardFilter { pub table: Option, pub column: String, pub operator: FilterOperator, pub value: FilterValue, + #[serde(default)] + pub transform: Option, +} + +impl serde::Serialize for HardFilter { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let omit_transform = serializer.is_human_readable() && self.transform.is_none(); + let field_count = if omit_transform { 4 } else { 5 }; + let mut state = serializer.serialize_struct("HardFilter", field_count)?; + state.serialize_field("table", &self.table)?; + state.serialize_field("column", &self.column)?; + state.serialize_field("operator", &self.operator)?; + state.serialize_field("value", &self.value)?; + if !omit_transform { + state.serialize_field("transform", &self.transform)?; + } + state.end() + } +} + +/// Descriptor of a whitelisted pure column transform in a filter. +/// Serialized as {"fn": "substr", "args": [1, 2]} per issue schema. +#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +pub struct FilterTransform { + #[serde(rename = "fn")] + pub fn_name: String, + pub args: Vec, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] pub enum FilterOperator { Eq, Neq, @@ -1951,7 +1989,7 @@ pub enum FilterOperator { IsNotNull, } -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] pub enum FilterValue { String(String), Integer(i64), @@ -1963,7 +2001,7 @@ pub enum FilterValue { } /// CASE/DECODE enum value mapping. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] pub struct EnumMapping { pub column: String, pub table_alias: Option, @@ -1972,28 +2010,28 @@ pub struct EnumMapping { } /// SELECT INTO variable assignment. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] pub struct SelectIntoMapping { pub column_expr: String, pub into_variable: String, } /// INSERT column info. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] pub struct InsertColumnInfo { pub table: String, pub columns: Vec, } /// UPDATE SET column info. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] pub struct UpdateColumnInfo { pub table: String, pub set_columns: Vec, } /// Procedure-level variable context collected by a pre-pass and seeded into per-statement -/// walks (issue #147): cursor sources, FETCH chains, `%ROWTYPE` records and `%TYPE` +/// walks: cursor sources, FETCH chains, `%ROWTYPE` records and `%TYPE` /// anchors. Per-statement walks would otherwise lose these cross-statement bindings /// (a cursor is declared in DECLARE, fetched in one statement, consumed in another). #[derive(Debug, Clone, Default)] @@ -2073,7 +2111,7 @@ impl ColumnAccessExtractor { } /// Build an extractor pre-seeded with procedure variable context collected by a - /// procedure-level pass (issue #147): a per-statement walk would otherwise lose the + /// procedure-level pass because a per-statement walk would otherwise lose the /// cross-statement cursor → FETCH → INSERT chain and `%ROWTYPE`/`%TYPE` anchors. pub fn new_with_context(ctx: &ProcedureVarContext) -> Self { let mut ext = Self::new(); @@ -2379,6 +2417,13 @@ impl ColumnAccessExtractor { join_type, is_explicit_on, ); + // A `%ROWTYPE` record field also parses as a + // multi-part ColumnRef, so `extract_join_condition` above + // silently no-ops on it (its alias prefix fails + // `resolve_alias`, a plain-table-alias lookup). Retry via + // `resolve_record_field` — a no-op itself when neither side + // is a registered record field. + self.extract_record_field_join(&l_names, &r_names, join_type); // Also add column refs in join context self.add_column_ref(&l_names, Some(ColumnContext::JoinCondition)); self.add_column_ref(&r_names, Some(ColumnContext::JoinCondition)); @@ -2390,6 +2435,24 @@ impl ColumnAccessExtractor { if let Some(val) = literal_to_filter_value(left) { self.add_hard_filter(&col_names, FilterOperator::Eq, val); } + } else if let Some((col_names, transform)) = column_transform_of(left) { + if let Some(val) = literal_to_filter_value(right) { + self.add_hard_filter_with_transform( + &col_names, + FilterOperator::Eq, + val, + Some(transform), + ); + } + } else if let Some((col_names, transform)) = column_transform_of(right) { + if let Some(val) = literal_to_filter_value(left) { + self.add_hard_filter_with_transform( + &col_names, + FilterOperator::Eq, + val, + Some(transform), + ); + } } } "<>" | "!=" => { @@ -2401,6 +2464,24 @@ impl ColumnAccessExtractor { if let Some(val) = literal_to_filter_value(left) { self.add_hard_filter(&col_names, FilterOperator::Neq, val); } + } else if let Some((col_names, transform)) = column_transform_of(left) { + if let Some(val) = literal_to_filter_value(right) { + self.add_hard_filter_with_transform( + &col_names, + FilterOperator::Neq, + val, + Some(transform), + ); + } + } else if let Some((col_names, transform)) = column_transform_of(right) { + if let Some(val) = literal_to_filter_value(left) { + self.add_hard_filter_with_transform( + &col_names, + FilterOperator::Neq, + val, + Some(transform), + ); + } } } ">" => { @@ -2408,6 +2489,15 @@ impl ColumnAccessExtractor { if let Some(val) = literal_to_filter_value(right) { self.add_hard_filter(&col_names, FilterOperator::Gt, val); } + } else if let Some((col_names, transform)) = column_transform_of(left) { + if let Some(val) = literal_to_filter_value(right) { + self.add_hard_filter_with_transform( + &col_names, + FilterOperator::Gt, + val, + Some(transform), + ); + } } } ">=" => { @@ -2415,6 +2505,15 @@ impl ColumnAccessExtractor { if let Some(val) = literal_to_filter_value(right) { self.add_hard_filter(&col_names, FilterOperator::Gte, val); } + } else if let Some((col_names, transform)) = column_transform_of(left) { + if let Some(val) = literal_to_filter_value(right) { + self.add_hard_filter_with_transform( + &col_names, + FilterOperator::Gte, + val, + Some(transform), + ); + } } } "<" => { @@ -2422,6 +2521,15 @@ impl ColumnAccessExtractor { if let Some(val) = literal_to_filter_value(right) { self.add_hard_filter(&col_names, FilterOperator::Lt, val); } + } else if let Some((col_names, transform)) = column_transform_of(left) { + if let Some(val) = literal_to_filter_value(right) { + self.add_hard_filter_with_transform( + &col_names, + FilterOperator::Lt, + val, + Some(transform), + ); + } } } "<=" => { @@ -2429,6 +2537,15 @@ impl ColumnAccessExtractor { if let Some(val) = literal_to_filter_value(right) { self.add_hard_filter(&col_names, FilterOperator::Lte, val); } + } else if let Some((col_names, transform)) = column_transform_of(left) { + if let Some(val) = literal_to_filter_value(right) { + self.add_hard_filter_with_transform( + &col_names, + FilterOperator::Lte, + val, + Some(transform), + ); + } } } "AND" => { @@ -2558,11 +2675,86 @@ impl ColumnAccessExtractor { } } + /// An equi-comparison where exactly one side is a `%ROWTYPE` record field + /// (resolved via `resolve_record_field`) and the other a plain table column produces + /// a cross-table `JoinCondition` tagged `JoinConditionSource::RecordField`. Symmetric + /// in `left_names`/`right_names` — the record may appear on either side — so a single + /// call covers both orientations, unlike `extract_join_condition`'s two-direction + /// dedup-by-reverse pattern. Produces nothing when both or neither side resolves as a + /// record field (plain equi-joins are `extract_join_condition`'s territory; a record + /// vs. a parameter/PL-variable/unregistered-record side resolves to `None` on both + /// legs and is dropped here, never guessing a table). + fn extract_record_field_join( + &mut self, + left_names: &[ogsql_parser::Ident], + right_names: &[ogsql_parser::Ident], + join_type: &AstJoinType, + ) { + let left_record = self.resolve_record_field(left_names); + let right_record = self.resolve_record_field(right_names); + let ((record_table, record_col), plain_names) = match (left_record, right_record) { + (Some(rec), None) => (rec, right_names), + (None, Some(rec)) => (rec, left_names), + _ => return, + }; + + let (plain_alias, plain_col) = split_alias_column(plain_names); + let Some(plain_table) = plain_alias + .as_ref() + .and_then(|a| self.resolve_alias(a)) + .map(|ta| ta.table.clone()) + else { + return; + }; + + let jt = match join_type { + AstJoinType::Inner => JoinType::Inner, + AstJoinType::Left => JoinType::Left, + AstJoinType::Right => JoinType::Right, + AstJoinType::Full => JoinType::Full, + AstJoinType::Cross => JoinType::Cross, + }; + + let candidate = JoinCondition { + left_table: plain_table.clone(), + left_column: plain_col.clone(), + right_table: record_table.clone(), + right_column: record_col.clone(), + join_type: jt, + source: JoinConditionSource::RecordField, + }; + let already_exists = self.join_conditions.iter().any(|existing| { + (existing.left_table == plain_table + && existing.left_column == plain_col + && existing.right_table == record_table + && existing.right_column == record_col) + || (existing.left_table == record_table + && existing.left_column == record_col + && existing.right_table == plain_table + && existing.right_column == plain_col) + }); + if !already_exists { + self.join_conditions.push(candidate); + } + } + fn add_hard_filter( &mut self, col_names: &[ogsql_parser::Ident], op: FilterOperator, val: FilterValue, + ) { + self.add_hard_filter_with_transform(col_names, op, val, None); + } + + /// Like `add_hard_filter`, but also records the whitelisted column transform + /// (`substr`/`nvl`/`trim`/`upper`/`lower`) the column was wrapped in, if any. + fn add_hard_filter_with_transform( + &mut self, + col_names: &[ogsql_parser::Ident], + op: FilterOperator, + val: FilterValue, + transform: Option, ) { let (alias_prefix, column) = split_alias_column(col_names); let table = alias_prefix @@ -2574,6 +2766,7 @@ impl ColumnAccessExtractor { column, operator: op, value: val, + transform, }); } @@ -2653,7 +2846,7 @@ impl Visitor for ColumnAccessExtractor { PlDeclaration::Variable(v) => { use ogsql_parser::ast::plpgsql::PlDataType; // `rec cursor_name%ROWTYPE`: record fields resolve via the cursor's - // SELECT sources (issue #147 L2). `%TYPE` anchors are deliberately NOT + // SELECT sources. `%TYPE` anchors are deliberately not // resolved: typing a variable as `t.col%TYPE` says nothing about where // its value comes from, so resolving it would fabricate data edges. if let PlDataType::PercentRowType(cursor) = &v.data_type { @@ -2669,7 +2862,7 @@ impl Visitor for ColumnAccessExtractor { fn visit_pl_statement(&mut self, stmt: &PlStatement) -> VisitorResult { match stmt { // Track literal-string assignments so `OPEN c FOR v_sql` can resolve the - // dynamic cursor's SELECT sources (issue #147). + // dynamic cursor's SELECT sources. PlStatement::Assignment { target: Expr::PlVariable(names), expression, @@ -2717,7 +2910,7 @@ impl Visitor for ColumnAccessExtractor { }; let vars: Vec = fetch.node.into.iter().map(expr_var_name).collect(); self.record_fetch(&cursor_name, vars.clone()); - // Review #3: the record's data comes from the FETCHing cursor, not + // The record's data comes from the FETCHing cursor, not // its declared %ROWTYPE type anchor — rebind so `column_source` // resolves through the cursor's SELECT sources. if !cursor_name.is_empty() && vars.len() == 1 { @@ -2800,7 +2993,7 @@ impl Visitor for ColumnAccessExtractor { } } // `FOR rec IN (SELECT ...)` — the loop variable is an implicit %ROWTYPE - // record over the inline query's sources (issue #147 L2). + // record over the inline query's sources. PlStatement::For(spanned) => { use ogsql_parser::ast::plpgsql::PlForKind; if let PlForKind::Query { @@ -3122,12 +3315,12 @@ impl Visitor for ColumnAccessExtractor { } // Subqueries carry their own scope; the generic walker would otherwise // recurse into their SELECT and leak its alias/join/filter state into - // this statement's analysis (review #153-2). Exists/Subquery have no + // this statement's analysis. Exists/Subquery have no // left operand, so skipping is complete. Expr::Subquery(_) | Expr::Exists(_) => return VisitorResult::SkipChildren, // InSubquery/ScalarSublink DO have a left operand (`t.x > ANY (...)`, // `t.id IN (...)`): collect its column references first, then skip the - // nested SELECT (review 5136742683). + // nested SELECT. Expr::InSubquery { expr, .. } | Expr::ScalarSublink { expr, .. } => { self.walk_expr_for_column_refs(expr); return VisitorResult::SkipChildren; @@ -3149,45 +3342,11 @@ impl ColumnAccessExtractor { fn column_source(&self, names: &[ogsql_parser::Ident]) -> ColumnSource { let (alias_prefix, column) = split_alias_column(names); - // `%ROWTYPE` record field (issue #147 L2): `rec.id` where rec is a record + // A `%ROWTYPE` record field such as `rec.id` // resolves to the cursor's source column by output name. if let Some(record) = &alias_prefix { - if let Some(cursor) = self.record_cursors.get(&record.to_lowercase()) { - if let Some(cols) = self.cursor_sources.get(cursor) { - if let Some(col) = cols - .iter() - .find(|c| c.output_name.eq_ignore_ascii_case(&column)) - { - if !col.source_col.is_empty() { - return ColumnSource::Column { - table: col.source_table.clone(), - column: col.source_col.clone(), - }; - } - } - // #142: a single catch-all cursor source (empty output name — - // `SELECT *` cursor, or dynamic-SQL attribution) covers every - // record field: the exact column is unknown, attribute to the - // cursor's table under the field's own name (same philosophy as - // `resolve_cursor_flows`). - if let [single] = cols.as_slice() { - if single.output_name.is_empty() { - if let Some(ref t) = single.source_table { - return ColumnSource::Column { - table: Some(t.clone()), - column: column.clone(), - }; - } - } - } - } else { - // A table-anchored %ROWTYPE record has no cursor_sources entry; - // its fields are the anchor table's columns. - return ColumnSource::Column { - table: Some(cursor.clone()), - column: column.clone(), - }; - } + if let Some(source) = self.record_field_source(record, &column) { + return source; } } @@ -3198,6 +3357,71 @@ impl ColumnAccessExtractor { ColumnSource::Column { table, column } } + /// Shared record-field resolution rules, factored out of + /// `column_source` so WHERE/JOIN record-field + /// extraction) can reuse the exact same three rules without duplicating them: + /// (1) cursor-anchored record whose SELECT output name matches `column` exactly → + /// the cursor's source column; (2) a single catch-all cursor source (`SELECT *` / + /// dynamic SQL, empty output name) → the cursor's anchor table + `column`'s own + /// name; (3) table-anchored `%ROWTYPE` (no `cursor_sources` entry) → anchor table + + /// `column`. Returns `None` when `alias_prefix` is not a registered record variable, + /// or none of the three rules apply — callers fall back to their own default (never + /// a guessed table). + fn record_field_source(&self, alias_prefix: &str, column: &str) -> Option { + let cursor = self.record_cursors.get(&alias_prefix.to_lowercase())?; + let Some(cols) = self.cursor_sources.get(cursor) else { + // A table-anchored %ROWTYPE record has no cursor_sources entry; its fields + // are the anchor table's columns. + return Some(ColumnSource::Column { + table: Some(cursor.clone()), + column: column.to_string(), + }); + }; + if let Some(col) = cols + .iter() + .find(|c| c.output_name.eq_ignore_ascii_case(column)) + { + if !col.source_col.is_empty() { + return Some(ColumnSource::Column { + table: col.source_table.clone(), + column: col.source_col.clone(), + }); + } + } + // A single catch-all cursor source (empty output name from `SELECT *` + // cursor, or dynamic-SQL attribution) covers every record field: the exact + // column is unknown, attribute to the cursor's table under the field's own + // name (same philosophy as `resolve_cursor_flows`). + if let [single] = cols.as_slice() { + if single.output_name.is_empty() { + if let Some(ref t) = single.source_table { + return Some(ColumnSource::Column { + table: Some(t.clone()), + column: column.to_string(), + }); + } + } + } + None + } + + /// Resolve a `%ROWTYPE` record field appearing in a WHERE/JOIN ON + /// equi-comparison to its underlying `(table, column)` pair, via `record_field_source`. + /// Returns `None` for anything that is not a resolvable record field: a plain table + /// column, a procedure parameter/PL variable, or a record variable with no registered + /// cursor/table anchor. Never guesses a table. + fn resolve_record_field(&self, names: &[ogsql_parser::Ident]) -> Option<(String, String)> { + let (alias_prefix, column) = split_alias_column(names); + let alias_prefix = alias_prefix?; + match self.record_field_source(&alias_prefix, &column)? { + ColumnSource::Column { + table: Some(t), + column, + } => Some((t, column)), + _ => None, + } + } + /// Describe how `expr` produces a value: which inputs feed it, and whether it is a /// plain copy, a computation, or an aggregate. fn classify_value_expr(&self, expr: &Expr) -> (Vec, MappingKind, Option) { @@ -3658,7 +3882,7 @@ fn peel_parenthesized(mut expr: &Expr) -> &Expr { expr } -fn format_expr_short(expr: &Expr) -> String { +pub(crate) fn format_expr_short(expr: &Expr) -> String { match expr { Expr::ColumnRef(names) => names.join("."), Expr::ColumnRefOuterJoin(names) => format!("{}(+)", names.join(".")), @@ -3869,7 +4093,7 @@ fn format_literal_short(lit: &Literal) -> String { } } -fn split_alias_column(names: &[ogsql_parser::Ident]) -> (Option, String) { +pub(crate) fn split_alias_column(names: &[ogsql_parser::Ident]) -> (Option, String) { if names.len() >= 2 { ( Some(names[0].to_string()), @@ -3892,15 +4116,69 @@ fn split_schema_table(name: &ObjectName) -> (Option, String) { } /// Check if an expression is a ColumnRef and return the names. -fn as_column_ref(expr: &Expr) -> Option> { +pub(crate) fn as_column_ref(expr: &Expr) -> Option> { match expr { Expr::ColumnRef(names) => Some(names.clone()), _ => None, } } +/// Closed set of pure single-column transforms eligible for filter metadata. +const FILTER_TRANSFORM_WHITELIST: &[&str] = &["substr", "nvl", "trim", "upper", "lower"]; + +/// Detect a whitelisted pure column transform (`substr(col, 1, 2)`, `nvl(col, 0)`, +/// keyword-syntax `substring(col FROM 1 FOR 2)`, ...) wrapping exactly one column +/// reference, with every other argument a literal. Handles both `Expr::FunctionCall` +/// (comma syntax) and `Expr::SpecialFunction` (keyword syntax) per D5. Returns the +/// target column's raw name segments plus the transform descriptor (function name +/// lowercased; "substring" normalized to "substr"). Returns `None` when: the function +/// is not whitelisted, zero or more-than-one argument is a column reference, or any +/// other argument fails `literal_to_filter_value` (e.g. a PL variable) — the caller then +/// produces no HardFilter for that side. +pub(crate) fn column_transform_of( + expr: &Expr, +) -> Option<(Vec, FilterTransform)> { + let (raw_name, args): (String, &[Expr]) = match expr { + Expr::FunctionCall { name, args, .. } => (name.join(".").to_lowercase(), args.as_slice()), + Expr::SpecialFunction { name, args, .. } => (name.to_lowercase(), args.as_slice()), + _ => return None, + }; + let fn_name = if raw_name == "substring" { + "substr".to_string() + } else { + raw_name + }; + if !FILTER_TRANSFORM_WHITELIST.contains(&fn_name.as_str()) { + return None; + } + + let mut target: Option> = None; + let mut other_args: Vec = Vec::new(); + for arg in args { + if let Some(col_names) = as_column_ref(arg).or_else(|| match arg { + Expr::PlVariable(names) => Some(names.clone()), + _ => None, + }) { + if target.is_some() { + return None; + } + target = Some(col_names); + } else { + other_args.push(literal_to_filter_value(arg)?); + } + } + let target = target?; + Some(( + target, + FilterTransform { + fn_name, + args: other_args, + }, + )) +} + /// Convert a Literal expression to FilterValue. Returns None for non-literal (PL variables, etc). -fn literal_to_filter_value(expr: &Expr) -> Option { +pub(crate) fn literal_to_filter_value(expr: &Expr) -> Option { match expr { Expr::Literal(lit) => Some(literal_to_fv(lit)), Expr::TypeCast { expr, .. } => literal_to_filter_value(expr), @@ -3913,6 +4191,38 @@ fn literal_to_filter_value(expr: &Expr) -> Option { } } +/// Resolve a `%ROWTYPE` record field from procedure context without coupling another +/// analysis pass to `ColumnAccessExtractor`'s mutable statement state. This is the same +/// three-rule policy used by `record_field_source`: exact cursor output, one catch-all +/// cursor source, then table-anchored `%ROWTYPE`; unresolved fields are never guessed. +pub(crate) fn resolve_record_field_from_context( + ctx: &ProcedureVarContext, + names: &[ogsql_parser::Ident], +) -> Option<(String, String)> { + let (record, column) = split_alias_column(names); + let cursor = ctx.record_cursors.get(&record?.to_lowercase())?; + let Some(cols) = ctx.cursor_sources.get(cursor) else { + return Some((cursor.clone(), column)); + }; + if let Some(source) = cols + .iter() + .find(|source| source.output_name.eq_ignore_ascii_case(&column)) + { + if let Some(table) = &source.source_table { + if !source.source_col.is_empty() { + return Some((table.clone(), source.source_col.clone())); + } + } + } + match cols.as_slice() { + [source] if source.output_name.is_empty() => source + .source_table + .as_ref() + .map(|table| (table.clone(), column)), + _ => None, + } +} + fn literal_to_fv(lit: &Literal) -> FilterValue { match lit { Literal::String(s) => FilterValue::String(s.clone()), @@ -4325,7 +4635,7 @@ mod tests { ); } - // ── #140: subquery table references must be extracted regardless of the + // ── Subquery table references must be extracted regardless of the // outer statement kind (SELECT / INSERT), not just UPDATE / DELETE. ── #[test] @@ -4468,7 +4778,7 @@ mod tests { /// The CTE scope must stay active while expression subqueries are walked: /// a subquery referencing the statement's own CTE must not produce a - /// spurious table edge (#140 regression guard). + /// spurious table edge. #[test] fn subquery_referencing_cte_is_filtered() { let sql = "WITH cte AS (SELECT id FROM t_parent) SELECT COUNT(1) FROM t_main m WHERE m.id IN (SELECT id FROM cte)"; @@ -4787,6 +5097,25 @@ mod column_tests { results } + /// Like `extract_column_analysis`, but seeded with a procedure variable + /// context (cursor/record bindings that in real procedures come from the DECLARE + /// block) — needed for tests exercising record-field WHERE/JOIN ON resolution. + fn extract_column_analysis_with_context( + sql: &str, + ctx: &ProcedureVarContext, + ) -> Vec { + let tokens = Tokenizer::new(sql).tokenize().unwrap(); + let mut parser = ogsql_parser::Parser::with_source(tokens, sql.to_string()); + let stmts = parser.parse_with_text(); + let mut results = Vec::new(); + for info in &stmts { + let mut extractor = ColumnAccessExtractor::new_with_context(ctx); + walk_statement(&mut extractor, &info.statement); + results.push(extractor.finish()); + } + results + } + fn find_column_ref<'a>(refs: &'a [ColumnRef], col: &str) -> Option<&'a ColumnRef> { refs.iter().find(|r| r.column == col) } @@ -4836,6 +5165,209 @@ mod column_tests { assert_eq!(&hf.value, &FilterValue::String("active".to_string())); } + // ── Record-field cross-table joins ───────────────────────────────────── + + /// A record field on the right of a WHERE equi-comparison resolves through + /// the cursor's SELECT source, producing a cross-table `JoinCondition` tagged + /// `RecordField` (par_sys_purchase.security_id ↔ mid_yjqs_detail.security_id). + #[test] + fn record_field_in_where_resolves_to_cross_table_join() { + let mut ctx = ProcedureVarContext::default(); + ctx.cursor_sources.insert( + "c_get_data".to_string(), + vec![ + CursorColumn { + output_name: "security_id".to_string(), + source_table: Some("mid_yjqs_detail".to_string()), + source_col: "security_id".to_string(), + }, + CursorColumn { + output_name: "fund_code".to_string(), + source_table: Some("mid_yjqs_detail".to_string()), + source_col: "fund_code".to_string(), + }, + ], + ); + ctx.record_cursors + .insert("r_get_purchase".to_string(), "c_get_data".to_string()); + + let analyses = extract_column_analysis_with_context( + "SELECT t.purchase_days INTO v_purchase_days FROM par_sys_purchase t \ + WHERE t.security_id = r_get_purchase.security_id", + &ctx, + ); + assert_eq!(analyses.len(), 1); + let jc = analyses[0] + .join_conditions + .iter() + .find(|jc| jc.source == JoinConditionSource::RecordField) + .unwrap_or_else(|| { + panic!( + "no RecordField join condition in {:#?}", + analyses[0].join_conditions + ) + }); + assert_eq!(jc.left_table, "par_sys_purchase"); + assert_eq!(jc.left_column, "security_id"); + assert_eq!(jc.right_table, "mid_yjqs_detail"); + assert_eq!(jc.right_column, "security_id"); + } + + /// The record field may appear on either side of `=`; the resolved join must + /// be the same regardless of source order. + #[test] + fn record_field_join_works_in_both_orientations() { + let mut ctx = ProcedureVarContext::default(); + ctx.cursor_sources.insert( + "c_get_data".to_string(), + vec![CursorColumn { + output_name: "fund_code".to_string(), + source_table: Some("mid_yjqs_detail".to_string()), + source_col: "fund_code".to_string(), + }], + ); + ctx.record_cursors + .insert("r_get_purchase".to_string(), "c_get_data".to_string()); + + let analyses = extract_column_analysis_with_context( + "SELECT t.purchase_days INTO v_purchase_days FROM par_sys_purchase t \ + WHERE r_get_purchase.fund_code = t.fund_code", + &ctx, + ); + assert_eq!(analyses.len(), 1); + let jc = analyses[0] + .join_conditions + .iter() + .find(|jc| jc.source == JoinConditionSource::RecordField) + .unwrap_or_else(|| { + panic!( + "no RecordField join condition in {:#?}", + analyses[0].join_conditions + ) + }); + assert_eq!(jc.left_table, "par_sys_purchase"); + assert_eq!(jc.left_column, "fund_code"); + assert_eq!(jc.right_table, "mid_yjqs_detail"); + assert_eq!(jc.right_column, "fund_code"); + } + + /// A plain `ON a.id = b.id` equi-join remains unchanged, with no + /// `RecordField` rows sneak in when neither side is a record. + #[test] + fn plain_on_equi_join_unchanged() { + let sql = "SELECT a.id FROM table_a a JOIN table_b b ON a.id = b.id"; + let analyses = extract_column_analysis(sql); + assert_eq!(analyses.len(), 1); + let a = &analyses[0]; + + assert_eq!(a.join_conditions.len(), 1); + let jc = &a.join_conditions[0]; + assert_eq!(jc.left_table, "table_a"); + assert_eq!(jc.left_column, "id"); + assert_eq!(jc.right_table, "table_b"); + assert_eq!(jc.right_column, "id"); + assert_eq!(jc.source, JoinConditionSource::ExplicitOn); + } + + /// A record vs. a procedure parameter/PL variable, and a record vs. an + /// unregistered record variable, must both produce no join — never guess a table. + #[test] + fn record_vs_param_or_unregistered_produces_no_join() { + let mut ctx = ProcedureVarContext::default(); + ctx.cursor_sources.insert( + "c_get_data".to_string(), + vec![CursorColumn { + output_name: "security_id".to_string(), + source_table: Some("mid_yjqs_detail".to_string()), + source_col: "security_id".to_string(), + }], + ); + ctx.record_cursors + .insert("r".to_string(), "c_get_data".to_string()); + let analyses = extract_column_analysis_with_context( + "SELECT t.id FROM par_sys_purchase t WHERE r.security_id = p_i_date", + &ctx, + ); + assert!( + analyses[0].join_conditions.is_empty(), + "record vs. param/PL variable must not produce a join: {:#?}", + analyses[0].join_conditions + ); + + let analyses2 = extract_column_analysis( + "SELECT t.purchase_days FROM par_sys_purchase t \ + WHERE t.security_id = r_unregistered.security_id", + ); + assert!( + analyses2[0].join_conditions.is_empty(), + "unregistered record variable must not produce a join (no table guessing): {:#?}", + analyses2[0].join_conditions + ); + } + + /// Table-anchored `%ROWTYPE` (no `cursor_sources` entry) resolves from its type + /// is a table, not a cursor) resolves through the WHERE/JOIN path too. + #[test] + fn table_anchored_rowtype_resolves_in_where() { + let mut ctx = ProcedureVarContext::default(); + ctx.record_cursors + .insert("r".to_string(), "t_src".to_string()); + let analyses = extract_column_analysis_with_context( + "SELECT t.id FROM par_sys_purchase t WHERE t.id = r.id", + &ctx, + ); + let jc = analyses[0] + .join_conditions + .iter() + .find(|jc| jc.source == JoinConditionSource::RecordField) + .unwrap_or_else(|| { + panic!( + "no RecordField join condition in {:#?}", + analyses[0].join_conditions + ) + }); + assert_eq!(jc.left_table, "par_sys_purchase"); + assert_eq!(jc.left_column, "id"); + assert_eq!(jc.right_table, "t_src"); + assert_eq!(jc.right_column, "id"); + } + + /// A `SELECT *` cursor's single catch-all source (empty output name) + /// resolves through the WHERE/JOIN path too, attributing to the cursor's table + /// under the field's own name. + #[test] + fn star_cursor_catch_all_resolves_in_where() { + let mut ctx = ProcedureVarContext::default(); + ctx.cursor_sources.insert( + "cur".to_string(), + vec![CursorColumn { + output_name: String::new(), + source_table: Some("t_src".to_string()), + source_col: String::new(), + }], + ); + ctx.record_cursors + .insert("r".to_string(), "cur".to_string()); + let analyses = extract_column_analysis_with_context( + "SELECT t.id FROM par_sys_purchase t WHERE t.id = r.id", + &ctx, + ); + let jc = analyses[0] + .join_conditions + .iter() + .find(|jc| jc.source == JoinConditionSource::RecordField) + .unwrap_or_else(|| { + panic!( + "no RecordField join condition in {:#?}", + analyses[0].join_conditions + ) + }); + assert_eq!(jc.left_table, "par_sys_purchase"); + assert_eq!(jc.left_column, "id"); + assert_eq!(jc.right_table, "t_src"); + assert_eq!(jc.right_column, "id"); + } + #[test] fn test_insert_columns() { let sql = "INSERT INTO t_log(product_id, delta, reason) VALUES (1, -5, 'RESERVE')"; @@ -5068,7 +5600,164 @@ mod column_tests { ); } - // ── Column mappings (#136) ──────────────────────────────────────────────── + // ── Function-wrapped column filters ───────────────────────────────────── + + /// A whitelisted pure column transform compared against a literal yields a + /// HardFilter on the underlying column, with a transform descriptor. + #[test] + fn substr_wrapped_column_literal_becomes_hard_filter_with_transform() { + let sql = "SELECT qs.stock_kind FROM t_quote_snapshot qs WHERE substr(qs.stock_kind, 1, 2) = '05'"; + let analyses = extract_column_analysis(sql); + assert_eq!(analyses.len(), 1); + let a = &analyses[0]; + let hf = find_hard_filter(&a.hard_filters, "stock_kind").expect("stock_kind filter"); + assert_eq!(hf.table, Some("t_quote_snapshot".to_string())); + assert_eq!(hf.operator, FilterOperator::Eq); + assert_eq!(hf.value, FilterValue::String("05".to_string())); + assert_eq!( + hf.transform, + Some(FilterTransform { + fn_name: "substr".to_string(), + args: vec![FilterValue::Integer(1), FilterValue::Integer(2)], + }) + ); + } + + /// Transformed and plain filters coexist in mixed cursor conditions. + #[test] + fn step3_cursor_mixed_filters_all_captured() { + let sql = "SELECT qs.stock_kind FROM t_quote_snapshot qs WHERE substr(qs.stock_kind,1,2)='05' AND qs.stock_kind <> '0509' AND qs.scdm = '001' AND qs.cjsl > 0"; + let analyses = extract_column_analysis(sql); + assert_eq!(analyses.len(), 1); + let a = &analyses[0]; + assert_eq!( + a.hard_filters.len(), + 4, + "expected 4 hard filters, got: {:?}", + a.hard_filters + ); + + let transformed = find_hard_filter(&a.hard_filters, "stock_kind").expect("stock_kind"); + assert!(transformed.transform.is_some()); + assert_eq!(transformed.operator, FilterOperator::Eq); + assert_eq!(transformed.value, FilterValue::String("05".to_string())); + + let scdm = find_hard_filter(&a.hard_filters, "scdm").expect("scdm"); + assert_eq!(scdm.transform, None); + let cjsl = find_hard_filter(&a.hard_filters, "cjsl").expect("cjsl"); + assert_eq!(cjsl.transform, None); + + let neq_filters: Vec<_> = a + .hard_filters + .iter() + .filter(|f| f.operator == FilterOperator::Neq) + .collect(); + assert_eq!(neq_filters.len(), 1); + assert_eq!(neq_filters[0].transform, None); + } + + /// Non-literal extra arguments exclude the filter. + #[test] + fn substr_with_variable_length_arg_is_excluded() { + let sql = "SELECT col FROM t WHERE substr(col, 1, v_len) = '05'"; + let analyses = extract_column_analysis(sql); + assert_eq!(analyses.len(), 1); + let a = &analyses[0]; + assert!( + a.hard_filters.is_empty(), + "substr with variable length arg should not produce a hard filter, got: {:?}", + a.hard_filters + ); + } + + /// Non-whitelisted functions and function-to-function comparisons stay excluded. + #[test] + fn non_whitelisted_or_double_sided_function_is_excluded() { + let sql1 = "SELECT col FROM t WHERE fnc_x(col) = '1'"; + let analyses1 = extract_column_analysis(sql1); + assert!( + analyses1[0].hard_filters.is_empty(), + "fnc_x is not whitelisted" + ); + + let sql2 = "SELECT a, b FROM t WHERE nvl(a,1) = nvl(b,2)"; + let analyses2 = extract_column_analysis(sql2); + assert!( + analyses2[0].hard_filters.is_empty(), + "func-vs-func comparison should not produce a hard filter" + ); + } + + /// Keyword-style special functions use the same transform handling. + #[test] + fn substr_keyword_syntax_produces_transform() { + let sql = "SELECT col FROM t WHERE substring(col FROM 1 FOR 2) = '05'"; + let analyses = extract_column_analysis(sql); + assert_eq!(analyses.len(), 1); + let a = &analyses[0]; + let hf = find_hard_filter(&a.hard_filters, "col").expect("col filter"); + assert_eq!( + hf.transform, + Some(FilterTransform { + fn_name: "substr".to_string(), + args: vec![FilterValue::Integer(1), FilterValue::Integer(2)], + }) + ); + } + + /// An `nvl` transform includes its default-value argument. + #[test] + fn nvl_transform_includes_default_arg() { + let sql = "SELECT col FROM t WHERE nvl(col, '0') = '1'"; + let analyses = extract_column_analysis(sql); + assert_eq!(analyses.len(), 1); + let a = &analyses[0]; + let hf = find_hard_filter(&a.hard_filters, "col").expect("col filter"); + assert_eq!( + hf.transform, + Some(FilterTransform { + fn_name: "nvl".to_string(), + args: vec![FilterValue::String("0".to_string())], + }) + ); + } + + /// Plain filters omit the `transform` key in JSON. + #[test] + fn plain_filter_json_has_no_transform_key_but_transformed_filter_does() { + let plain = HardFilter { + table: Some("t".to_string()), + column: "status".to_string(), + operator: FilterOperator::Eq, + value: FilterValue::String("active".to_string()), + transform: None, + }; + let json = serde_json::to_string(&plain).unwrap(); + assert!( + !json.contains("transform"), + "plain filter JSON must omit transform key, got: {}", + json + ); + + let transformed = HardFilter { + table: Some("t".to_string()), + column: "stock_kind".to_string(), + operator: FilterOperator::Eq, + value: FilterValue::String("05".to_string()), + transform: Some(FilterTransform { + fn_name: "substr".to_string(), + args: vec![FilterValue::Integer(1), FilterValue::Integer(2)], + }), + }; + let json2 = serde_json::to_string(&transformed).unwrap(); + assert!( + json2.contains(r#""transform":{"fn":"substr","args":["#), + "transformed filter JSON must contain a transform.fn key, got: {}", + json2 + ); + } + + // ── Column mappings ───────────────────────────────────────────────────── fn column_mappings_of(sql: &str) -> Vec { extract_column_analysis(sql) @@ -5077,7 +5766,7 @@ mod column_tests { .collect() } - /// Column mappings with a seeded procedure variable context (#142): lets a + /// Column mappings with a seeded procedure variable context let a /// standalone INSERT walk see cursor/record bindings that in real procedures /// come from the DECLARE block. fn column_mappings_of_with_context(sql: &str, ctx: &ProcedureVarContext) -> Vec { @@ -5273,7 +5962,7 @@ mod column_tests { ); } - /// #142: a scalar subquery as an INSERT..SELECT target contributes the inner + /// A scalar subquery as an INSERT..SELECT target contributes the inner /// select's FIRST expression as the source, resolved in the subquery's own FROM /// scope. Correlated refs (`s.id` in WHERE) must NOT leak as sources. #[test] @@ -5287,7 +5976,7 @@ mod column_tests { assert_eq!(m.sources, vec![col(Some("t_ref"), "code")]); } - /// #142: the choke point is push_column_mapping, so INSERT..VALUES subqueries + /// The choke point is `push_column_mapping`, so INSERT..VALUES subqueries /// resolve too. #[test] fn scalar_subquery_in_insert_values_resolves() { @@ -5300,7 +5989,7 @@ mod column_tests { ); } - /// Review #1: a scalar subquery whose first expression is a TRANSFORMED column + /// A scalar subquery whose first expression is a transformed column /// (`UPPER`, `+1`, `NVL`, `CAST`, …) must classify as Derived and keep the /// expression text — not masquerade as a Direct copy. #[test] @@ -5319,7 +6008,7 @@ mod column_tests { ); } - /// Review #1: a literal-only scalar subquery is a constant → Direct + Literal + /// A literal-only scalar subquery is a constant source. /// source, consistent with how `classify_value_expr` treats a bare literal. #[test] fn literal_only_scalar_subquery_classifies_direct() { @@ -5334,7 +6023,7 @@ mod column_tests { ); } - /// Review (5136742683): a multi-column subquery applied to a multi-column + /// A multi-column subquery applied to a multi-column /// UPDATE SET target must align each target column to its OWN select-list /// position — not copy the first expression into every column. #[test] @@ -5351,7 +6040,7 @@ mod column_tests { ); } - /// Review #2: a JOIN inside the scalar subquery's FROM must not leak its + /// A join inside the scalar subquery's FROM must not leak its /// join/filter state into the enclosing statement's analysis — the subquery /// carries its own scope. #[test] @@ -5369,7 +6058,7 @@ mod column_tests { ); } - /// Review (5136742683): `t.x > ANY (SELECT ...)` — the left operand `t.x` is a + /// In `t.x > ANY (SELECT ...)`, the left operand is a /// real column reference of the enclosing query and must still be collected; /// only the nested SELECT's own scope must be skipped. #[test] @@ -5391,7 +6080,7 @@ mod column_tests { assert_eq!(x_refs[0].resolved_table.as_deref(), Some("t")); } - /// Review (5136742683): the left operand of `IN (SELECT ...)` in an ON clause + /// The left operand of `IN (SELECT ...)` in an ON clause /// must still be collected (the generic walker was its only collector). #[test] fn in_subquery_left_operand_in_join_condition_is_collected() { @@ -5411,7 +6100,7 @@ mod column_tests { ); } - /// #142: a `rec t%ROWTYPE` record (anchor is a TABLE, not a registered cursor) + /// A `rec t%ROWTYPE` record anchored to a table /// resolves its fields to that table's columns. #[test] fn table_rowtype_record_field_resolves_to_table_column() { @@ -5432,7 +6121,7 @@ mod column_tests { ); } - /// #142: a `SELECT *` cursor produces a single catch-all cursor source (empty + /// A `SELECT *` cursor produces a single catch-all cursor source (empty /// output name, table attributed). Record fields over it attribute to the /// cursor's table under the field's own name. #[test] @@ -5462,7 +6151,7 @@ mod column_tests { ); } - /// #142: `INSERT INTO t (a, b) VALUES r` with a cursor-anchored %ROWTYPE record + /// `INSERT INTO t (a, b) VALUES r` with a cursor-anchored `%ROWTYPE` record /// expands the record's fields positionally through the cursor's SELECT sources. #[test] fn whole_record_insert_expands_cursor_rowtype_fields() { @@ -5495,7 +6184,7 @@ mod column_tests { ); } - /// Review #5: whole-record insert from a `SELECT *` cursor has no exact column + /// Whole-record insert from a `SELECT *` cursor has no exact column /// names — attributing each INSERT column under its own name would silently /// misattribute a reordered column list. Leave such mappings unmapped instead. #[test] @@ -5518,7 +6207,7 @@ mod column_tests { ); } - /// Review #3: a `%ROWTYPE` record's data comes from the FETCH that fills it. + /// A `%ROWTYPE` record's data comes from the FETCH that fills it. /// `r t_type%ROWTYPE` + `FETCH cur INTO r` (cur reads t_other) must resolve /// `r.id` to t_other.id, not the declared type table. #[test] @@ -5540,7 +6229,7 @@ mod column_tests { ); } - /// Review #4: a %ROWTYPE record field as a scalar subquery's first expression + /// A `%ROWTYPE` record field as a scalar subquery's first expression /// penetrates through record_cursors to the cursor's source column — ogsql-parser /// parses `rec.field` as a dotted ColumnRef, which column_source already resolves. #[test] diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 88b2b55..89dd18d 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -10,6 +10,7 @@ pub mod jsp_preprocessor; #[cfg(feature = "jsp")] pub mod jsp_types; mod loader; +mod predicates; pub mod scanner; pub mod snippet; @@ -17,10 +18,10 @@ pub mod snippet; pub use extractor::{ extract_body_sql, pl_type_decl_name, CallEdge, CallExtractor, ColumnAccessExtractor, ColumnAnalysis, ColumnContext, ColumnMapping, ColumnRef, ColumnSource, CursorColumn, - EnumMapping, FilterOperator, FilterValue, HardFilter, InsertColumnInfo, JoinCondition, - JoinConditionSource, JoinType, MappingKind, ProcedureBodySql, ProcedureSqlExtractor, - ProcedureVarContext, SelectIntoMapping, SequenceRef, SequenceRefVia, TableAccessExtractor, - TableAlias, TypeRef, TypeSequenceRefExtractor, UpdateColumnInfo, + EnumMapping, FilterOperator, FilterTransform, FilterValue, HardFilter, InsertColumnInfo, + JoinCondition, JoinConditionSource, JoinType, MappingKind, ProcedureBodySql, + ProcedureSqlExtractor, ProcedureVarContext, SelectIntoMapping, SequenceRef, SequenceRefVia, + TableAccessExtractor, TableAlias, TypeRef, TypeSequenceRefExtractor, UpdateColumnInfo, }; #[allow(unused_imports)] pub use ibatis_loader::{ @@ -38,4 +39,9 @@ pub use java_method::{ }; pub use loader::{load_all_files, load_sql_files, parse_sql_files, AllParsedFiles, ParsedFile}; #[allow(unused_imports)] +pub use predicates::{ + extract_predicates, Confidence, ParamTableHint, PlPredicate, PredicateClause, + PredicateExtractor, PredicateKind, TablePredicate, +}; +#[allow(unused_imports)] pub use scanner::{build_exclude_matcher, scan_directory, ScannedFiles}; diff --git a/src/parser/predicates.rs b/src/parser/predicates.rs new file mode 100644 index 0000000..489d509 --- /dev/null +++ b/src/parser/predicates.rs @@ -0,0 +1,1102 @@ +//! PL/SQL branch predicates extracted from `IF` and `CASE WHEN` conditions. + +use super::extractor::{ + as_column_ref, column_transform_of, format_expr_short, literal_to_filter_value, + resolve_record_field_from_context, split_alias_column, +}; +use super::{FilterOperator, FilterTransform, FilterValue, ProcedureVarContext}; +use ogsql_parser::ast::plpgsql::{PlBlock, PlStatement}; +use ogsql_parser::ast::{Expr, SelectStatement, SelectTarget, Statement, TableRef}; +use ogsql_parser::{Visitor, VisitorResult}; +use std::collections::{BTreeMap, HashMap, HashSet}; + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct PlPredicate { + pub id: String, + pub line: usize, + pub origin: String, + pub kind: PredicateKind, + pub confidence: Confidence, + pub table_predicate: Option, + pub needs_review: Option, + pub param_table_hint: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub enum PredicateKind { + If, + CaseWhen, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub enum Confidence { + High, + Medium, + Low, +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct TablePredicate { + pub table: String, + pub clauses: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Deserialize)] +pub struct PredicateClause { + pub column: String, + pub op: FilterOperator, + pub value: FilterValue, + /// A pure transform applied before comparison, preventing transformed equality + /// from being mistaken for exact column equality. + #[serde(default)] + pub transform: Option, +} + +/// Human-readable serializers may omit `transform`, while bincode requires a fixed field count. +impl serde::Serialize for PredicateClause { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let omit_transform = serializer.is_human_readable() && self.transform.is_none(); + let field_count = if omit_transform { 3 } else { 4 }; + let mut state = serializer.serialize_struct("PredicateClause", field_count)?; + state.serialize_field("column", &self.column)?; + state.serialize_field("op", &self.op)?; + state.serialize_field("value", &self.value)?; + if !omit_transform { + state.serialize_field("transform", &self.transform)?; + } + state.end() + } +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct ParamTableHint { + pub table: String, + pub filters: Vec, + pub set: Vec<(String, FilterValue)>, +} + +pub struct PredicateExtractor<'a> { + ctx: &'a ProcedureVarContext, + predicates: Vec, + var_sources: HashMap, +} + +#[derive(Debug, Clone)] +struct VarSource { + table: String, + column: String, + filters: Vec, + role: VarSourceRole, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum VarSourceRole { + Main, + Parameter, +} + +#[derive(Debug)] +enum ConditionResolution { + Direct(Vec<(String, PredicateClause)>), + Derived(Vec<(VarSource, PredicateClause)>), +} + +impl<'a> PredicateExtractor<'a> { + pub fn new_with_context(ctx: &'a ProcedureVarContext) -> Self { + Self { + ctx, + predicates: Vec::new(), + var_sources: HashMap::new(), + } + } + + pub fn finish(self) -> Vec { + self.predicates + } + + fn push_condition(&mut self, condition: &Expr, kind: PredicateKind, line: usize) { + let resolved = condition_clauses(condition, self.ctx, &self.var_sources); + let id = format!("B{:03}", self.predicates.len() + 1); + let origin = format!( + "{} {}", + match kind { + PredicateKind::If => "IF", + PredicateKind::CaseWhen => "WHEN", + }, + format_condition(condition) + ); + let (confidence, table_predicate, needs_review, param_table_hint) = match resolved { + Some(ConditionResolution::Direct(clauses)) => { + let table_predicate = one_table_predicate(clauses); + if table_predicate.is_some() { + (Confidence::High, table_predicate, None, None) + } else { + ( + Confidence::Low, + None, + Some("condition spans multiple or unresolved tables".to_string()), + None, + ) + } + } + Some(ConditionResolution::Derived(clauses)) => { + let first = clauses.first().map(|(source, _)| source.clone()); + let same_source = first.as_ref().is_some_and(|source| { + clauses.iter().all(|(candidate, _)| { + candidate.table.eq_ignore_ascii_case(&source.table) + && candidate.role == source.role + }) + }); + match (first, same_source) { + (Some(source), true) if source.role == VarSourceRole::Main => ( + Confidence::Medium, + Some(TablePredicate { + table: source.table, + clauses: clauses.into_iter().map(|(_, clause)| clause).collect(), + }), + Some("predicate derived through a SELECT INTO variable".to_string()), + None, + ), + (Some(source), true) => ( + Confidence::Low, + None, + Some("condition derives from a parameter/dimension table".to_string()), + Some(ParamTableHint { + table: source.table, + filters: source.filters, + set: clauses + .into_iter() + .map(|(_, clause)| (clause.column, clause.value)) + .collect(), + }), + ), + _ => ( + Confidence::Low, + None, + Some("condition has mixed SELECT INTO sources".to_string()), + None, + ), + } + } + None => ( + Confidence::Low, + None, + Some("condition could not be resolved to one table with certainty".to_string()), + None, + ), + }; + self.predicates.push(PlPredicate { + id, + line, + origin, + kind, + confidence, + table_predicate, + needs_review, + param_table_hint, + }); + } + + fn record_select_into(&mut self, select: &SelectStatement) { + let Some(into_targets) = &select.into_targets else { + return; + }; + let aliases = table_aliases(&select.from); + let sole_table = sole_table(&aliases); + let filters = select + .where_clause + .as_ref() + .and_then(|expr| { + direct_clauses(expr, self.ctx, Some((&aliases, sole_table.as_deref()))) + }) + .and_then(one_table_predicate) + .map(|predicate| predicate.clauses) + .unwrap_or_default(); + + // Report mismatched targets because `zip` necessarily drops extras. + if select.targets.len() != into_targets.len() { + crate::parse_log::warn( + "predicates", + &format!( + "SELECT INTO target/variable count mismatch ({} SELECT targets vs {} INTO \ + variables) — extra entries are dropped in predicate extraction; statement: {}", + select.targets.len(), + into_targets.len(), + select + .raw_body + .as_deref() + .unwrap_or("